diff options
Diffstat (limited to 'src')
42 files changed, 7873 insertions, 4973 deletions
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index b0f80db32..832235cfd 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -70,7 +70,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); if (a == v->attrs->end()) - throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); + throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; } @@ -82,7 +82,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, % attrPath % showType(*v)); if (attrIndex >= v->listSize()) - throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 46a341950..1eae64625 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -7,6 +7,8 @@ namespace nix { +MakeError(AttrPathNotFound, Error); + Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 0785897d2..b1d61a285 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } +Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +{ + return allocAttr(vAttrs, symbols.create(name)); +} + + void Bindings::sort() { std::sort(begin(), end()); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 3119a1848..6c5fb21ad 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -4,6 +4,7 @@ #include "symbol-table.hh" #include <algorithm> +#include <optional> namespace nix { @@ -63,6 +64,14 @@ public: return end(); } + std::optional<Attr *> get(const Symbol & name) + { + Attr key(name, 0); + iterator i = std::lower_bound(begin(), end(), key); + if (i != end() && i->name == name) return &*i; + return {}; + } + iterator begin() { return &attrs[0]; } iterator end() { return &attrs[size_]; } diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f28..7c0d268bd 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -26,6 +26,22 @@ MixEvalArgs::MixEvalArgs() .description("add a path to the list of locations used to look up <...> file names") .label("path") .handler([&](std::string s) { searchPath.push_back(s); }); + + mkFlag() + .longName("impure") + .description("allow access to mutable paths and repositories") + .handler([&](std::vector<std::string> ss) { + evalSettings.pureEval = false; + }); + + mkFlag() + .longName("override-flake") + .labels({"original-ref", "resolved-ref"}) + .description("override a flake registry value") + .arity(2) + .handler([&](std::vector<std::string> ss) { + registryOverrides.push_back(std::make_pair(ss[0], ss[1])); + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) @@ -45,9 +61,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh index be7fda783..54fb731de 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libexpr/common-eval-args.hh @@ -16,6 +16,8 @@ struct MixEvalArgs : virtual Args Strings searchPath; + std::vector<std::pair<std::string, std::string>> registryOverrides; + private: std::map<std::string, std::string> autoArgs; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8e10d9f2..0f8a105b1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,6 +7,7 @@ #include "eval-inline.hh" #include "download.hh" #include "json.hh" +#include "primops/flake.hh" #include <algorithm> #include <cstring> @@ -302,6 +303,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) + , sDescription(symbols.create("description")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) @@ -451,14 +453,21 @@ Value * EvalState::addConstant(const string & name, Value & v) Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { + auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + + /* Hack to make constants lazy: turn them into a application of + the primop to a dummy value. */ if (arity == 0) { + auto vPrimOp = allocValue(); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = new PrimOp(primOp, 1, sym); Value v; - primOp(*this, noPos, nullptr, v); + mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } + Value * v = allocValue(); - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - Symbol sym = symbols.create(name2); v->type = tPrimOp; v->primOp = new PrimOp(primOp, arity, sym); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a314e01e0..46c6ea271 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,6 +18,10 @@ class Store; class EvalState; enum RepairFlag : bool; +namespace flake { +struct FlakeRegistry; +} + typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -62,6 +66,8 @@ typedef std::list<SearchPathElem> SearchPath; /* Initialise the Boehm GC, if applicable. */ void initGC(); +typedef std::vector<std::pair<std::string, std::string>> RegistryOverrides; + class EvalState { @@ -72,7 +78,8 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sOutputHash, sOutputHashAlgo, sOutputHashMode; + sOutputHash, sOutputHashAlgo, sOutputHashMode, + sDescription; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -87,6 +94,9 @@ public: const ref<Store> store; + RegistryOverrides registryOverrides; + + private: SrcToStore srcToStore; @@ -209,6 +219,8 @@ public: path. Nothing is copied to the store. */ Path coerceToPath(const Pos & pos, Value & v, PathSet & context); + void addRegistryOverrides(RegistryOverrides overrides) { registryOverrides = overrides; } + public: /* The base environment, containing the builtin functions and @@ -264,6 +276,7 @@ public: Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); + Value * allocAttr(Value & vAttrs, const std::string & name); Bindings * allocBindings(size_t capacity); @@ -310,6 +323,16 @@ private: friend struct ExprOpConcatLists; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); + +public: + + const std::vector<std::shared_ptr<flake::FlakeRegistry>> getFlakeRegistries(); + + std::shared_ptr<flake::FlakeRegistry> getGlobalFlakeRegistry(); + +private: + std::shared_ptr<flake::FlakeRegistry> _globalFlakeRegistry; + std::once_flag _globalFlakeRegistryInit; }; @@ -349,6 +372,9 @@ struct EvalSettings : Config Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting<std::string> flakeRegistry{this, "https://raw.githubusercontent.com/NixOS/flake-registry/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 78a503907..967c88d9b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -677,7 +677,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 06f577f36..070e72f3a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2050,9 +2050,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, const std::string & defaultName) { - string url; - Hash expectedHash; - string name = defaultName; + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; state.forceValue(*args[0]); @@ -2063,27 +2063,27 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, for (auto & attr : *args[0]->attrs) { string n(attr.name); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); } - if (url.empty()) + if (request.uri.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(url); + state.checkURI(request.uri); - if (evalSettings.pureEval && !expectedHash) + if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + Path res = getDownloader()->downloadCached(state.store, request).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index aaf02c856..10f6b6f72 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -1,3 +1,4 @@ +#include "fetchGit.hh" #include "primops.hh" #include "eval-inline.hh" #include "download.hh" @@ -15,24 +16,21 @@ using namespace std::string_literals; namespace nix { -struct GitInfo -{ - Path storePath; - std::string rev; - std::string shortRev; - uint64_t revCount = 0; -}; - -std::regex revRegex("^[0-9a-fA-F]{40}$"); +extern std::regex revRegex; -GitInfo exportGit(ref<Store> store, const std::string & uri, - std::optional<std::string> ref, std::string rev, +GitInfo exportGit(ref<Store> store, std::string uri, + std::optional<std::string> ref, + std::optional<Hash> rev, const std::string & name) { - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + assert(!rev || rev->type == htSHA1); - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { + bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git"); + + // If this is a local directory (but not a file:// URI) and no ref + // or revision is given, then allow the use of an unclean working + // tree. + if (!ref && !rev && isLocal) { bool clean = true; @@ -49,8 +47,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, files. */ GitInfo gitInfo; - gitInfo.rev = "0000000000000000000000000000000000000000"; - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + gitInfo.ref = "HEAD"; auto files = tokenizeString<std::set<std::string>>( runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); @@ -71,90 +68,124 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, }; gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })); + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" })); return gitInfo; } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); - ref = "HEAD"s; } - if (!ref) ref = "HEAD"s; + if (!ref) ref = isLocal ? "HEAD" : "master"; - if (rev != "" && !std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); + // Don't clone file:// URIs (but otherwise treat them the same as + // remote URIs, i.e. don't use the working tree or HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + if (!forceHttp && hasPrefix(uri, "file://")) { + uri = std::string(uri, 7); + isLocal = true; + } deletePath(getCacheDir() + "/nix/git"); + deletePath(getCacheDir() + "/nix/gitv2"); - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, uri).to_string(Base32, false); + Path repoDir; - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } + if (isLocal) { - Path localRefFile = cacheDir + "/refs/heads/" + *ref; + if (!rev) + rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", *ref })), htSHA1); - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev != "") { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; + if (!pathExists(cacheDir)) + createDirs(cacheDir); + + repoDir = uri; + + } else { + + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = repoDir + "/refs/heads/" + *ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + st.st_mtime + settings.tarballTtl <= now; } - } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", uri); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } - utimes(localRefFile.c_str(), times); + if (!rev) + rev = Hash(chomp(readFile(localRefFile)), htSHA1); } // FIXME: check whether rev is an ancestor of ref. GitInfo gitInfo; - gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + gitInfo.ref = *ref; + gitInfo.rev = *rev; printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + std::string storeLinkName = hashString(htSHA512, + name + std::string("\0"s) + gitInfo.rev.gitRev()).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken try { auto json = nlohmann::json::parse(readFile(storeLink)); - assert(json["name"] == name && json["rev"] == gitInfo.rev); + assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == gitInfo.rev); - gitInfo.storePath = json["storePath"]; + Path storePath = json["storePath"]; - if (store->isValidPath(gitInfo.storePath)) { + if (store->isValidPath(storePath)) { + gitInfo.storePath = storePath; gitInfo.revCount = json["revCount"]; + gitInfo.lastModified = json["lastModified"]; return gitInfo; } @@ -164,7 +195,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, // FIXME: should pipe this, or find some better way to extract a // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + auto tar = runProgram("git", true, { "-C", repoDir, "archive", gitInfo.rev.gitRev() }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -173,14 +204,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, gitInfo.storePath = store->addToStore(name, tmpDir); - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); + gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "show", "-s", "--format=%ct", gitInfo.rev.gitRev() })); nlohmann::json json; json["storePath"] = gitInfo.storePath; json["uri"] = uri; json["name"] = name; - json["rev"] = gitInfo.rev; + json["rev"] = gitInfo.rev.gitRev(); json["revCount"] = gitInfo.revCount; + json["lastModified"] = gitInfo.lastModified; writeFile(storeLink, json.dump()); @@ -191,7 +224,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va { std::string url; std::optional<std::string> ref; - std::string rev; + std::optional<Hash> rev; std::string name = "source"; PathSet context; @@ -208,7 +241,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -225,17 +258,20 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + auto gitInfo = exportGit(state.store, url, ref, rev, name); state.mkAttrs(v, 8); mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); + mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.rev.gitShortRev()); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(gitInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh new file mode 100644 index 000000000..006fa8b5f --- /dev/null +++ b/src/libexpr/primops/fetchGit.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "store-api.hh" + +#include <regex> + +namespace nix { + +struct GitInfo +{ + Path storePath; + std::string ref; + Hash rev{htSHA1}; + uint64_t revCount; + time_t lastModified; +}; + +GitInfo exportGit(ref<Store> store, std::string uri, + std::optional<std::string> ref, + std::optional<Hash> rev, + const std::string & name); + +} diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 66f49f374..596047ce3 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -27,9 +27,6 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); HgInfo exportMercurial(ref<Store> store, const std::string & uri, std::string rev, const std::string & name) { - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; @@ -203,6 +200,9 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // whitelist. Ah well. state.checkURI(url); + if (evalSettings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + auto hgInfo = exportMercurial(state.store, url, rev, name); state.mkAttrs(v, 8); @@ -214,7 +214,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(hgInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc new file mode 100644 index 000000000..235e10922 --- /dev/null +++ b/src/libexpr/primops/flake.cc @@ -0,0 +1,647 @@ +#include "flake.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "fetchGit.hh" +#include "download.hh" +#include "args.hh" + +#include <iostream> +#include <queue> +#include <regex> +#include <ctime> +#include <iomanip> +#include <nlohmann/json.hpp> + +namespace nix { + +using namespace flake; + +namespace flake { + +/* Read a registry. */ +std::shared_ptr<FlakeRegistry> readRegistry(const Path & path) +{ + auto registry = std::make_shared<FlakeRegistry>(); + + if (!pathExists(path)) + return std::make_shared<FlakeRegistry>(); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) + registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); + + return registry; +} + +/* Write a registry to a file. */ +void writeRegistry(const FlakeRegistry & registry, const Path & path) +{ + nlohmann::json json; + json["version"] = 1; + for (auto elem : registry.entries) + json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. +} + +LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) +{ + FlakeRef flakeRef(json["uri"]); + if (!flakeRef.isImmutable()) + throw Error("cannot use mutable flake '%s' in pure mode", flakeRef); + + LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["narHash"])); + + auto nonFlakeInputs = json["nonFlakeInputs"]; + + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); + LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); + entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); + } + + auto inputs = json["inputs"]; + + for (auto i = inputs.begin(); i != inputs.end(); ++i) + entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return entry; +} + +LockFile readLockFile(const Path & path) +{ + LockFile lockFile; + + if (!pathExists(path)) + return lockFile; + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("lock file '%s' has unsupported version %d", path, version); + + auto nonFlakeInputs = json["nonFlakeInputs"]; + + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); + if (!flakeRef.isImmutable()) + throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); + } + + auto inputs = json["inputs"]; + + for (auto i = inputs.begin(); i != inputs.end(); ++i) + lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return lockFile; +} + +nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) +{ + nlohmann::json json; + json["uri"] = entry.ref.to_string(); + json["narHash"] = entry.narHash.to_string(SRI); + for (auto & x : entry.nonFlakeEntries) { + json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); + json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); + } + for (auto & x : entry.flakeEntries) + json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); + return json; +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + nlohmann::json json; + json["version"] = 1; + json["nonFlakeInputs"] = nlohmann::json::object(); + for (auto & x : lockFile.nonFlakeEntries) { + json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); + json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); + } + json["inputs"] = nlohmann::json::object(); + for (auto & x : lockFile.flakeEntries) + json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); + stream << json.dump(4); // '4' = indentation in json file + return stream; +} + +void writeLockFile(const LockFile & lockFile, const Path & path) +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", lockFile)); +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr<FlakeRegistry> getUserRegistry() +{ + return readRegistry(getUserRegistryPath()); +} + +std::shared_ptr<FlakeRegistry> getFlagRegistry(RegistryOverrides registryOverrides) +{ + auto flagRegistry = std::make_shared<FlakeRegistry>(); + for (auto const & x : registryOverrides) { + flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); + } + return flagRegistry; +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector<FlakeRef> pastSearches = {}); + +FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector<FlakeRef> pastSearches) +{ + std::string errorMsg = "found cycle in flake registries: "; + for (FlakeRef oldRef : pastSearches) { + errorMsg += oldRef.to_string(); + if (oldRef == newRef) + throw Error(errorMsg); + errorMsg += " - "; + } + pastSearches.push_back(newRef); + return lookupFlake(state, newRef, registries, pastSearches); +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector<FlakeRef> pastSearches) +{ + if (registries.empty() && !flakeRef.isDirect()) + throw Error("indirect flake reference '%s' is not allowed", flakeRef); + + for (std::shared_ptr<FlakeRegistry> registry : registries) { + auto i = registry->entries.find(flakeRef); + if (i != registry->entries.end()) { + auto newRef = i->second; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + + auto j = registry->entries.find(flakeRef.baseRef()); + if (j != registry->entries.end()) { + auto newRef = j->second; + newRef.ref = flakeRef.ref; + newRef.rev = flakeRef.rev; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + } + + if (!flakeRef.isDirect()) + throw Error("could not resolve flake reference '%s'", flakeRef); + + return flakeRef; +} + +// Lookups happen here too +static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + FlakeRef resolvedRef = lookupFlake(state, flakeRef, + impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>()); + + if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) + throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); + + auto doGit = [&](const GitInfo & gitInfo) { + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; + SourceInfo info(ref); + info.storePath = gitInfo.storePath; + info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = gitInfo.lastModified; + return info; + }; + + // This only downloads only one revision of the repo, not the entire history. + if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) { + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + refData->owner, refData->repo, + resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) + : resolvedRef.ref ? *resolvedRef.ref : "master"); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; + request.getLastModified = true; + auto result = getDownloader()->downloadCached(state.store, request); + + if (!result.etag) + throw Error("did not receive an ETag header from '%s'", url); + + if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') + throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); + + FlakeRef ref(resolvedRef.baseRef()); + ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + SourceInfo info(ref); + info.storePath = result.storePath; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = result.lastModified; + + return info; + } + + // This downloads the entire git history + else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) { + return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); + } + + else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) { + if (!pathExists(refData->path + "/.git")) + throw Error("flake '%s' does not reference a Git repository", refData->path); + return doGit(exportGit(state.store, refData->path, {}, {}, "source")); + } + + else abort(); +} + +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + state.store->assertStorePath(sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); + + // Guard against symlink attacks. + Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); + Path realFlakeFile = state.store->toRealPath(flakeFile); + if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); + + Flake flake(flakeRef, sourceInfo); + + if (!pathExists(realFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); + + Value vInfo; + state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack + + state.forceAttrs(vInfo); + + auto sEpoch = state.symbols.create("epoch"); + + if (auto epoch = vInfo.attrs->get(sEpoch)) { + flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); + if (flake.epoch > 2019) + throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); + } else + throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); + + if (auto name = vInfo.attrs->get(state.sName)) + flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); + else + throw Error("flake '%s' lacks attribute 'name'", flakeRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) + flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) { + state.forceList(*(**inputs).value, *(**inputs).pos); + for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) + flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( + *(**inputs).value->listElems()[n], *(**inputs).pos))); + } + + auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); + + if (std::optional<Attr *> nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { + state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); + for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { + std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); + FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); + flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); + } + } + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + state.forceFunction(*(**outputs).value, *(**outputs).pos); + flake.vOutputs = (**outputs).value; + } else + throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEpoch && + attr.name != state.sName && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sNonFlakeInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + flakeRef, attr.name, *attr.pos); + } + + return flake; +} + +// Get the `NonFlake` corresponding to a `FlakeRef`. +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false) +{ + auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + NonFlake nonFlake(flakeRef, sourceInfo); + + state.store->assertStorePath(nonFlake.sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(nonFlake.sourceInfo.storePath); + + nonFlake.alias = alias; + + return nonFlake; +} + +LockFile entryToLockFile(const LockFile::FlakeEntry & entry) +{ + LockFile lockFile; + lockFile.flakeEntries = entry.flakeEntries; + lockFile.nonFlakeEntries = entry.nonFlakeEntries; + return lockFile; +} + +LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) +{ + LockFile::FlakeEntry entry( + resolvedFlake.flake.sourceInfo.resolvedRef, + resolvedFlake.flake.sourceInfo.narHash); + + for (auto & info : resolvedFlake.flakeDeps) + entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second)); + + for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { + LockFile::NonFlakeEntry nonEntry( + nonFlake.sourceInfo.resolvedRef, + nonFlake.sourceInfo.narHash); + entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); + } + + return entry; +} + +bool allowedToWrite(HandleLockFile handle) +{ + return handle == UpdateLockFile || handle == RecreateLockFile; +} + +bool recreateLockFile(HandleLockFile handle) +{ + return handle == RecreateLockFile || handle == UseNewLockFile; +} + +bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) +{ + if (handle == AllPure) return false; + else if (handle == TopRefUsesRegistries) return isTopRef; + else if (handle == UpdateLockFile) return true; + else if (handle == UseUpdatedLockFile) return true; + else if (handle == RecreateLockFile) return true; + else if (handle == UseNewLockFile) return true; + else assert(false); +} + +ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, + HandleLockFile handleLockFile, LockFile lockFile = {}, bool topRef = false) +{ + Flake flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); + + ResolvedFlake deps(flake); + + for (auto & nonFlakeInfo : flake.nonFlakeInputs) { + FlakeRef ref = nonFlakeInfo.second; + auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); + if (i != lockFile.nonFlakeEntries.end()) { + NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); + if (nonFlake.sourceInfo.narHash != i->second.narHash) + throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); + deps.nonFlakeDeps.push_back(nonFlake); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update non-flake dependency '%s' in pure mode", nonFlakeInfo.first); + deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first, allowedToUseRegistries(handleLockFile, false))); + } + } + + for (auto newFlakeRef : flake.inputs) { + auto i = lockFile.flakeEntries.find(newFlakeRef); + if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible + ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); + if (newResFlake.flake.sourceInfo.narHash != i->second.narHash) + throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); + deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update flake dependency '%s' in pure mode", newFlakeRef.to_string()); + deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile)); + } + } + + return deps; +} + +/* Given a flake reference, recursively fetch it and its dependencies. + FIXME: this should return a graph of flakes. +*/ +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) +{ + Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + LockFile oldLockFile; + + if (!recreateLockFile(handleLockFile)) { + // If recreateLockFile, start with an empty lockfile + // FIXME: symlink attack + oldLockFile = readLockFile( + state.store->toRealPath(flake.sourceInfo.storePath) + + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); + } + + LockFile lockFile(oldLockFile); + + ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true); + lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake)); + + if (!(lockFile == oldLockFile)) { + if (allowedToWrite(handleLockFile)) { + if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) { + writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + + // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. + runProgram("git", true, { "-C", refData->path, "add", + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + } else + warn("cannot write lockfile of remote flake '%s'", topRef); + } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) + warn("using updated lockfile without writing it to file"); + } + + return resFlake; +} + +void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) +{ + resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); +} + +static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) +{ + auto & path = sourceInfo.storePath; + state.store->isValidPath(path); + mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); + + if (sourceInfo.resolvedRef.rev) { + mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")), + sourceInfo.resolvedRef.rev->gitRev()); + mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")), + sourceInfo.resolvedRef.rev->gitShortRev()); + } + + if (sourceInfo.revCount) + mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); + + if (sourceInfo.lastModified) + mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), + fmt("%s", + std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); +} + +void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) +{ + // Construct the resulting attrset '{description, outputs, + // ...}'. This attrset is passed lazily as an argument to 'outputs'. + + state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8); + + for (auto info : resFlake.flakeDeps) { + const ResolvedFlake newResFlake = info.second; + auto vFlake = state.allocAttr(v, newResFlake.flake.id); + callFlake(state, newResFlake, *vFlake); + } + + for (const NonFlake nonFlake : resFlake.nonFlakeDeps) { + auto vNonFlake = state.allocAttr(v, nonFlake.alias); + state.mkAttrs(*vNonFlake, 8); + + state.store->isValidPath(nonFlake.sourceInfo.storePath); + mkString(*state.allocAttr(*vNonFlake, state.sOutPath), + nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); + + emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake); + } + + mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); + + emitSourceInfoAttrs(state, resFlake.flake.sourceInfo, v); + + auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); + mkApp(*vOutputs, *resFlake.flake.vOutputs, v); + + v.attrs->push_back(Attr(state.symbols.create("self"), &v)); + + v.attrs->sort(); +} + +// This function is exposed to be used in nix files. +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + callFlake(state, resolveFlake(state, state.forceStringNoCtx(*args[0], pos), + evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v); +} + +static RegisterPrimOp r2("getFlake", 1, prim_getFlake); + +void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) +{ + flakeRef = lookupFlake(state, flakeRef, registries); + + std::string uri; + + Strings args = {"clone"}; + + if (auto refData = std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) { + uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; + args.push_back(uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } else if (auto refData = std::get_if<FlakeRef::IsGit>(&flakeRef.data)) { + args.push_back(refData->uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } + + if (destDir != "") + args.push_back(destDir); + + runProgram("git", true, args); +} + +} + +std::shared_ptr<flake::FlakeRegistry> EvalState::getGlobalFlakeRegistry() +{ + std::call_once(_globalFlakeRegistryInit, [&]() { + auto path = evalSettings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(evalSettings.flakeRegistry); + request.name = "flake-registry.json"; + request.gcRoot = true; + path = getDownloader()->downloadCached(store, request).path; + } + + _globalFlakeRegistry = readRegistry(path); + }); + + return _globalFlakeRegistry; +} + +// This always returns a vector with flakeReg, userReg, globalReg. +// If one of them doesn't exist, the registry is left empty but does exist. +const Registries EvalState::getFlakeRegistries() +{ + Registries registries; + registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalFlakeRegistry()); + return registries; +} + +} diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh new file mode 100644 index 000000000..82b0973f6 --- /dev/null +++ b/src/libexpr/primops/flake.hh @@ -0,0 +1,147 @@ +#include "types.hh" +#include "flakeref.hh" + +#include <variant> + +namespace nix { + +struct Value; +class EvalState; + +namespace flake { + +static const size_t FLAG_REGISTRY = 0; +static const size_t USER_REGISTRY = 1; +static const size_t GLOBAL_REGISTRY = 2; + +struct FlakeRegistry +{ + std::map<FlakeRef, FlakeRef> entries; +}; + +struct LockFile +{ + struct NonFlakeEntry + { + FlakeRef ref; + Hash narHash; + NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; + + bool operator ==(const NonFlakeEntry & other) const + { + return ref == other.ref && narHash == other.narHash; + } + }; + + struct FlakeEntry + { + FlakeRef ref; + Hash narHash; + std::map<FlakeRef, FlakeEntry> flakeEntries; + std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries; + FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; + + bool operator ==(const FlakeEntry & other) const + { + return + ref == other.ref + && narHash == other.narHash + && flakeEntries == other.flakeEntries + && nonFlakeEntries == other.nonFlakeEntries; + } + }; + + std::map<FlakeRef, FlakeEntry> flakeEntries; + std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries; + + bool operator ==(const LockFile & other) const + { + return + flakeEntries == other.flakeEntries + && nonFlakeEntries == other.nonFlakeEntries; + } +}; + +typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries; + +Path getUserRegistryPath(); + +enum HandleLockFile : unsigned int + { AllPure // Everything is handled 100% purely + , TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely + , UpdateLockFile // Update the existing lockfile and write it to file + , UseUpdatedLockFile // `UpdateLockFile` without writing to file + , RecreateLockFile // Recreate the lockfile from scratch and write it to file + , UseNewLockFile // `RecreateLockFile` without writing to file + }; + +std::shared_ptr<FlakeRegistry> readRegistry(const Path &); + +void writeRegistry(const FlakeRegistry &, const Path &); + +struct SourceInfo +{ + // Immutable flakeref that this source tree was obtained from. + FlakeRef resolvedRef; + + Path storePath; + + // Number of ancestors of the most recent commit. + std::optional<uint64_t> revCount; + + // NAR hash of the store path. + Hash narHash; + + // A stable timestamp of this source tree. For Git and GitHub + // flakes, the commit date (not author date!) of the most recent + // commit. + std::optional<time_t> lastModified; + + SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; +}; + +struct Flake +{ + FlakeId id; + FlakeRef originalRef; + std::string description; + SourceInfo sourceInfo; + std::vector<FlakeRef> inputs; + std::map<FlakeAlias, FlakeRef> nonFlakeInputs; + Value * vOutputs; // FIXME: gc + unsigned int epoch; + + Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; +}; + +struct NonFlake +{ + FlakeAlias alias; + FlakeRef originalRef; + SourceInfo sourceInfo; + NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; +}; + +Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); + +struct ResolvedFlake +{ + Flake flake; + std::map<FlakeRef, ResolvedFlake> flakeDeps; // The key in this map, is the originalRef as written in flake.nix + std::vector<NonFlake> nonFlakeDeps; + ResolvedFlake(const Flake & flake) : flake(flake) {} +}; + +ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); + +void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v); + +void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); + +void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); + +} + +} diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc new file mode 100644 index 000000000..6c90c3b64 --- /dev/null +++ b/src/libexpr/primops/flakeref.cc @@ -0,0 +1,251 @@ +#include "flakeref.hh" +#include "store-api.hh" + +#include <regex> + +namespace nix { + +// A Git ref (i.e. branch or tag name). +const static std::string refRegex = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +std::regex revRegex(revRegexS, std::regex::ECMAScript); + +// A Git ref or revision. +const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))"; + +// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref +// optionally followed by a rev (e.g. "master" or +// "master/e72daba8250068216d79d2aeef40d4d95aff6666"). +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// GitHub references. +const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// URI stuff. +const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; +const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; +const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; +const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; + +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; + + +FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) +{ + // FIXME: could combine this into one regex. + + static std::regex flakeRegex( + "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", + std::regex::ECMAScript); + + static std::regex githubRegex( + "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", + std::regex::ECMAScript); + + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?://(" + authorityRegex + "))?" + + "(" + pathRegex + "))", + std::regex::ECMAScript); + + static std::regex refRegex2(refRegex, std::regex::ECMAScript); + + static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); + + auto [uri, params] = splitUriAndParams(uri_); + + auto handleSubdir = [&](const std::string & name, const std::string & value) { + if (name == "dir") { + if (value != "" && !std::regex_match(value, subDirRegex2)) + throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value); + subdir = value; + return true; + } else + return false; + }; + + auto handleGitParams = [&](const std::string & name, const std::string & value) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadFlakeRef("invalid Git revision '%s'", value); + rev = Hash(value, htSHA1); + } else if (name == "ref") { + if (!std::regex_match(value, refRegex2)) + throw BadFlakeRef("invalid Git ref '%s'", value); + ref = value; + } else if (handleSubdir(name, value)) + ; + else return false; + return true; + }; + + std::cmatch match; + if (std::regex_match(uri.c_str(), match, flakeRegex)) { + IsAlias d; + d.alias = match[1]; + if (match[2].matched) + rev = Hash(match[2], htSHA1); + else if (match[3].matched) { + ref = match[3]; + if (match[4].matched) + rev = Hash(match[4], htSHA1); + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, githubRegex)) { + IsGitHub d; + d.owner = match[1]; + d.repo = match[2]; + if (match[3].matched) + rev = Hash(match[3], htSHA1); + else if (match[4].matched) { + ref = match[4]; + } + for (auto & param : params) { + if (handleSubdir(param.first, param.second)) + ; + else + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, uriRegex) + && (match[2] == "file" || hasSuffix(match[4], ".git"))) + { + IsGit d; + d.uri = match[1]; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + // FIXME: should probably pass through unknown parameters + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + if (rev && !ref) + throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); + data = d; + } + + else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) + && uri.find(':') == std::string::npos) + { + IsPath d; + if (allowRelative) { + d.path = absPath(uri); + while (true) { + if (pathExists(d.path + "/.git")) break; + subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); + d.path = dirOf(d.path); + if (d.path == "/") + throw BadFlakeRef("path '%s' does not reference a Git repository", uri); + } + } else + d.path = canonPath(uri); + data = d; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + } + + else + throw BadFlakeRef("'%s' is not a valid flake reference", uri); +} + +std::string FlakeRef::to_string() const +{ + std::string string; + bool first = true; + + auto addParam = + [&](const std::string & name, std::string value) { + string += first ? '?' : '&'; + first = false; + string += name; + string += '='; + string += value; // FIXME: escaping + }; + + if (auto refData = std::get_if<FlakeRef::IsAlias>(&data)) { + string = refData->alias; + if (ref) string += '/' + *ref; + if (rev) string += '/' + rev->gitRev(); + } + + else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) { + string = refData->path; + if (ref) addParam("ref", *ref); + if (rev) addParam("rev", rev->gitRev()); + if (subdir != "") addParam("dir", subdir); + } + + else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) { + assert(!(ref && rev)); + string = "github:" + refData->owner + "/" + refData->repo; + if (ref) { string += '/'; string += *ref; } + if (rev) { string += '/'; string += rev->gitRev(); } + if (subdir != "") addParam("dir", subdir); + } + + else if (auto refData = std::get_if<FlakeRef::IsGit>(&data)) { + assert(!rev || ref); + string = refData->uri; + + if (ref) { + addParam("ref", *ref); + if (rev) + addParam("rev", rev->gitRev()); + } + + if (subdir != "") addParam("dir", subdir); + } + + else abort(); + + assert(FlakeRef(string) == *this); + + return string; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +bool FlakeRef::isImmutable() const +{ + return (bool) rev; +} + +FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. +{ + FlakeRef result(*this); + result.ref = std::nullopt; + result.rev = std::nullopt; + return result; +} + +std::optional<FlakeRef> parseFlakeRef( + const std::string & uri, bool allowRelative) +{ + try { + return FlakeRef(uri, allowRelative); + } catch (BadFlakeRef & e) { + return {}; + } +} + +} diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh new file mode 100644 index 000000000..52bb82ddb --- /dev/null +++ b/src/libexpr/primops/flakeref.hh @@ -0,0 +1,188 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +#include <variant> + +namespace nix { + +/* Flake references are a URI-like syntax to specify a flake. + + Examples: + + * <flake-id>(/rev-or-ref(/rev)?)? + + Look up a flake by ID in the flake lock file or in the flake + registry. These must specify an actual location for the flake + using the formats listed below. Note that in pure evaluation + mode, the flake registry is empty. + + Optionally, the rev or ref from the dereferenced flake can be + overriden. For example, + + nixpkgs/19.09 + + uses the "19.09" branch of the nixpkgs' flake GitHub repository, + while + + nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + uses the specified revision. For Git (rather than GitHub) + repositories, both the rev and ref must be given, e.g. + + nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + * github:<owner>/<repo>(/<rev-or-ref>)? + + A repository on GitHub. These differ from Git references in that + they're downloaded in a efficient way (via the tarball mechanism) + and that they support downloading a specific revision without + specifying a branch. <rev-or-ref> is either a commit hash ("rev") + or a branch or tag name ("ref"). The default is: "master" if none + is specified. Note that in pure evaluation mode, a commit hash + must be used. + + Flakes fetched in this manner expose "rev" and "lastModified" + attributes, but not "revCount". + + Examples: + + github:edolstra/dwarffs + github:edolstra/dwarffs/unstable + github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 + + * https://<server>/<path>.git(\?attr(&attr)*)? + ssh://<server>/<path>.git(\?attr(&attr)*)? + git://<server>/<path>.git(\?attr(&attr)*)? + file:///<path>(\?attr(&attr)*)? + + where 'attr' is one of: + rev=<rev> + ref=<ref> + + A Git repository fetched through https. Note that the path must + end in ".git". The default for "ref" is "master". + + Examples: + + https://example.org/my/repo.git + https://example.org/my/repo.git?ref=release-1.2.3 + https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 + + * /path.git(\?attr(&attr)*)? + + Like file://path.git, but if no "ref" or "rev" is specified, the + (possibly dirty) working tree will be used. Using a working tree + is not allowed in pure evaluation mode. + + Examples: + + /path/to/my/repo + /path/to/my/repo?ref=develop + /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + + * https://<server>/<path>.tar.xz(?hash=<sri-hash>) + file:///<path>.tar.xz(?hash=<sri-hash>) + + A flake distributed as a tarball. In pure evaluation mode, an SRI + hash is mandatory. It exposes a "lastModified" attribute, being + the newest file inside the tarball. + + Example: + + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c + + Note: currently, there can be only one flake per Git repository, and + it must be at top-level. In the future, we may want to add a field + (e.g. "dir=<dir>") to specify a subdirectory inside the repository. +*/ + +typedef std::string FlakeId; +typedef std::string FlakeAlias; +typedef std::string FlakeUri; + +struct FlakeRef +{ + struct IsAlias + { + FlakeAlias alias; + bool operator<(const IsAlias & b) const { return alias < b.alias; }; + bool operator==(const IsAlias & b) const { return alias == b.alias; }; + }; + + struct IsGitHub { + std::string owner, repo; + bool operator<(const IsGitHub & b) const { + return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo); + } + bool operator==(const IsGitHub & b) const { + return owner == b.owner && repo == b.repo; + } + }; + + // Git, Tarball + struct IsGit + { + std::string uri; + bool operator<(const IsGit & b) const { return uri < b.uri; } + bool operator==(const IsGit & b) const { return uri == b.uri; } + }; + + struct IsPath + { + Path path; + bool operator<(const IsPath & b) const { return path < b.path; } + bool operator==(const IsPath & b) const { return path == b.path; } + }; + + // Git, Tarball + + std::variant<IsAlias, IsGitHub, IsGit, IsPath> data; + + std::optional<std::string> ref; + std::optional<Hash> rev; + Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root. + + bool operator<(const FlakeRef & flakeRef) const + { + return std::make_tuple(data, ref, rev, subdir) < + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir); + } + + bool operator==(const FlakeRef & flakeRef) const + { + return std::make_tuple(data, ref, rev, subdir) == + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir); + } + + // Parse a flake URI. + FlakeRef(const std::string & uri, bool allowRelative = false); + + // FIXME: change to operator <<. + std::string to_string() const; + + /* Check whether this is a "direct" flake reference, that is, not + a flake ID, which requires a lookup in the flake registry. */ + bool isDirect() const + { + return !std::get_if<FlakeRef::IsAlias>(&data); + } + + /* Check whether this is an "immutable" flake reference, that is, + one that contains a commit hash or content hash. */ + bool isImmutable() const; + + FlakeRef baseRef() const; +}; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + +MakeError(BadFlakeRef, Error); + +std::optional<FlakeRef> parseFlakeRef( + const std::string & uri, bool allowRelative = false); + +} diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0bd738809..a69592219 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1841,7 +1841,7 @@ void DerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), drvPath, settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep<StringSet>(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); @@ -3170,7 +3170,7 @@ void DerivationGoal::registerOutputs() valid. */ delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - dest, h.to_string(), h2.to_string())); + dest, h.to_string(SRI), h2.to_string(SRI))); Path actualDest = worker.store.toRealPath(dest); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 22382ab1d..0338727c1 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -319,10 +319,10 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); @@ -790,20 +790,27 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref<Store> store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.storePath = expectedStorePath; + result.path = store->toRealPath(expectedStorePath); + assert(!request.getLastModified); // FIXME + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -822,6 +829,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -829,10 +838,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -845,17 +854,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -871,10 +880,11 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa } catch (DownloadError & e) { if (storePath.empty()) throw; printError(format("warning: %1%; using cached result") % e.msg()); + result.etag = expectedETag; } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -883,28 +893,43 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa store->addTempRoot(unpackedStorePath); if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = ""; + else + result.lastModified = lstat(unpackedLink).st_mtime; } if (unpackedStorePath.empty()) { printInfo(format("unpacking '%1%'...") % url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir}); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + result.lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } - replaceSymlink(unpackedStorePath, unpackedLink); + // Store the last-modified date of the tarball in the symlink + // mtime. This saves us from having to store it somewhere + // else. + replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified); storePath = unpackedStorePath; } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, expectedHash.to_string(), gotHash.to_string()); + url, request.expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + if (request.gcRoot) + store->addIndirectRoot(fileLink); + + result.storePath = storePath; + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..43b1c5c09 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -36,11 +36,36 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr<std::string> data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + bool gcRoot = false; + bool getLastModified = false; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + +struct CachedDownloadResult +{ + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; + Path path; + std::optional<std::string> etag; + std::string effectiveUri; + std::optional<time_t> lastModified; +}; + class Store; struct Downloader @@ -64,8 +89,7 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 53efc6a90..80d70fba3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -344,6 +344,9 @@ public: Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + + Setting<std::string> githubAccessToken{this, "", "github-acces-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d..105e1dcdd 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -160,10 +160,11 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); store->init(); return store; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff1156..8fabeeea4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -726,12 +726,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string showPaths(const PathSet & paths) { - string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + return concatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7af2a1bf7..2837dacc9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -200,4 +200,73 @@ void printTable(std::ostream & out, const Table2 & table) } } +void Command::printHelp(const string & programName, std::ostream & out) +{ + Args::printHelp(programName, out); + + auto exs = examples(); + if (!exs.empty()) { + out << "\n"; + out << "Examples:\n"; + for (auto & ex : exs) + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } +} + +MultiCommand::MultiCommand(const std::vector<ref<Command>> & _commands) +{ + for (auto & command : _commands) + commands.emplace(command->name(), command); + + expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { + assert(!command); + auto i = commands.find(ss[0]); + if (i == commands.end()) + throw UsageError("'%s' is not a recognised command", ss[0]); + command = i->second; + }}); +} + +void MultiCommand::printHelp(const string & programName, std::ostream & out) +{ + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } + + out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; + + out << "\n"; + out << "Common flags:\n"; + printFlags(out); + + out << "\n"; + out << "Available commands:\n"; + + Table2 table; + for (auto & command : commands) { + auto descr = command.second->description(); + if (!descr.empty()) + table.push_back(std::make_pair(command.second->name(), descr)); + } + printTable(out, table); +} + +bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (Args::processFlag(pos, end)) return true; + if (command && command->processFlag(pos, end)) return true; + return false; +} + +bool MultiCommand::processArgs(const Strings & args, bool finish) +{ + if (command) + return command->processArgs(args, finish); + else + return Args::processArgs(args, finish); +} + } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ad5fcca39..bf69bf4b6 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -188,6 +188,47 @@ public: friend class MultiCommand; }; +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args +{ + virtual std::string name() = 0; + virtual void prepare() { }; + virtual void run() = 0; + + struct Example + { + std::string description; + std::string command; + }; + + typedef std::list<Example> Examples; + + virtual Examples examples() { return Examples(); } + + void printHelp(const string & programName, std::ostream & out) override; +}; + +typedef std::map<std::string, ref<Command>> Commands; + +/* An argument parser that supports multiple subcommands, + i.e. ‘<command> <subcommand>’. */ +class MultiCommand : virtual Args +{ +public: + Commands commands; + + std::shared_ptr<Command> command; + + MultiCommand(const std::vector<ref<Command>> & commands); + + void printHelp(const string & programName, std::ostream & out) override; + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + Strings argvToStrings(int argc, char * * argv); /* Helper function for rendering argument labels. */ diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2dbc3b630..edede8ace 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -80,6 +80,18 @@ struct Hash or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ std::string to_string(Base base = Base32, bool includeType = true) const; + + std::string gitRev() const + { + assert(type == htSHA1); + return to_string(Base16, false); + } + + std::string gitShortRev() const + { + assert(type == htSHA1); + return std::string(to_string(Base16, false), 0, 7); + } }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 17aee2d5c..92c8957ff 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -22,6 +22,7 @@ #include <sys/ioctl.h> #include <sys/types.h> #include <sys/wait.h> +#include <sys/time.h> #include <unistd.h> #ifdef __APPLE__ @@ -348,7 +349,6 @@ void writeFile(const Path & path, Source & source, mode_t mode) } } - string readLine(int fd) { string s; @@ -466,6 +466,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR", "/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + + static Lazy<Path> getHome2([]() { Path homeDir = getEnv("HOME"); if (homeDir.empty()) { @@ -542,20 +553,31 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link) +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { if (symlink(target.c_str(), link.c_str())) throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp); + createSymlink(target, tmp, mtime); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; @@ -967,12 +989,14 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) return res; } - +// Output = "standard out" output stream string runProgram(Path program, bool searchPath, const Strings & args, const std::optional<std::string> & input) { RunOptions opts(program, args); opts.searchPath = searchPath; + // This allows you to refer to a program with a pathname relative to the + // PATH variable. opts.input = input; auto res = runProgram(opts); @@ -983,6 +1007,7 @@ string runProgram(Path program, bool searchPath, const Strings & args, return res.second; } +// Output = error code + "standard out" output stream std::pair<int, std::string> runProgram(const RunOptions & options_) { RunOptions options(options_); @@ -1043,6 +1068,8 @@ void runProgram2(const RunOptions & options) if (options.searchPath) execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. else execv(options.program.c_str(), stringsToCharPtrs(args_).data()); @@ -1177,28 +1204,6 @@ template StringSet tokenizeString(const string & s, const string & separators); template vector<string> tokenizeString(const string & s, const string & separators); -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - string chomp(const string & s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fce3cab8d..e05ef1e7d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -122,10 +122,6 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); @@ -146,10 +142,12 @@ Path getDataDir(); Paths createDirs(const Path & path); /* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Wrappers arount read()/write() that read/write exactly the @@ -203,6 +201,14 @@ public: }; +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a temporary file, returning a file handle and its path. */ +std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix"); + + class Pipe { public: @@ -341,8 +347,26 @@ template<class C> C tokenizeString(const string & s, const string & separators = /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); +template<class C> +string concatStringsSep(const string & sep, const C & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +/* Add quotes around a collection of strings. */ +template<class C> Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} /* Remove trailing whitespace from a string. */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index c6a4d4166..d4749f8fd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -106,7 +106,7 @@ static void _main(int argc, char * * argv) // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". - if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) { + if (runEnv && argc > 1 && !std::regex_search(baseNameOf(argv[1]), std::regex("nix-shell"))) { script = argv[1]; try { auto lines = tokenizeString<Strings>(readFile(script), "\n"); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b66cc7e3..bd1371dba 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -86,10 +86,12 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - std::string effectiveUrl; + CachedDownloadRequest request(url); + request.ttl = 0; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); - url = chomp(std::move(effectiveUrl)); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -121,12 +123,10 @@ static void update(const StringSet & channelNames) } // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; } chomp(filename); } diff --git a/src/nix/build.cc b/src/nix/build.cc index b329ac38a..c08ec0e62 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,3 +1,4 @@ +#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -52,6 +53,8 @@ struct CmdBuild : MixDryRun, InstallablesCommand { auto buildables = build(store, dryRun ? DryRun : Build, installables); + auto evalState = std::make_shared<EvalState>(searchPath, store); + evalState->addRegistryOverrides(registryOverrides); if (dryRun) return; for (size_t i = 0; i < buildables.size(); ++i) { diff --git a/src/nix/command.cc b/src/nix/command.cc index 3d7d582d6..5967ab36c 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -4,79 +4,7 @@ namespace nix { -Commands * RegisterCommand::commands = 0; - -void Command::printHelp(const string & programName, std::ostream & out) -{ - Args::printHelp(programName, out); - - auto exs = examples(); - if (!exs.empty()) { - out << "\n"; - out << "Examples:\n"; - for (auto & ex : exs) - out << "\n" - << " " << ex.description << "\n" // FIXME: wrap - << " $ " << ex.command << "\n"; - } -} - -MultiCommand::MultiCommand(const Commands & _commands) - : commands(_commands) -{ - expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { - assert(!command); - auto i = commands.find(ss[0]); - if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", ss[0]); - command = i->second; - }}); -} - -void MultiCommand::printHelp(const string & programName, std::ostream & out) -{ - if (command) { - command->printHelp(programName + " " + command->name(), out); - return; - } - - out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; - - out << "\n"; - out << "Common flags:\n"; - printFlags(out); - - out << "\n"; - out << "Available commands:\n"; - - Table2 table; - for (auto & command : commands) { - auto descr = command.second->description(); - if (!descr.empty()) - table.push_back(std::make_pair(command.second->name(), descr)); - } - printTable(out, table); - -#if 0 - out << "\n"; - out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n"; -#endif -} - -bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) -{ - if (Args::processFlag(pos, end)) return true; - if (command && command->processFlag(pos, end)) return true; - return false; -} - -bool MultiCommand::processArgs(const Strings & args, bool finish) -{ - if (command) - return command->processArgs(args, finish); - else - return Args::processArgs(args, finish); -} +std::vector<ref<Command>> * RegisterCommand::commands = 0; StoreCommand::StoreCommand() { diff --git a/src/nix/command.hh b/src/nix/command.hh index 97a6fee7f..26c308331 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -2,6 +2,7 @@ #include "args.hh" #include "common-eval-args.hh" +#include <optional> namespace nix { @@ -10,30 +11,12 @@ extern std::string programPath; struct Value; class Bindings; class EvalState; - -/* A command is an argument parser that can be executed by calling its - run() method. */ -struct Command : virtual Args -{ - virtual std::string name() = 0; - virtual void prepare() { }; - virtual void run() = 0; - - struct Example - { - std::string description; - std::string command; - }; - - typedef std::list<Example> Examples; - - virtual Examples examples() { return Examples(); } - - void printHelp(const string & programName, std::ostream & out) override; -}; - class Store; +namespace flake { +enum HandleLockFile : unsigned int; +} + /* A command that require a Nix store. */ struct StoreCommand : virtual Command { @@ -72,25 +55,44 @@ struct Installable } }; -struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs +struct EvalCommand : virtual StoreCommand, MixEvalArgs { - Path file; - - SourceExprCommand(); - - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * getSourceExpr(EvalState & state); - ref<EvalState> getEvalState(); private: std::shared_ptr<EvalState> evalState; +}; + +struct MixFlakeOptions : virtual Args +{ + bool recreateLockFile = false; + + bool saveLockFile = true; + + bool useRegistries = true; + + MixFlakeOptions(); - Value * vSourceExpr = 0; + flake::HandleLockFile getLockFileMode(); +}; + +struct SourceExprCommand : virtual Args, EvalCommand, MixFlakeOptions +{ + std::optional<Path> file; + + SourceExprCommand(); + + std::vector<std::shared_ptr<Installable>> parseInstallables( + ref<Store> store, std::vector<std::string> ss); + + std::shared_ptr<Installable> parseInstallable( + ref<Store> store, const std::string & installable); + + virtual Strings getDefaultFlakeAttrPaths() + { + return {"defaultPackage"}; + } }; enum RealiseMode { Build, NoBuild, DryRun }; @@ -121,14 +123,14 @@ struct InstallableCommand : virtual Args, SourceExprCommand InstallableCommand() { - expectArg("installable", &_installable); + expectArg("installable", &_installable, true); } void prepare() override; private: - std::string _installable; + std::string _installable{"."}; }; /* A command that operates on zero or more store paths. */ @@ -162,42 +164,18 @@ struct StorePathCommand : public InstallablesCommand void run(ref<Store> store) override; }; -typedef std::map<std::string, ref<Command>> Commands; - -/* An argument parser that supports multiple subcommands, - i.e. ‘<command> <subcommand>’. */ -class MultiCommand : virtual Args -{ -public: - Commands commands; - - std::shared_ptr<Command> command; - - MultiCommand(const Commands & commands); - - void printHelp(const string & programName, std::ostream & out) override; - - bool processFlag(Strings::iterator & pos, Strings::iterator end) override; - - bool processArgs(const Strings & args, bool finish) override; -}; - /* A helper class for registering commands globally. */ struct RegisterCommand { - static Commands * commands; + static std::vector<ref<Command>> * commands; RegisterCommand(ref<Command> command) { - if (!commands) commands = new Commands; - commands->emplace(command->name(), command); + if (!commands) commands = new std::vector<ref<Command>>; + commands->push_back(command); } }; -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables); - Buildables build(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables); diff --git a/src/nix/flake-template.nix b/src/nix/flake-template.nix new file mode 100644 index 000000000..fe89e647e --- /dev/null +++ b/src/nix/flake-template.nix @@ -0,0 +1,15 @@ +{ + name = "hello"; + + description = "A flake for building Hello World"; + + epoch = 2019; + + requires = [ "nixpkgs" ]; + + provides = deps: rec { + + packages.hello = deps.nixpkgs.provides.packages.hello; + + }; +} diff --git a/src/nix/flake.cc b/src/nix/flake.cc new file mode 100644 index 000000000..8d6716391 --- /dev/null +++ b/src/nix/flake.cc @@ -0,0 +1,554 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "progress-bar.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "primops/flake.hh" +#include "get-drvs.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> +#include <queue> +#include <iomanip> + +using namespace nix; +using namespace nix::flake; + +class FlakeCommand : virtual Args, public EvalCommand, public MixFlakeOptions +{ + std::string flakeUri = "."; + +public: + + FlakeCommand() + { + expectArg("flake-uri", &flakeUri, true); + } + + FlakeRef getFlakeRef() + { + if (flakeUri.find('/') != std::string::npos || flakeUri == ".") + return FlakeRef(flakeUri, true); + else + return FlakeRef(flakeUri); + } + + Flake getFlake() + { + auto evalState = getEvalState(); + return flake::getFlake(*evalState, getFlakeRef(), useRegistries); + } + + ResolvedFlake resolveFlake() + { + return flake::resolveFlake(*getEvalState(), getFlakeRef(), getLockFileMode()); + } +}; + +struct CmdFlakeList : EvalCommand +{ + std::string name() override + { + return "list"; + } + + std::string description() override + { + return "list available Nix flakes"; + } + + void run(nix::ref<nix::Store> store) override + { + auto registries = getEvalState()->getFlakeRegistries(); + + stopProgressBar(); + + for (auto & entry : registries[FLAG_REGISTRY]->entries) + std::cout << entry.first.to_string() << " flags " << entry.second.to_string() << "\n"; + + for (auto & entry : registries[USER_REGISTRY]->entries) + std::cout << entry.first.to_string() << " user " << entry.second.to_string() << "\n"; + + for (auto & entry : registries[GLOBAL_REGISTRY]->entries) + std::cout << entry.first.to_string() << " global " << entry.second.to_string() << "\n"; + } +}; + +static void printSourceInfo(const SourceInfo & sourceInfo) +{ + std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string()); + if (sourceInfo.resolvedRef.ref) + std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref); + if (sourceInfo.resolvedRef.rev) + std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false)); + if (sourceInfo.revCount) + std::cout << fmt("Revisions: %s\n", *sourceInfo.revCount); + if (sourceInfo.lastModified) + std::cout << fmt("Last modified: %s\n", + std::put_time(std::localtime(&*sourceInfo.lastModified), "%F %T")); + std::cout << fmt("Path: %s\n", sourceInfo.storePath); +} + +static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) +{ + j["uri"] = sourceInfo.resolvedRef.to_string(); + if (sourceInfo.resolvedRef.ref) + j["branch"] = *sourceInfo.resolvedRef.ref; + if (sourceInfo.resolvedRef.rev) + j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false); + if (sourceInfo.revCount) + j["revCount"] = *sourceInfo.revCount; + if (sourceInfo.lastModified) + j["lastModified"] = *sourceInfo.lastModified; + j["path"] = sourceInfo.storePath; +} + +static void printFlakeInfo(const Flake & flake) +{ + std::cout << fmt("ID: %s\n", flake.id); + std::cout << fmt("Description: %s\n", flake.description); + std::cout << fmt("Epoch: %s\n", flake.epoch); + printSourceInfo(flake.sourceInfo); +} + +static nlohmann::json flakeToJson(const Flake & flake) +{ + nlohmann::json j; + j["id"] = flake.id; + j["description"] = flake.description; + j["epoch"] = flake.epoch; + sourceInfoToJson(flake.sourceInfo, j); + return j; +} + +static void printNonFlakeInfo(const NonFlake & nonFlake) +{ + std::cout << fmt("ID: %s\n", nonFlake.alias); + printSourceInfo(nonFlake.sourceInfo); +} + +static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake) +{ + nlohmann::json j; + j["id"] = nonFlake.alias; + sourceInfoToJson(nonFlake.sourceInfo, j); + return j; +} + +// FIXME: merge info CmdFlakeInfo? +struct CmdFlakeDeps : FlakeCommand +{ + std::string name() override + { + return "deps"; + } + + std::string description() override + { + return "list informaton about dependencies"; + } + + void run(nix::ref<nix::Store> store) override + { + auto evalState = getEvalState(); + evalState->addRegistryOverrides(registryOverrides); + + std::queue<ResolvedFlake> todo; + todo.push(resolveFlake()); + + stopProgressBar(); + + while (!todo.empty()) { + auto resFlake = std::move(todo.front()); + todo.pop(); + + for (auto & nonFlake : resFlake.nonFlakeDeps) + printNonFlakeInfo(nonFlake); + + for (auto & info : resFlake.flakeDeps) { + printFlakeInfo(info.second.flake); + todo.push(info.second); + } + } + } +}; + +struct CmdFlakeUpdate : FlakeCommand +{ + std::string name() override + { + return "update"; + } + + std::string description() override + { + return "update flake lock file"; + } + + void run(nix::ref<nix::Store> store) override + { + auto evalState = getEvalState(); + + auto flakeRef = getFlakeRef(); + + if (std::get_if<FlakeRef::IsPath>(&flakeRef.data)) + updateLockFile(*evalState, flakeRef, true); + else + throw Error("cannot update lockfile of flake '%s'", flakeRef); + } +}; + +static void enumerateOutputs(EvalState & state, Value & vFlake, + std::function<void(const std::string & name, Value & vProvide)> callback) +{ + state.forceAttrs(vFlake); + + auto vOutputs = (*vFlake.attrs->get(state.symbols.create("outputs")))->value; + + state.forceAttrs(*vOutputs); + + for (auto & attr : *vOutputs->attrs) + callback(attr.name, *attr.value); +} + +struct CmdFlakeInfo : FlakeCommand, MixJSON +{ + std::string name() override + { + return "info"; + } + + std::string description() override + { + return "list info about a given flake"; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = getFlake(); + stopProgressBar(); + + if (json) { + auto json = flakeToJson(flake); + + auto state = getEvalState(); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + auto outputs = nlohmann::json::object(); + + enumerateOutputs(*state, + *vFlake, + [&](const std::string & name, Value & vProvide) { + auto provide = nlohmann::json::object(); + + if (name == "checks" || name == "packages") { + state->forceAttrs(vProvide); + for (auto & aCheck : *vProvide.attrs) + provide[aCheck.name] = nlohmann::json::object(); + } + + outputs[name] = provide; + }); + + json["outputs"] = std::move(outputs); + + std::cout << json.dump() << std::endl; + } else + printFlakeInfo(flake); + } +}; + +struct CmdFlakeCheck : FlakeCommand, MixJSON +{ + bool build = true; + + CmdFlakeCheck() + { + mkFlag() + .longName("no-build") + .description("do not build checks") + .set(&build, false); + } + + std::string name() override + { + return "check"; + } + + std::string description() override + { + return "check whether the flake evaluates and run its tests"; + } + + void run(nix::ref<nix::Store> store) override + { + settings.readOnlyMode = !build; + + auto state = getEvalState(); + auto flake = resolveFlake(); + + auto checkDerivation = [&](const std::string & attrPath, Value & v) { + try { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("flake attribute '%s' is not a derivation", attrPath); + // FIXME: check meta attributes + return drvInfo->queryDrvPath(); + } catch (Error & e) { + e.addPrefix(fmt("while checking flake output attribute '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + throw; + } + }; + + PathSet drvPaths; + + { + Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + enumerateOutputs(*state, + *vFlake, + [&](const std::string & name, Value & vProvide) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking flake output '%s'", name)); + + try { + state->forceValue(vProvide); + + if (name == "checks") { + state->forceAttrs(vProvide); + for (auto & aCheck : *vProvide.attrs) + drvPaths.insert(checkDerivation( + name + "." + (std::string) aCheck.name, *aCheck.value)); + } + + else if (name == "packages") { + state->forceAttrs(vProvide); + for (auto & aCheck : *vProvide.attrs) + checkDerivation( + name + "." + (std::string) aCheck.name, *aCheck.value); + } + + else if (name == "defaultPackage" || name == "devShell") + checkDerivation(name, vProvide); + + } catch (Error & e) { + e.addPrefix(fmt("while checking flake output '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", name)); + throw; + } + }); + } + + if (build) { + Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + store->buildPaths(drvPaths); + } + } +}; + +struct CmdFlakeAdd : MixEvalArgs, Command +{ + FlakeUri alias; + FlakeUri uri; + + std::string name() override + { + return "add"; + } + + std::string description() override + { + return "upsert flake in user flake registry"; + } + + CmdFlakeAdd() + { + expectArg("alias", &alias); + expectArg("flake-uri", &uri); + } + + void run() override + { + FlakeRef aliasRef(alias); + Path userRegistryPath = getUserRegistryPath(); + auto userRegistry = readRegistry(userRegistryPath); + userRegistry->entries.erase(aliasRef); + userRegistry->entries.insert_or_assign(aliasRef, FlakeRef(uri)); + writeRegistry(*userRegistry, userRegistryPath); + } +}; + +struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command +{ + FlakeUri alias; + + std::string name() override + { + return "remove"; + } + + std::string description() override + { + return "remove flake from user flake registry"; + } + + CmdFlakeRemove() + { + expectArg("alias", &alias); + } + + void run() override + { + Path userRegistryPath = getUserRegistryPath(); + auto userRegistry = readRegistry(userRegistryPath); + userRegistry->entries.erase(FlakeRef(alias)); + writeRegistry(*userRegistry, userRegistryPath); + } +}; + +struct CmdFlakePin : virtual Args, EvalCommand +{ + FlakeUri alias; + + std::string name() override + { + return "pin"; + } + + std::string description() override + { + return "pin flake require in user flake registry"; + } + + CmdFlakePin() + { + expectArg("alias", &alias); + } + + void run(nix::ref<nix::Store> store) override + { + auto evalState = getEvalState(); + + Path userRegistryPath = getUserRegistryPath(); + FlakeRegistry userRegistry = *readRegistry(userRegistryPath); + auto it = userRegistry.entries.find(FlakeRef(alias)); + if (it != userRegistry.entries.end()) { + it->second = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; + writeRegistry(userRegistry, userRegistryPath); + } else { + std::shared_ptr<FlakeRegistry> globalReg = evalState->getGlobalFlakeRegistry(); + it = globalReg->entries.find(FlakeRef(alias)); + if (it != globalReg->entries.end()) { + auto newRef = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; + userRegistry.entries.insert_or_assign(alias, newRef); + writeRegistry(userRegistry, userRegistryPath); + } else + throw Error("the flake alias '%s' does not exist in the user or global registry", alias); + } + } +}; + +struct CmdFlakeInit : virtual Args, Command +{ + std::string name() override + { + return "init"; + } + + std::string description() override + { + return "create a skeleton 'flake.nix' file in the current directory"; + } + + void run() override + { + Path flakeDir = absPath("."); + + if (!pathExists(flakeDir + "/.git")) + throw Error("the directory '%s' is not a Git repository", flakeDir); + + Path flakePath = flakeDir + "/flake.nix"; + + if (pathExists(flakePath)) + throw Error("file '%s' already exists", flakePath); + + writeFile(flakePath, +#include "flake-template.nix.gen.hh" + ); + } +}; + +struct CmdFlakeClone : FlakeCommand +{ + Path destDir; + + std::string name() override + { + return "clone"; + } + + std::string description() override + { + return "clone flake repository"; + } + + CmdFlakeClone() + { + expectArg("dest-dir", &destDir, true); + } + + void run(nix::ref<nix::Store> store) override + { + auto evalState = getEvalState(); + + Registries registries = evalState->getFlakeRegistries(); + gitCloneFlake(getFlakeRef().to_string(), *evalState, registries, destDir); + } +}; + +struct CmdFlake : virtual MultiCommand, virtual Command +{ + CmdFlake() + : MultiCommand({make_ref<CmdFlakeList>() + , make_ref<CmdFlakeUpdate>() + , make_ref<CmdFlakeInfo>() + , make_ref<CmdFlakeCheck>() + , make_ref<CmdFlakeDeps>() + , make_ref<CmdFlakeAdd>() + , make_ref<CmdFlakeRemove>() + , make_ref<CmdFlakePin>() + , make_ref<CmdFlakeInit>() + , make_ref<CmdFlakeClone>() + }) + { + } + + std::string name() override + { + return "flake"; + } + + std::string description() override + { + return "manage Nix flakes"; + } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + command->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static RegisterCommand r1(make_ref<CmdFlake>()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0c1ad3ab3..eb3c27d6b 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,71 +7,53 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" +#include "primops/flake.hh" #include <regex> +#include <queue> namespace nix { +MixFlakeOptions::MixFlakeOptions() +{ + mkFlag() + .longName("recreate-lock-file") + .description("recreate lock file from scratch") + .set(&recreateLockFile, true); + + mkFlag() + .longName("no-save-lock-file") + .description("do not save the newly generated lock file") + .set(&saveLockFile, false); + + mkFlag() + .longName("no-registries") + .description("don't use flake registries") + .set(&useRegistries, false); +} + +flake::HandleLockFile MixFlakeOptions::getLockFileMode() +{ + using namespace flake; + return + useRegistries + ? recreateLockFile + ? (saveLockFile ? RecreateLockFile : UseNewLockFile) + : (saveLockFile ? UpdateLockFile : UseUpdatedLockFile) + : AllPure; +} + SourceExprCommand::SourceExprCommand() { mkFlag() .shortName('f') .longName("file") .label("file") - .description("evaluate FILE rather than the default") + .description("evaluate a set of attributes from FILE (deprecated)") .dest(&file); } -Value * SourceExprCommand::getSourceExpr(EvalState & state) -{ - if (vSourceExpr) return vSourceExpr; - - auto sToplevel = state.symbols.create("_toplevel"); - - vSourceExpr = state.allocValue(); - - if (file != "") - state.evalFile(lookupFileArg(state, file), *vSourceExpr); - - else { - - /* Construct the installation source from $NIX_PATH. */ - - auto searchPath = state.getSearchPath(); - - state.mkAttrs(*vSourceExpr, searchPath.size() + 1); - - mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); - - std::unordered_set<std::string> seen; - - for (auto & i : searchPath) { - if (i.first == "") continue; - if (seen.count(i.first)) continue; - seen.insert(i.first); -#if 0 - auto res = state.resolveSearchPathElem(i); - if (!res.first) continue; - if (!pathExists(res.second)) continue; - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), - state.getBuiltin("import"), - mkString(*state.allocValue(), res.second)); -#endif - Value * v1 = state.allocValue(); - mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); - Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), i.first)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), - state.getBuiltin("import"), *v2); - } - - vSourceExpr->attrs->sort(); - } - - return vSourceExpr; -} - -ref<EvalState> SourceExprCommand::getEvalState() +ref<EvalState> EvalCommand::getEvalState() { if (!evalState) evalState = std::make_shared<EvalState>(searchPath, getStore()); @@ -165,24 +147,131 @@ struct InstallableExpr : InstallableValue struct InstallableAttrPath : InstallableValue { + Value * v; std::string attrPath; - InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) - : InstallableValue(cmd), attrPath(attrPath) + InstallableAttrPath(SourceExprCommand & cmd, Value * v, const std::string & attrPath) + : InstallableValue(cmd), v(v), attrPath(attrPath) { } std::string what() override { return attrPath; } Value * toValue(EvalState & state) override { - auto source = cmd.getSourceExpr(state); + auto vRes = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), *v); + state.forceValue(*vRes); + return vRes; + } +}; + +void makeFlakeClosureGCRoot(Store & store, + const FlakeRef & origFlakeRef, + const flake::ResolvedFlake & resFlake) +{ + if (std::get_if<FlakeRef::IsPath>(&origFlakeRef.data)) return; - Bindings & autoArgs = *cmd.getAutoArgs(state); + /* Get the store paths of all non-local flakes. */ + PathSet closure; - Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); - state.forceValue(*v); + std::queue<std::reference_wrapper<const flake::ResolvedFlake>> queue; + queue.push(resFlake); - return v; + while (!queue.empty()) { + const flake::ResolvedFlake & flake = queue.front(); + queue.pop(); + if (!std::get_if<FlakeRef::IsPath>(&flake.flake.sourceInfo.resolvedRef.data)) + closure.insert(flake.flake.sourceInfo.storePath); + for (const auto & dep : flake.flakeDeps) + queue.push(dep.second); + } + + if (closure.empty()) return; + + /* Write the closure to a file in the store. */ + auto closurePath = store.addTextToStore("flake-closure", concatStringsSep(" ", closure), closure); + + Path cacheDir = getCacheDir() + "/nix/flake-closures"; + createDirs(cacheDir); + + auto s = origFlakeRef.to_string(); + assert(s[0] != '.'); + s = replaceStrings(s, "%", "%25"); + s = replaceStrings(s, "/", "%2f"); + s = replaceStrings(s, ":", "%3a"); + Path symlink = cacheDir + "/" + s; + debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); + replaceSymlink(closurePath, symlink); + store.addIndirectRoot(symlink); +} + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + bool searchPackages = false; + + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, Strings attrPaths) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(std::move(attrPaths)) + { } + + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, std::string attrPath) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath}, searchPackages(true) + { } + + std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } + + Value * toValue(EvalState & state) override + { + auto vFlake = state.allocValue(); + + auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); + + callFlake(state, resFlake, *vFlake); + + makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); + + auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; + + state.forceValue(*vOutputs); + + auto emptyArgs = state.allocBindings(0); + + if (searchPackages) { + // As a convenience, look for the attribute in + // 'outputs.packages'. + if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("packages"))) { + try { + auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } + + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("legacyPackages"))) { + try { + auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } + } + + // Otherwise, look for it in 'outputs'. + for (auto & attrPath : attrPaths) { + try { + auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); } }; @@ -190,45 +279,75 @@ struct InstallableAttrPath : InstallableValue std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); -static std::vector<std::shared_ptr<Installable>> parseInstallables( - SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables) +std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( + ref<Store> store, std::vector<std::string> ss) { std::vector<std::shared_ptr<Installable>> result; - if (ss.empty() && useDefaultInstallables) { - if (cmd.file == "") - cmd.file = "."; - ss = {""}; - } + if (file) { + // FIXME: backward compatibility hack + evalSettings.pureEval = false; - for (auto & s : ss) { + auto state = getEvalState(); + auto vFile = state->allocValue(); + state->evalFile(lookupFileArg(*state, *file), *vFile); - if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(cmd, s)); + if (ss.empty()) + ss = {""}; - else if (s.find("/") != std::string::npos) { + for (auto & s : ss) + result.push_back(std::make_shared<InstallableAttrPath>(*this, vFile, s)); - auto path = store->toStorePath(store->followLinksToStore(s)); + } else { - if (store->isStorePath(path)) - result.push_back(std::make_shared<InstallableStorePath>(path)); - } + for (auto & s : ss) { + + size_t colon; + + if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared<InstallableExpr>(*this, s)); - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared<InstallableAttrPath>(cmd, s)); + else if (hasPrefix(s, "nixpkgs.")) { + bool static warned; + warnOnce(warned, "the syntax 'nixpkgs.<attr>' is deprecated; use 'nixpkgs:<attr>' instead"); + result.push_back(std::make_shared<InstallableFlake>(*this, FlakeRef("nixpkgs"), + Strings{"packages." + std::string(s, 8)})); + } - else - throw UsageError("don't know what to do with argument '%s'", s); + else if (auto flakeRef = parseFlakeRef(s, true)) + result.push_back(std::make_shared<InstallableFlake>(*this, std::move(*flakeRef), + getDefaultFlakeAttrPaths())); + + else if ((colon = s.rfind(':')) != std::string::npos) { + auto flakeRef = std::string(s, 0, colon); + auto attrPath = std::string(s, colon + 1); + result.push_back(std::make_shared<InstallableFlake>(*this, FlakeRef(flakeRef, true), attrPath)); + } + + else if (s.find('/') != std::string::npos || s == ".") { + Path storePath; + try { + storePath = store->toStorePath(store->followLinksToStore(s)); + } catch (Error) { } + if (storePath != "") + result.push_back(std::make_shared<InstallableStorePath>(storePath)); + else + result.push_back(std::make_shared<InstallableFlake>(*this, FlakeRef(s, true), + getDefaultFlakeAttrPaths())); + } + + else + result.push_back(std::make_shared<InstallableFlake>(*this, FlakeRef("nixpkgs"), s)); + } } return result; } -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables) +std::shared_ptr<Installable> SourceExprCommand::parseInstallable( + ref<Store> store, const std::string & installable) { - auto installables = parseInstallables(cmd, store, {installable}, false); + auto installables = parseInstallables(store, {installable}); assert(installables.size() == 1); return installables.front(); } @@ -284,7 +403,7 @@ Path toStorePath(ref<Store> store, RealiseMode mode, auto paths = toStorePaths(store, mode, {installable}); if (paths.size() != 1) - throw Error("argument '%s' should evaluate to one store path", installable->what()); + throw Error("argument '%s' should evaluate to one store path", installable->what()); return *paths.begin(); } @@ -315,12 +434,16 @@ PathSet toDerivations(ref<Store> store, void InstallablesCommand::prepare() { - installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); + if (_installables.empty() && !file && useDefaultInstallables()) + // FIXME: commands like "nix install" should not have a + // default, probably. + _installables.push_back("."); + installables = parseInstallables(getStore(), _installables); } void InstallableCommand::prepare() { - installable = parseInstallable(*this, getStore(), _installable, false); + installable = parseInstallable(getStore(), _installable); } } diff --git a/src/nix/local.mk b/src/nix/local.mk index ca4604d56..4003d0005 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -23,3 +23,5 @@ $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ $(eval $(call install-symlink, nix, $(bindir)/$(name)))) $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) + +$(d)/flake.cc: $(d)/flake-template.nix.gen.hh diff --git a/src/nix/main.cc b/src/nix/main.cc index 4f87ad72b..a1fcb892a 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -64,10 +64,20 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs "--help-config' for a list of configuration settings.\n"; } + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + +#if 0 + out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n"; +#endif + + std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + } + void showHelpAndExit() { printHelp(programName, std::cout); - std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; throw Exit(); } }; @@ -94,6 +104,7 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlError; settings.verboseBuild = false; + evalSettings.pureEval = true; NixArgs args; diff --git a/src/nix/search.cc b/src/nix/search.cc index e086de226..55f8d106a 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -257,7 +257,9 @@ struct CmdSearch : SourceExprCommand, MixJSON auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr; - doExpr(getSourceExpr(*state), "", true, cache.get()); + // FIXME + throw Error("NOT IMPLEMENTED"); + //doExpr(getSourceExpr(*state), "", true, cache.get()); } catch (std::exception &) { /* Fun fact: catching std::ios::failure does not work diff --git a/src/nix/shell.cc b/src/nix/shell.cc new file mode 100644 index 000000000..2ccad930f --- /dev/null +++ b/src/nix/shell.cc @@ -0,0 +1,283 @@ +#include "eval.hh" +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct BuildEnvironment +{ + // FIXME: figure out which vars should be exported. + std::map<std::string, std::string> env; + std::map<std::string, std::string> functions; +}; + +BuildEnvironment readEnvironment(const Path & path) +{ + BuildEnvironment res; + + auto lines = tokenizeString<Strings>(readFile(path), "\n"); + + auto getLine = + [&]() { + if (lines.empty()) + throw Error("shell environment '%s' ends unexpectedly", path); + auto line = lines.front(); + lines.pop_front(); + return line; + }; + + while (!lines.empty()) { + auto line = getLine(); + + auto eq = line.find('='); + if (eq != std::string::npos) { + std::string name(line, 0, eq); + std::string value(line, eq + 1); + // FIXME: parse arrays + res.env.insert({name, value}); + } + + else if (hasSuffix(line, " () ")) { + std::string name(line, 0, line.size() - 4); + // FIXME: validate name + auto l = getLine(); + if (l != "{ ") throw Error("shell environment '%s' has unexpected line '%s'", path, l); + std::string body; + while ((l = getLine()) != "}") { + body += l; + body += '\n'; + } + res.functions.insert({name, body}); + } + + else throw Error("shell environment '%s' has unexpected line '%s'", path, line); + } + + return res; +} + +/* Given an existing derivation, return the shell environment as + initialised by stdenv's setup script. We do this by building a + modified derivation with the same dependencies and nearly the same + initial environment variables, that just writes the resulting + environment to a file and exits. */ +BuildEnvironment getDerivationEnvironment(ref<Store> store, Derivation drv) +{ + auto builder = baseNameOf(drv.builder); + if (builder != "bash") + throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); + + drv.args = {"-c", "set -e; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; + + /* Remove derivation checks. */ + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + + // FIXME: handle structured attrs + + /* Rehash and write the derivation. FIXME: would be nice to use + 'buildDerivation', but that's privileged. */ + auto drvName = drv.env["name"] + "-env"; + for (auto & output : drv.outputs) + drv.env.erase(output.first); + drv.env["out"] = ""; + drv.env["outputs"] = "out"; + drv.outputs["out"] = DerivationOutput("", "", ""); + Hash h = hashDerivationModulo(*store, drv); + Path shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs["out"].path = shellOutPath; + drv.env["out"] = shellOutPath; + Path shellDrvPath2 = writeDerivation(store, drv, drvName); + + /* Build the derivation. */ + store->buildPaths({shellDrvPath2}); + + assert(store->isValidPath(shellOutPath)); + + return readEnvironment(shellOutPath); +} + +struct Common : InstallableCommand +{ + /* + std::set<string> keepVars{ + "DISPLAY", + "HOME", + "IN_NIX_SHELL", + "LOGNAME", + "NIX_BUILD_SHELL", + "PAGER", + "PATH", + "TERM", + "TZ", + "USER", + }; + */ + + std::set<string> ignoreVars{ + "BASHOPTS", + "EUID", + "HOME", // FIXME: don't ignore in pure mode? + "NIX_BUILD_TOP", + "NIX_ENFORCE_PURITY", + "PPID", + "PWD", + "SHELLOPTS", + "SHLVL", + "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt + "TEMP", + "TEMPDIR", + "TERM", + "TMP", + "TMPDIR", + "TZ", + "UID", + }; + + void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + { + out << "export IN_NIX_SHELL=1\n"; + out << "nix_saved_PATH=\"$PATH\"\n"; + + for (auto & i : buildEnvironment.env) { + // FIXME: shellEscape + // FIXME: figure out what to export + // FIXME: handle arrays + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) + out << fmt("export %s=%s\n", i.first, i.second); + } + + out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + + for (auto & i : buildEnvironment.functions) { + out << fmt("%s () {\n%s\n}\n", i.first, i.second); + } + + // FIXME: set outputs + + out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) + out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + + out << "eval \"$shellHook\"\n"; + } + + Strings getDefaultFlakeAttrPaths() override + { + return {"devShell", "defaultPackage"}; + } +}; + +struct CmdDevShell : Common +{ + + std::string name() override + { + return "dev-shell"; + } + + std::string description() override + { + return "run a bash shell that provides the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To get the build environment of GNU hello:", + "nix dev-shell nixpkgs:hello" + }, + Example{ + "To get the build environment of the default package of flake in the current directory:", + "nix dev-shell" + }, + }; + } + + void run(ref<Store> store) override + { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); + + std::ostringstream ss; + makeRcScript(buildEnvironment, ss); + + ss << fmt("rm -f '%s'\n", rcFilePath); + + writeFull(rcFileFd.get(), ss.str()); + + stopProgressBar(); + + auto shell = getEnv("SHELL", "bash"); + + auto args = Strings{baseNameOf(shell), "--rcfile", rcFilePath}; + + restoreAffinity(); + restoreSignals(); + + execvp(shell.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing shell '%s'", shell); + } +}; + +struct CmdPrintDevEnv : Common +{ + + std::string name() override + { + return "print-dev-env"; + } + + std::string description() override + { + return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To apply the build environment of GNU hello to the current shell:", + ". <(nix print-dev-env nixpkgs:hello)" + }, + }; + } + + void run(ref<Store> store) override + { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + stopProgressBar(); + + makeRcScript(buildEnvironment, std::cout); + } +}; + +static RegisterCommand r1(make_ref<CmdPrintDevEnv>()); +static RegisterCommand r2(make_ref<CmdDevShell>()); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 325a2be0a..32ba5a1ad 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -74,9 +74,9 @@ struct CmdWhyDepends : SourceExprCommand void run(ref<Store> store) override { - auto package = parseInstallable(*this, store, _package, false); + auto package = parseInstallable(store, _package); auto packagePath = toStorePath(store, Build, package); - auto dependency = parseInstallable(*this, store, _dependency, false); + auto dependency = parseInstallable(store, _dependency); auto dependencyPath = toStorePath(store, NoBuild, dependency); auto dependencyPathHash = storePathToHash(dependencyPath); diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp index c9af0bed3..5003a4fa2 100644 --- a/src/nlohmann/json.hpp +++ b/src/nlohmann/json.hpp @@ -1,12 +1,12 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.5.0 +| | |__ | | | | | | version 3.6.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License <http://opensource.org/licenses/MIT>. SPDX-License-Identifier: MIT -Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>. +Copyright (c) 2013-2019 Niels Lohmann <http://nlohmann.me>. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -27,12 +27,12 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ #define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 5 -#define NLOHMANN_JSON_VERSION_PATCH 0 +#define NLOHMANN_JSON_VERSION_MINOR 6 +#define NLOHMANN_JSON_VERSION_PATCH 1 #include <algorithm> // all_of, find, for_each #include <cassert> // assert @@ -42,79 +42,417 @@ SOFTWARE. #include <initializer_list> // initializer_list #include <iosfwd> // istream, ostream #include <iterator> // random_access_iterator_tag +#include <memory> // unique_ptr #include <numeric> // accumulate #include <string> // string, stoi, to_string #include <utility> // declval, forward, move, pair, swap +#include <vector> // vector -// #include <nlohmann/json_fwd.hpp> -#ifndef NLOHMANN_JSON_FWD_HPP -#define NLOHMANN_JSON_FWD_HPP +// #include <nlohmann/adl_serializer.hpp> -#include <cstdint> // int64_t, uint64_t + +#include <utility> + +// #include <nlohmann/detail/conversions/from_json.hpp> + + +#include <algorithm> // transform +#include <array> // array +#include <ciso646> // and, not +#include <forward_list> // forward_list +#include <iterator> // inserter, front_inserter, end #include <map> // map -#include <memory> // allocator #include <string> // string -#include <vector> // vector +#include <tuple> // tuple, make_tuple +#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include <unordered_map> // unordered_map +#include <utility> // pair, declval +#include <valarray> // valarray + +// #include <nlohmann/detail/exceptions.hpp> + + +#include <exception> // exception +#include <stdexcept> // runtime_error +#include <string> // to_string + +// #include <nlohmann/detail/input/position_t.hpp> + + +#include <cstddef> // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// /*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 +@brief general exception of the @ref basic_json class + +This class is an extension of `std::exception` objects with a member @a id for +exception ids. It is used as the base class for all exceptions thrown by the +@ref basic_json class. This class can hence be used as "wildcard" to catch +exceptions. + +Subclasses: +- @ref parse_error for exceptions indicating a parse error +- @ref invalid_iterator for exceptions indicating errors with iterators +- @ref type_error for exceptions indicating executing a member function with + a wrong type +- @ref out_of_range for exceptions indicating access out of the defined range +- @ref other_error for exceptions indicating other library errors + +@internal +@note To have nothrow-copy-constructible exceptions, we internally use + `std::runtime_error` which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. +@endinternal + +@liveexample{The following code shows how arbitrary library exceptions can be +caught.,exception} + +@since version 3.0.0 */ -namespace nlohmann +class exception : public std::exception { + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + /*! -@brief default JSONSerializer template argument +@brief exception indicating a parse error -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) -for serialization. +This exception is thrown by the library when a parse error occurs. Parse errors +can occur during the deserialization of JSON text, CBOR, MessagePack, as well +as when using JSON Patch. + +Member @a byte holds the byte index of the last read character in the input +file. + +Exceptions have ids 1xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). + +@note For an input with n bytes, 1 is the index of the first character and n+1 + is the index of the terminating null byte or the end of file. This also + holds true when reading a byte vector (CBOR or MessagePack). + +@liveexample{The following code shows how a `parse_error` exception can be +caught.,parse_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 */ -template<typename T = void, typename SFINAE = void> -struct adl_serializer; +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] pos the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } -template<template<typename U, typename V, typename... Args> class ObjectType = - std::map, - template<typename U, typename... Args> class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template<typename U> class AllocatorType = std::allocator, - template<typename T, typename SFINAE = void> class JSONSerializer = - adl_serializer> -class basic_json; + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id_, byte_, w.c_str()); + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; /*! -@brief JSON Pointer +@brief exception indicating errors with iterators -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. +This exception is thrown if iterators passed to a library function do not match +the expected semantics. -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) +Exceptions have ids 2xx. -@since version 2.0.0 +name / id | example message | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@liveexample{The following code shows how an `invalid_iterator` exception can be +caught.,invalid_iterator} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 */ -template<typename BasicJsonType> -class json_pointer; +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id_, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id_) + what_arg; + return invalid_iterator(id_, w.c_str()); + } + + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; /*! -@brief default JSON class +@brief exception indicating executing a member function with a wrong type -This type is the default specialization of the @ref basic_json class which -uses the standard template types. +This exception is thrown in case of a type error; that is, a library function is +executed on a JSON value whose type does not match the expected semantics. -@since version 1.0.0 +Exceptions have ids 3xx. + +name / id | example message | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t &. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. +json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | + +@liveexample{The following code shows how a `type_error` exception can be +caught.,type_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 */ -using json = basic_json<>; -} // namespace nlohmann +class type_error : public exception +{ + public: + static type_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("type_error", id_) + what_arg; + return type_error(id_, w.c_str()); + } -#endif + private: + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating access out of the defined range + +This exception is thrown in case a library function is called on an input +parameter that exceeds the expected range, for instance in case of array +indices or nonexisting object keys. + +Exceptions have ids 4xx. + +name / id | example message | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | + +@liveexample{The following code shows how an `out_of_range` exception can be +caught.,out_of_range} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class out_of_range : public exception +{ + public: + static out_of_range create(int id_, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id_) + what_arg; + return out_of_range(id_, w.c_str()); + } + + private: + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating other library errors + +This exception is thrown in case of errors that cannot be classified with the +other exception types. + +Exceptions have ids 5xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range + +@liveexample{The following code shows how an `other_error` exception can be +caught.,other_error} + +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("other_error", id_) + what_arg; + return other_error(id_, w.c_str()); + } + + private: + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; +} // namespace detail +} // namespace nlohmann // #include <nlohmann/detail/macro_scope.hpp> +#include <utility> // pair + // This file contains all internal macro definitions // You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them @@ -152,6 +490,19 @@ using json = basic_json<>; #define JSON_DEPRECATED #endif +// allow for portable nodiscard warnings +#if defined(__has_cpp_attribute) + #if __has_cpp_attribute(nodiscard) + #define JSON_NODISCARD [[nodiscard]] + #elif __has_cpp_attribute(gnu::warn_unused_result) + #define JSON_NODISCARD [[gnu::warn_unused_result]] + #else + #define JSON_NODISCARD + #endif +#else + #define JSON_NODISCARD +#endif + // allow to disable exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception @@ -159,6 +510,7 @@ using json = basic_json<>; #define JSON_CATCH(exception) catch(exception) #define JSON_INTERNAL_CATCH(exception) catch(exception) #else + #include <cstdlib> #define JSON_THROW(exception) std::abort() #define JSON_TRY if(true) #define JSON_CATCH(exception) if(false) @@ -187,8 +539,8 @@ using json = basic_json<>; // manual branch prediction #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) - #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) + #define JSON_LIKELY(x) __builtin_expect(x, 1) + #define JSON_UNLIKELY(x) __builtin_expect(x, 0) #else #define JSON_LIKELY(x) x #define JSON_UNLIKELY(x) x @@ -322,8 +674,6 @@ constexpr T static_const<T>::value; #include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type #include <utility> // declval -// #include <nlohmann/json_fwd.hpp> - // #include <nlohmann/detail/iterators/iterator_traits.hpp> @@ -389,8 +739,10 @@ struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>> using pointer = T*; using reference = T&; }; -} -} +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> // #include <nlohmann/detail/meta/cpp_future.hpp> @@ -412,7 +764,9 @@ struct nonesuch nonesuch() = delete; ~nonesuch() = delete; nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; }; template <class Default, @@ -453,7 +807,71 @@ using is_detected_convertible = } // namespace detail } // namespace nlohmann -// #include <nlohmann/detail/macro_scope.hpp> +// #include <nlohmann/json_fwd.hpp> +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + +#include <cstdint> // int64_t, uint64_t +#include <map> // map +#include <memory> // allocator +#include <string> // string +#include <vector> // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template<typename T = void, typename SFINAE = void> +struct adl_serializer; + +template<template<typename U, typename V, typename... Args> class ObjectType = + std::map, + template<typename U, typename... Args> class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template<typename U> class AllocatorType = std::allocator, + template<typename T, typename SFINAE = void> class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template<typename BasicJsonType> +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ namespace nlohmann @@ -792,385 +1210,6 @@ struct is_compatible_type } // namespace detail } // namespace nlohmann -// #include <nlohmann/detail/exceptions.hpp> - - -#include <exception> // exception -#include <stdexcept> // runtime_error -#include <string> // to_string - -// #include <nlohmann/detail/input/position_t.hpp> - - -#include <cstddef> // size_t - -namespace nlohmann -{ -namespace detail -{ -/// struct to capture the start position of the current token -struct position_t -{ - /// the total number of characters read - std::size_t chars_read_total = 0; - /// the number of characters read in the current line - std::size_t chars_read_current_line = 0; - /// the number of lines read - std::size_t lines_read = 0; - - /// conversion to size_t to preserve SAX interface - constexpr operator size_t() const - { - return chars_read_total; - } -}; - -} -} - - -namespace nlohmann -{ -namespace detail -{ -//////////////// -// exceptions // -//////////////// - -/*! -@brief general exception of the @ref basic_json class - -This class is an extension of `std::exception` objects with a member @a id for -exception ids. It is used as the base class for all exceptions thrown by the -@ref basic_json class. This class can hence be used as "wildcard" to catch -exceptions. - -Subclasses: -- @ref parse_error for exceptions indicating a parse error -- @ref invalid_iterator for exceptions indicating errors with iterators -- @ref type_error for exceptions indicating executing a member function with - a wrong type -- @ref out_of_range for exceptions indicating access out of the defined range -- @ref other_error for exceptions indicating other library errors - -@internal -@note To have nothrow-copy-constructible exceptions, we internally use - `std::runtime_error` which can cope with arbitrary-length error messages. - Intermediate strings are built with static functions and then passed to - the actual constructor. -@endinternal - -@liveexample{The following code shows how arbitrary library exceptions can be -caught.,exception} - -@since version 3.0.0 -*/ -class exception : public std::exception -{ - public: - /// returns the explanatory string - const char* what() const noexcept override - { - return m.what(); - } - - /// the id of the exception - const int id; - - protected: - exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} - - static std::string name(const std::string& ename, int id_) - { - return "[json.exception." + ename + "." + std::to_string(id_) + "] "; - } - - private: - /// an exception object as storage for error messages - std::runtime_error m; -}; - -/*! -@brief exception indicating a parse error - -This exception is thrown by the library when a parse error occurs. Parse errors -can occur during the deserialization of JSON text, CBOR, MessagePack, as well -as when using JSON Patch. - -Member @a byte holds the byte index of the last read character in the input -file. - -Exceptions have ids 1xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. -json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. -json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. -json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. -json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. -json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. -json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. -json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. -json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. -json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. -json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. -json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. -json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). - -@note For an input with n bytes, 1 is the index of the first character and n+1 - is the index of the terminating null byte or the end of file. This also - holds true when reading a byte vector (CBOR or MessagePack). - -@liveexample{The following code shows how a `parse_error` exception can be -caught.,parse_error} - -@sa @ref exception for the base class of the library exceptions -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref out_of_range for exceptions indicating access out of the defined range -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class parse_error : public exception -{ - public: - /*! - @brief create a parse error exception - @param[in] id_ the id of the exception - @param[in] position the position where the error occurred (or with - chars_read_total=0 if the position cannot be - determined) - @param[in] what_arg the explanatory string - @return parse_error object - */ - static parse_error create(int id_, const position_t& pos, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - position_string(pos) + ": " + what_arg; - return parse_error(id_, pos.chars_read_total, w.c_str()); - } - - static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + - ": " + what_arg; - return parse_error(id_, byte_, w.c_str()); - } - - /*! - @brief byte index of the parse error - - The byte index of the last read character in the input file. - - @note For an input with n bytes, 1 is the index of the first character and - n+1 is the index of the terminating null byte or the end of file. - This also holds true when reading a byte vector (CBOR or MessagePack). - */ - const std::size_t byte; - - private: - parse_error(int id_, std::size_t byte_, const char* what_arg) - : exception(id_, what_arg), byte(byte_) {} - - static std::string position_string(const position_t& pos) - { - return " at line " + std::to_string(pos.lines_read + 1) + - ", column " + std::to_string(pos.chars_read_current_line); - } -}; - -/*! -@brief exception indicating errors with iterators - -This exception is thrown if iterators passed to a library function do not match -the expected semantics. - -Exceptions have ids 2xx. - -name / id | example message | description ------------------------------------ | --------------- | ------------------------- -json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. -json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. -json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. -json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. -json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. -json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. -json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. -json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. -json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. -json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). - -@liveexample{The following code shows how an `invalid_iterator` exception can be -caught.,invalid_iterator} - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref out_of_range for exceptions indicating access out of the defined range -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class invalid_iterator : public exception -{ - public: - static invalid_iterator create(int id_, const std::string& what_arg) - { - std::string w = exception::name("invalid_iterator", id_) + what_arg; - return invalid_iterator(id_, w.c_str()); - } - - private: - invalid_iterator(int id_, const char* what_arg) - : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating executing a member function with a wrong type - -This exception is thrown in case of a type error; that is, a library function is -executed on a JSON value whose type does not match the expected semantics. - -Exceptions have ids 3xx. - -name / id | example message | description ------------------------------ | --------------- | ------------------------- -json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. -json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. -json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&. -json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. -json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. -json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. -json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. -json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. -json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. -json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. -json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. -json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. -json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. -json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. -json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. -json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | -json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | - -@liveexample{The following code shows how a `type_error` exception can be -caught.,type_error} - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref out_of_range for exceptions indicating access out of the defined range -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class type_error : public exception -{ - public: - static type_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("type_error", id_) + what_arg; - return type_error(id_, w.c_str()); - } - - private: - type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating access out of the defined range - -This exception is thrown in case a library function is called on an input -parameter that exceeds the expected range, for instance in case of array -indices or nonexisting object keys. - -Exceptions have ids 4xx. - -name / id | example message | description -------------------------------- | --------------- | ------------------------- -json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. -json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. -json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. -json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. -json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. -json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | -json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | -json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | - -@liveexample{The following code shows how an `out_of_range` exception can be -caught.,out_of_range} - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class out_of_range : public exception -{ - public: - static out_of_range create(int id_, const std::string& what_arg) - { - std::string w = exception::name("out_of_range", id_) + what_arg; - return out_of_range(id_, w.c_str()); - } - - private: - out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating other library errors - -This exception is thrown in case of errors that cannot be classified with the -other exception types. - -Exceptions have ids 5xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref out_of_range for exceptions indicating access out of the defined range - -@liveexample{The following code shows how an `other_error` exception can be -caught.,other_error} - -@since version 3.0.0 -*/ -class other_error : public exception -{ - public: - static other_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("other_error", id_) + what_arg; - return other_error(id_, w.c_str()); - } - - private: - other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; -} // namespace detail -} // namespace nlohmann - // #include <nlohmann/detail/value_t.hpp> @@ -1178,6 +1217,7 @@ class other_error : public exception #include <ciso646> // and #include <cstddef> // size_t #include <cstdint> // uint8_t +#include <string> // string namespace nlohmann { @@ -1249,32 +1289,6 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept } // namespace detail } // namespace nlohmann -// #include <nlohmann/detail/conversions/from_json.hpp> - - -#include <algorithm> // transform -#include <array> // array -#include <ciso646> // and, not -#include <forward_list> // forward_list -#include <iterator> // inserter, front_inserter, end -#include <map> // map -#include <string> // string -#include <tuple> // tuple, make_tuple -#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible -#include <unordered_map> // unordered_map -#include <utility> // pair, declval -#include <valarray> // valarray - -// #include <nlohmann/detail/exceptions.hpp> - -// #include <nlohmann/detail/macro_scope.hpp> - -// #include <nlohmann/detail/meta/cpp_future.hpp> - -// #include <nlohmann/detail/meta/type_traits.hpp> - -// #include <nlohmann/detail/value_t.hpp> - namespace nlohmann { @@ -1629,32 +1643,28 @@ constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::va // #include <nlohmann/detail/conversions/to_json.hpp> +#include <algorithm> // copy #include <ciso646> // or, and, not #include <iterator> // begin, end +#include <string> // string #include <tuple> // tuple, get #include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type #include <utility> // move, forward, declval, pair #include <valarray> // valarray #include <vector> // vector -// #include <nlohmann/detail/meta/cpp_future.hpp> - -// #include <nlohmann/detail/meta/type_traits.hpp> - -// #include <nlohmann/detail/value_t.hpp> - // #include <nlohmann/detail/iterators/iteration_proxy.hpp> #include <cstddef> // size_t -#include <string> // string, to_string #include <iterator> // input_iterator_tag +#include <string> // string, to_string #include <tuple> // tuple_size, get, tuple_element -// #include <nlohmann/detail/value_t.hpp> - // #include <nlohmann/detail/meta/type_traits.hpp> +// #include <nlohmann/detail/value_t.hpp> + namespace nlohmann { @@ -1700,13 +1710,13 @@ template <typename IteratorType> class iteration_proxy_value } /// equality operator (needed for InputIterator) - bool operator==(const iteration_proxy_value& o) const noexcept + bool operator==(const iteration_proxy_value& o) const { return anchor == o.anchor; } /// inequality operator (needed for range-based for) - bool operator!=(const iteration_proxy_value& o) const noexcept + bool operator!=(const iteration_proxy_value& o) const { return anchor != o.anchor; } @@ -1795,6 +1805,11 @@ auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decl // And see https://github.com/nlohmann/json/pull/1391 namespace std { +#if defined(__clang__) + // Fix: https://github.com/nlohmann/json/issues/1401 + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wmismatched-tags" +#endif template <typename IteratorType> class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>> : public std::integral_constant<std::size_t, 2> {}; @@ -1807,7 +1822,17 @@ class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >> get<N>(std::declval < ::nlohmann::detail::iteration_proxy_value<IteratorType >> ())); }; -} +#if defined(__clang__) + #pragma clang diagnostic pop +#endif +} // namespace std + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + +// #include <nlohmann/detail/value_t.hpp> + namespace nlohmann { @@ -2137,11 +2162,81 @@ constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; } // namespace } // namespace nlohmann + +namespace nlohmann +{ + +template<typename, typename> +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template<typename BasicJsonType, typename ValueType> + static auto from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) + -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void()) + { + ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template <typename BasicJsonType, typename ValueType> + static auto to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) + -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void()) + { + ::nlohmann::to_json(j, std::forward<ValueType>(val)); + } +}; + +} // namespace nlohmann + +// #include <nlohmann/detail/conversions/from_json.hpp> + +// #include <nlohmann/detail/conversions/to_json.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/input/binary_reader.hpp> + + +#include <algorithm> // generate_n +#include <array> // array +#include <cassert> // assert +#include <cmath> // ldexp +#include <cstddef> // size_t +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstdio> // snprintf +#include <cstring> // memcpy +#include <iterator> // back_inserter +#include <limits> // numeric_limits +#include <string> // char_traits, string +#include <utility> // make_pair, move + +// #include <nlohmann/detail/exceptions.hpp> + // #include <nlohmann/detail/input/input_adapters.hpp> +#include <array> // array #include <cassert> // assert #include <cstddef> // size_t +#include <cstdio> //FILE * #include <cstring> // strlen #include <istream> // istream #include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next @@ -2150,7 +2245,8 @@ constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; #include <string> // string, char_traits #include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer #include <utility> // pair, declval -#include <cstdio> //FILE * + +// #include <nlohmann/detail/iterators/iterator_traits.hpp> // #include <nlohmann/detail/macro_scope.hpp> @@ -2198,10 +2294,18 @@ class file_input_adapter : public input_adapter_protocol : m_file(f) {} + // make class move-only + file_input_adapter(const file_input_adapter&) = delete; + file_input_adapter(file_input_adapter&&) = default; + file_input_adapter& operator=(const file_input_adapter&) = delete; + file_input_adapter& operator=(file_input_adapter&&) = default; + ~file_input_adapter() override = default; + std::char_traits<char>::int_type get_character() noexcept override { return std::fgetc(m_file); } + private: /// the file pointer to read from std::FILE* m_file; @@ -2293,7 +2397,11 @@ template<typename WideStringType, size_t T> struct wide_string_input_helper { // UTF-32 - static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + static void fill_buffer(const WideStringType& str, + size_t& current_wchar, + std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) { utf8_bytes_index = 0; @@ -2305,39 +2413,39 @@ struct wide_string_input_helper else { // get the current character - const auto wc = static_cast<int>(str[current_wchar++]); + const auto wc = static_cast<unsigned int>(str[current_wchar++]); // UTF-32 to UTF-8 encoding if (wc < 0x80) { - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc); utf8_bytes_filled = 1; } else if (wc <= 0x7FF) { - utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); - utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xC0u | ((wc >> 6u) & 0x1Fu)); + utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 2; } else if (wc <= 0xFFFF) { - utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); - utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xE0u | ((wc >> 12u) & 0x0Fu)); + utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((wc >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 3; } else if (wc <= 0x10FFFF) { - utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07); - utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); - utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[3] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | ((wc >> 18u) & 0x07u)); + utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((wc >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | ((wc >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast<std::char_traits<char>::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 4; } else { // unknown character - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc); utf8_bytes_filled = 1; } } @@ -2348,7 +2456,11 @@ template<typename WideStringType> struct wide_string_input_helper<WideStringType, 2> { // UTF-16 - static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + static void fill_buffer(const WideStringType& str, + size_t& current_wchar, + std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) { utf8_bytes_index = 0; @@ -2360,44 +2472,44 @@ struct wide_string_input_helper<WideStringType, 2> else { // get the current character - const auto wc = static_cast<int>(str[current_wchar++]); + const auto wc = static_cast<unsigned int>(str[current_wchar++]); // UTF-16 to UTF-8 encoding if (wc < 0x80) { - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc); utf8_bytes_filled = 1; } else if (wc <= 0x7FF) { - utf8_bytes[0] = 0xC0 | ((wc >> 6)); - utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xC0u | ((wc >> 6u))); + utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 2; } else if (0xD800 > wc or wc >= 0xE000) { - utf8_bytes[0] = 0xE0 | ((wc >> 12)); - utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xE0u | ((wc >> 12u))); + utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((wc >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 3; } else { if (current_wchar < str.size()) { - const auto wc2 = static_cast<int>(str[current_wchar++]); - const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF)); - utf8_bytes[0] = 0xf0 | (charcode >> 18); - utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F); - utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F); - utf8_bytes[3] = 0x80 | (charcode & 0x3F); + const auto wc2 = static_cast<unsigned int>(str[current_wchar++]); + const auto charcode = 0x10000u + (((wc & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | (charcode >> 18u)); + utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast<std::char_traits<char>::int_type>(0x80u | (charcode & 0x3Fu)); utf8_bytes_filled = 4; } else { // unknown character ++current_wchar; - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc); utf8_bytes_filled = 1; } } @@ -2409,7 +2521,7 @@ template<typename WideStringType> class wide_string_input_adapter : public input_adapter_protocol { public: - explicit wide_string_input_adapter(const WideStringType& w) noexcept + explicit wide_string_input_adapter(const WideStringType& w) noexcept : str(w) {} @@ -2561,23 +2673,2847 @@ class input_adapter } // namespace detail } // namespace nlohmann +// #include <nlohmann/detail/input/json_sax.hpp> + + +#include <cassert> // assert +#include <cstddef> +#include <string> // string +#include <utility> // move +#include <vector> // vector + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template<typename BasicJsonType> +struct json_sax +{ + /// type for (signed) integers + using number_integer_t = typename BasicJsonType::number_integer_t; + /// type for unsigned integers + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type for floating-point numbers + using number_float_t = typename BasicJsonType::number_float_t; + /// type for strings + using string_t = typename BasicJsonType::string_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief an floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template<typename BasicJsonType> +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + /*! + @param[in, out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + // make class move-only + json_sax_dom_parser(const json_sax_dom_parser&) = delete; + json_sax_dom_parser(json_sax_dom_parser&&) = default; + json_sax_dom_parser& operator=(const json_sax_dom_parser&) = delete; + json_sax_dom_parser& operator=(json_sax_dom_parser&&) = default; + ~json_sax_dom_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + ref_stack.pop_back(); + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*static_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*static_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*static_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*static_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*static_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template<typename Value> + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward<Value>(v)); + return &root; + } + + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v)); + return &(ref_stack.back()->m_value.array->back()); + } + + assert(ref_stack.back()->is_object()); + assert(object_element); + *object_element = BasicJsonType(std::forward<Value>(v)); + return object_element; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template<typename BasicJsonType> +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + // make class move-only + json_sax_dom_callback_parser(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser(json_sax_dom_callback_parser&&) = default; + json_sax_dom_callback_parser& operator=(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser& operator=(json_sax_dom_callback_parser&&) = default; + ~json_sax_dom_callback_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back() and JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep and ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back() and not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (not ref_stack.empty() and ref_stack.back() and ref_stack.back()->is_object()) + { + // remove discarded value + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back() and JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + // discard array + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (not keep and not ref_stack.empty() and ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*static_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*static_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*static_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*static_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*static_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template<typename Value> + std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false) + { + assert(not keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (not keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward<Value>(v)); + + // check callback + const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (not keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (not ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + // array + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->push_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + + // object + assert(ref_stack.back()->is_object()); + // check if we should store an element for the current key + assert(not key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (not store_element) + { + return {false, nullptr}; + } + + assert(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack {}; + /// stack to manage which values to keep + std::vector<bool> keep_stack {}; + /// stack to manage which object keys to keep + std::vector<bool> key_keep_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template<typename BasicJsonType> +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + + +#include <cstdint> // size_t +#include <utility> // declval +#include <string> // string + +// #include <nlohmann/detail/meta/detected.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename T> +using null_function_t = decltype(std::declval<T&>().null()); + +template <typename T> +using boolean_function_t = + decltype(std::declval<T&>().boolean(std::declval<bool>())); + +template <typename T, typename Integer> +using number_integer_function_t = + decltype(std::declval<T&>().number_integer(std::declval<Integer>())); + +template <typename T, typename Unsigned> +using number_unsigned_function_t = + decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>())); + +template <typename T, typename Float, typename String> +using number_float_function_t = decltype(std::declval<T&>().number_float( + std::declval<Float>(), std::declval<const String&>())); + +template <typename T, typename String> +using string_function_t = + decltype(std::declval<T&>().string(std::declval<String&>())); + +template <typename T> +using start_object_function_t = + decltype(std::declval<T&>().start_object(std::declval<std::size_t>())); + +template <typename T, typename String> +using key_function_t = + decltype(std::declval<T&>().key(std::declval<String&>())); + +template <typename T> +using end_object_function_t = decltype(std::declval<T&>().end_object()); + +template <typename T> +using start_array_function_t = + decltype(std::declval<T&>().start_array(std::declval<std::size_t>())); + +template <typename T> +using end_array_function_t = decltype(std::declval<T&>().end_array()); + +template <typename T, typename Exception> +using parse_error_function_t = decltype(std::declval<T&>().parse_error( + std::declval<std::size_t>(), std::declval<const std::string&>(), + std::declval<const Exception&>())); + +template <typename SAX, typename BasicJsonType> +struct is_sax +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact<bool, null_function_t, SAX>::value && + is_detected_exact<bool, boolean_function_t, SAX>::value && + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value && + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value && + is_detected_exact<bool, number_float_function_t, SAX, number_float_t, + string_t>::value && + is_detected_exact<bool, string_function_t, SAX, string_t>::value && + is_detected_exact<bool, start_object_function_t, SAX>::value && + is_detected_exact<bool, key_function_t, SAX, string_t>::value && + is_detected_exact<bool, end_object_function_t, SAX>::value && + is_detected_exact<bool, start_array_function_t, SAX>::value && + is_detected_exact<bool, end_array_function_t, SAX>::value && + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value; +}; + +template <typename SAX, typename BasicJsonType> +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact<bool, null_function_t, SAX>::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact<bool, number_float_function_t, SAX, + number_float_t, string_t>::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact<bool, string_function_t, SAX, string_t>::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary reader // +/////////////////// + +/*! +@brief deserialization of CBOR, MessagePack, and UBJSON values +*/ +template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>> +class binary_reader +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using json_sax_t = SAX; + + public: + /*! + @brief create a binary reader + + @param[in] adapter input adapter to read from + */ + explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) + { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; + assert(ia); + } + + // make class move-only + binary_reader(const binary_reader&) = delete; + binary_reader(binary_reader&&) = default; + binary_reader& operator=(const binary_reader&) = delete; + binary_reader& operator=(binary_reader&&) = default; + ~binary_reader() = default; + + /*! + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor + @param[in] strict whether to expect the input to be consumed completed + + @return + */ + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true) + { + sax = sax_; + bool result = false; + + switch (format) + { + case input_format_t::bson: + result = parse_bson_internal(); + break; + + case input_format_t::cbor: + result = parse_cbor_internal(); + break; + + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; + + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + + // strict mode: next byte must be EOF + if (result and strict) + { + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); + } + } + + return result; + } + + /*! + @brief determine system byte order + + @return true if and only if system's byte order is little endian + + @note from http://stackoverflow.com/a/1001328/266378 + */ + static constexpr bool little_endianess(int num = 1) noexcept + { + return *reinterpret_cast<char*>(&num) == 1; + } + + private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast<char>(current); + } + + return true; + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template<typename NumberType> + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); + } + + return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof(); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const int element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number; + return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0x02: // string + { + std::int32_t len; + string_t value; + return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value; + return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value; + return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + std::array<char, 3> cr{{}}; + (std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast<unsigned char>(element_type)); + return sax->parse_error(element_type_parse_position, std::string(cr.data()), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr.data()))); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + while (int element_type = get()) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_UNLIKELY(not get_bson_cstr(key))) + { + return false; + } + + if (not is_array and not sax->key(key)) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid CBOR value was passed to the SAX parser + */ + bool parse_cbor_internal(const bool get_char = true) + { + switch (get_char ? get() : current) + { + // EOF + case std::char_traits<char>::eof(): + return unexpect_eof(input_format_t::cbor, "value"); + + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + std::uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + std::uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + std::uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + std::uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + return sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - current)); + + case 0x38: // Negative integer (one-byte uint8_t follows) + { + std::uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); + } + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + { + std::uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); + } + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + { + std::uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); + } + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + { + std::uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) + - static_cast<number_integer_t>(number)); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + return get_cbor_string(s) and sax->string(s); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + return get_cbor_array(static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu)); + + case 0x98: // array (one-byte uint8_t for n follows) + { + std::uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x99: // array (two-byte uint16_t for n follow) + { + std::uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x9A: // array (four-byte uint32_t for n follow) + { + std::uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x9B: // array (eight-byte uint64_t for n follow) + { + std::uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x9F: // array (indefinite length) + return get_cbor_array(std::size_t(-1)); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return get_cbor_object(static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu)); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + std::uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + std::uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + std::uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + std::uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xBF: // map (indefinite length) + return get_cbor_object(std::size_t(-1)); + + case 0xF4: // false + return sax->boolean(false); + + case 0xF5: // true + return sax->boolean(true); + + case 0xF6: // null + return sax->null(); + + case 0xF9: // Half-Precision Float (two-byte IEEE 754) + { + const int byte1_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const int byte2_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast<unsigned char>(byte1_raw); + const auto byte2 = static_cast<unsigned char>(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast<unsigned int>((byte1 << 8u) + byte2); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + assert(0 <= exp and exp <= 32); + assert(0 <= mant and mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits<double>::infinity() + : std::numeric_limits<double>::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000u) != 0 + ? static_cast<number_float_t>(-val) + : static_cast<number_float_t>(val), ""); + } + + case 0xFA: // Single-Precision Float (four-byte IEEE 754) + { + float number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) + { + double number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, static_cast<unsigned int>(current) & 0x1Fu, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + std::uint8_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + std::uint16_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + std::uint32_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + std::uint64_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (not get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array or std::size_t(-1) for an + array of indefinite size + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not parse_cbor_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or std::size_t(-1) for an + object of indefinite size + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } + + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + switch (get()) + { + // EOF + case std::char_traits<char>::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + return get_msgpack_object(static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + return get_msgpack_array(static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } + + case 0xC0: // nil + return sax->null(); + + case 0xC2: // false + return sax->boolean(false); + + case 0xC3: // true + return sax->boolean(true); + + case 0xCA: // float 32 + { + float number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0xCB: // float 64 + { + double number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0xCC: // uint 8 + { + std::uint8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCD: // uint 16 + { + std::uint16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCE: // uint 32 + { + std::uint32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCF: // uint 64 + { + std::uint64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xD0: // int 8 + { + std::int8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD1: // int 16 + { + std::int16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD2: // int 32 + { + std::int32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD3: // int 64 + { + std::int64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } + + case 0xDC: // array 16 + { + std::uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); + } + + case 0xDD: // array 32 + { + std::uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); + } + + case 0xDE: // map 16 + { + std::uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); + } + + case 0xDF: // map 32 + { + std::uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); + } + + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + return sax->number_integer(static_cast<std::int8_t>(current)); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @brief reads a MessagePack string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_msgpack_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } + + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, static_cast<unsigned int>(current) & 0x1Fu, result); + } + + case 0xD9: // str 8 + { + std::uint8_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + case 0xDA: // str 16 + { + std::uint16_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + case 0xDB: // str 32 + { + std::uint32_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array + @return whether array creation completed + */ + bool get_msgpack_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } + + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + key.clear(); + } + + return sax->end_object(); + } + + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + + /*! + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether string creation completed + */ + bool get_ubjson_string(string_t& result, const bool get_char = true) + { + if (get_char) + { + get(); // TODO(niels): may we ignore N here? + } + + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + + switch (current) + { + case 'U': + { + std::uint8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'i': + { + std::int8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'I': + { + std::int16_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'l': + { + std::int32_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'L': + { + std::int64_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': + { + std::uint8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'i': + { + std::int8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'I': + { + std::int16_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'l': + { + std::int32_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'L': + { + std::int64_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); + } + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair<std::size_t, int>& result) + { + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') + { + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } + + get_ignore_noop(); + if (JSON_UNLIKELY(current != '#')) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); + } + + return get_ubjson_size_value(result.first); + } + + if (current == '#') + { + return get_ubjson_size_value(result.first); + } + + return true; + } + + /*! + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits<char>::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); + + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); + + case 'U': + { + std::uint8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); + } + + case 'i': + { + std::int8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'I': + { + std::int16_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'l': + { + std::int32_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'L': + { + std::int64_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'd': + { + float number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'D': + { + double number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'C': // char + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); + } + string_t s(1, static_cast<char>(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) and sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @return whether array creation completed + */ + bool get_ubjson_array() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + while (current != ']') + { + if (JSON_UNLIKELY(not parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); + } + } + + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); + } + } + + return sax->end_object(); + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits<char>::eof()` in that case. + + @return character read from the input + */ + int get() + { + ++chars_read; + return current = ia->get_character(); + } + + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianess, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. + */ + template<typename NumberType, bool InputIsLittleEndian = false> + bool get_number(const input_format_t format, NumberType& result) + { + // step 1: read input into array with system's byte order + std::array<std::uint8_t, sizeof(NumberType)> vec; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) + { + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian != InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast<std::uint8_t>(current); + } + else + { + vec[i] = static_cast<std::uint8_t>(current); // LCOV_EXCL_LINE + } + } + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template<typename NumberType> + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + std::generate_n(std::back_inserter(result), len, [this, &success, &format]() + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) + { + success = false; + } + return static_cast<char>(current); + }); + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, "<end of file>", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + std::array<char, 3> cr{{}}; + (std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast<unsigned char>(current)); + return std::string{cr.data()}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further contect information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + + return error_msg + " " + context + ": " + detail; + } + + private: + /// input adapter + input_adapter_t ia = nullptr; + + /// the current character + int current = std::char_traits<char>::eof(); + + /// the number of characters read + std::size_t chars_read = 0; + + /// whether we can assume little endianess + const bool is_little_endian = little_endianess(); + + /// the SAX parser + json_sax_t* sax = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/input_adapters.hpp> + // #include <nlohmann/detail/input/lexer.hpp> +#include <array> // array #include <clocale> // localeconv #include <cstddef> // size_t -#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull #include <cstdio> // snprintf +#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull #include <initializer_list> // initializer_list #include <string> // char_traits, string +#include <utility> // move #include <vector> // vector -// #include <nlohmann/detail/macro_scope.hpp> - // #include <nlohmann/detail/input/input_adapters.hpp> // #include <nlohmann/detail/input/position_t.hpp> +// #include <nlohmann/detail/macro_scope.hpp> + namespace nlohmann { @@ -2715,22 +5651,22 @@ class lexer assert(current == 'u'); int codepoint = 0; - const auto factors = { 12, 8, 4, 0 }; + const auto factors = { 12u, 8u, 4u, 0u }; for (const auto factor : factors) { get(); if (current >= '0' and current <= '9') { - codepoint += ((current - 0x30) << factor); + codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x30u) << factor); } else if (current >= 'A' and current <= 'F') { - codepoint += ((current - 0x37) << factor); + codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x37u) << factor); } else if (current >= 'a' and current <= 'f') { - codepoint += ((current - 0x57) << factor); + codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x57u) << factor); } else { @@ -2888,15 +5824,15 @@ class lexer if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF)) { // overwrite codepoint - codepoint = - // high surrogate occupies the most significant 22 bits - (codepoint1 << 10) - // low surrogate occupies the least significant 15 bits - + codepoint2 - // there is still the 0xD800, 0xDC00 and 0x10000 noise - // in the result so we have to subtract with: - // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 - - 0x35FDC00; + codepoint = static_cast<int>( + // high surrogate occupies the most significant 22 bits + (static_cast<unsigned int>(codepoint1) << 10u) + // low surrogate occupies the least significant 15 bits + + static_cast<unsigned int>(codepoint2) + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00u); } else { @@ -2931,23 +5867,23 @@ class lexer else if (codepoint <= 0x7FF) { // 2-byte characters: 110xxxxx 10xxxxxx - add(0xC0 | (codepoint >> 6)); - add(0x80 | (codepoint & 0x3F)); + add(static_cast<int>(0xC0u | (static_cast<unsigned int>(codepoint) >> 6u))); + add(static_cast<int>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu))); } else if (codepoint <= 0xFFFF) { // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - add(0xE0 | (codepoint >> 12)); - add(0x80 | ((codepoint >> 6) & 0x3F)); - add(0x80 | (codepoint & 0x3F)); + add(static_cast<int>(0xE0u | (static_cast<unsigned int>(codepoint) >> 12u))); + add(static_cast<int>(0x80u | ((static_cast<unsigned int>(codepoint) >> 6u) & 0x3Fu))); + add(static_cast<int>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu))); } else { // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - add(0xF0 | (codepoint >> 18)); - add(0x80 | ((codepoint >> 12) & 0x3F)); - add(0x80 | ((codepoint >> 6) & 0x3F)); - add(0x80 | (codepoint & 0x3F)); + add(static_cast<int>(0xF0u | (static_cast<unsigned int>(codepoint) >> 18u))); + add(static_cast<int>(0x80u | ((static_cast<unsigned int>(codepoint) >> 12u) & 0x3Fu))); + add(static_cast<int>(0x80u | ((static_cast<unsigned int>(codepoint) >> 6u) & 0x3Fu))); + add(static_cast<int>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu))); } break; @@ -3473,13 +6409,9 @@ class lexer goto scan_number_any1; } - // LCOV_EXCL_START - default: - { - // all other characters are rejected outside scan_number() - assert(false); - } - // LCOV_EXCL_STOP + // all other characters are rejected outside scan_number() + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } scan_number_minus: @@ -3829,7 +6761,7 @@ scan_number_done: if (current == '\n') { ++position.lines_read; - ++position.chars_read_current_line = 0; + position.chars_read_current_line = 0; } return current; @@ -3864,7 +6796,7 @@ scan_number_done: if (JSON_LIKELY(current != std::char_traits<char>::eof())) { - assert(token_string.size() != 0); + assert(not token_string.empty()); token_string.pop_back(); } } @@ -3926,9 +6858,9 @@ scan_number_done: if ('\x00' <= c and c <= '\x1F') { // escape control characters - char cs[9]; - (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c)); - result += cs; + std::array<char, 9> cs{{}}; + (std::snprintf)(cs.data(), cs.size(), "<U+%.4X>", static_cast<unsigned char>(c)); + result += cs.data(); } else { @@ -4050,7 +6982,7 @@ scan_number_done: bool next_unget = false; /// the start position of the current token - position_t position; + position_t position {}; /// raw input token string (for error messages) std::vector<char> token_string {}; @@ -4081,864 +7013,19 @@ scan_number_done: #include <functional> // function #include <string> // string #include <utility> // move +#include <vector> // vector // #include <nlohmann/detail/exceptions.hpp> -// #include <nlohmann/detail/macro_scope.hpp> - -// #include <nlohmann/detail/meta/is_sax.hpp> - - -#include <cstdint> // size_t -#include <utility> // declval - -// #include <nlohmann/detail/meta/detected.hpp> - -// #include <nlohmann/detail/meta/type_traits.hpp> - - -namespace nlohmann -{ -namespace detail -{ -template <typename T> -using null_function_t = decltype(std::declval<T&>().null()); - -template <typename T> -using boolean_function_t = - decltype(std::declval<T&>().boolean(std::declval<bool>())); - -template <typename T, typename Integer> -using number_integer_function_t = - decltype(std::declval<T&>().number_integer(std::declval<Integer>())); - -template <typename T, typename Unsigned> -using number_unsigned_function_t = - decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>())); - -template <typename T, typename Float, typename String> -using number_float_function_t = decltype(std::declval<T&>().number_float( - std::declval<Float>(), std::declval<const String&>())); - -template <typename T, typename String> -using string_function_t = - decltype(std::declval<T&>().string(std::declval<String&>())); - -template <typename T> -using start_object_function_t = - decltype(std::declval<T&>().start_object(std::declval<std::size_t>())); - -template <typename T, typename String> -using key_function_t = - decltype(std::declval<T&>().key(std::declval<String&>())); - -template <typename T> -using end_object_function_t = decltype(std::declval<T&>().end_object()); - -template <typename T> -using start_array_function_t = - decltype(std::declval<T&>().start_array(std::declval<std::size_t>())); - -template <typename T> -using end_array_function_t = decltype(std::declval<T&>().end_array()); - -template <typename T, typename Exception> -using parse_error_function_t = decltype(std::declval<T&>().parse_error( - std::declval<std::size_t>(), std::declval<const std::string&>(), - std::declval<const Exception&>())); - -template <typename SAX, typename BasicJsonType> -struct is_sax -{ - private: - static_assert(is_basic_json<BasicJsonType>::value, - "BasicJsonType must be of type basic_json<...>"); - - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using exception_t = typename BasicJsonType::exception; - - public: - static constexpr bool value = - is_detected_exact<bool, null_function_t, SAX>::value && - is_detected_exact<bool, boolean_function_t, SAX>::value && - is_detected_exact<bool, number_integer_function_t, SAX, - number_integer_t>::value && - is_detected_exact<bool, number_unsigned_function_t, SAX, - number_unsigned_t>::value && - is_detected_exact<bool, number_float_function_t, SAX, number_float_t, - string_t>::value && - is_detected_exact<bool, string_function_t, SAX, string_t>::value && - is_detected_exact<bool, start_object_function_t, SAX>::value && - is_detected_exact<bool, key_function_t, SAX, string_t>::value && - is_detected_exact<bool, end_object_function_t, SAX>::value && - is_detected_exact<bool, start_array_function_t, SAX>::value && - is_detected_exact<bool, end_array_function_t, SAX>::value && - is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value; -}; - -template <typename SAX, typename BasicJsonType> -struct is_sax_static_asserts -{ - private: - static_assert(is_basic_json<BasicJsonType>::value, - "BasicJsonType must be of type basic_json<...>"); - - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using exception_t = typename BasicJsonType::exception; - - public: - static_assert(is_detected_exact<bool, null_function_t, SAX>::value, - "Missing/invalid function: bool null()"); - static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, - "Missing/invalid function: bool boolean(bool)"); - static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, - "Missing/invalid function: bool boolean(bool)"); - static_assert( - is_detected_exact<bool, number_integer_function_t, SAX, - number_integer_t>::value, - "Missing/invalid function: bool number_integer(number_integer_t)"); - static_assert( - is_detected_exact<bool, number_unsigned_function_t, SAX, - number_unsigned_t>::value, - "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); - static_assert(is_detected_exact<bool, number_float_function_t, SAX, - number_float_t, string_t>::value, - "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); - static_assert( - is_detected_exact<bool, string_function_t, SAX, string_t>::value, - "Missing/invalid function: bool string(string_t&)"); - static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value, - "Missing/invalid function: bool start_object(std::size_t)"); - static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value, - "Missing/invalid function: bool key(string_t&)"); - static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value, - "Missing/invalid function: bool end_object()"); - static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value, - "Missing/invalid function: bool start_array(std::size_t)"); - static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value, - "Missing/invalid function: bool end_array()"); - static_assert( - is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value, - "Missing/invalid function: bool parse_error(std::size_t, const " - "std::string&, const exception&)"); -}; -} // namespace detail -} // namespace nlohmann - // #include <nlohmann/detail/input/input_adapters.hpp> // #include <nlohmann/detail/input/json_sax.hpp> +// #include <nlohmann/detail/input/lexer.hpp> -#include <cstddef> -#include <string> -#include <vector> - -// #include <nlohmann/detail/input/parser.hpp> - -// #include <nlohmann/detail/exceptions.hpp> - - -namespace nlohmann -{ - -/*! -@brief SAX interface - -This class describes the SAX interface used by @ref nlohmann::json::sax_parse. -Each function is called in different situations while the input is parsed. The -boolean return value informs the parser whether to continue processing the -input. -*/ -template<typename BasicJsonType> -struct json_sax -{ - /// type for (signed) integers - using number_integer_t = typename BasicJsonType::number_integer_t; - /// type for unsigned integers - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - /// type for floating-point numbers - using number_float_t = typename BasicJsonType::number_float_t; - /// type for strings - using string_t = typename BasicJsonType::string_t; - - /*! - @brief a null value was read - @return whether parsing should proceed - */ - virtual bool null() = 0; - - /*! - @brief a boolean value was read - @param[in] val boolean value - @return whether parsing should proceed - */ - virtual bool boolean(bool val) = 0; - - /*! - @brief an integer number was read - @param[in] val integer value - @return whether parsing should proceed - */ - virtual bool number_integer(number_integer_t val) = 0; - - /*! - @brief an unsigned integer number was read - @param[in] val unsigned integer value - @return whether parsing should proceed - */ - virtual bool number_unsigned(number_unsigned_t val) = 0; - - /*! - @brief an floating-point number was read - @param[in] val floating-point value - @param[in] s raw token value - @return whether parsing should proceed - */ - virtual bool number_float(number_float_t val, const string_t& s) = 0; - - /*! - @brief a string was read - @param[in] val string value - @return whether parsing should proceed - @note It is safe to move the passed string. - */ - virtual bool string(string_t& val) = 0; - - /*! - @brief the beginning of an object was read - @param[in] elements number of object elements or -1 if unknown - @return whether parsing should proceed - @note binary formats may report the number of elements - */ - virtual bool start_object(std::size_t elements) = 0; - - /*! - @brief an object key was read - @param[in] val object key - @return whether parsing should proceed - @note It is safe to move the passed string. - */ - virtual bool key(string_t& val) = 0; - - /*! - @brief the end of an object was read - @return whether parsing should proceed - */ - virtual bool end_object() = 0; - - /*! - @brief the beginning of an array was read - @param[in] elements number of array elements or -1 if unknown - @return whether parsing should proceed - @note binary formats may report the number of elements - */ - virtual bool start_array(std::size_t elements) = 0; - - /*! - @brief the end of an array was read - @return whether parsing should proceed - */ - virtual bool end_array() = 0; - - /*! - @brief a parse error occurred - @param[in] position the position in the input where the error occurs - @param[in] last_token the last read token - @param[in] ex an exception object describing the error - @return whether parsing should proceed (must return false) - */ - virtual bool parse_error(std::size_t position, - const std::string& last_token, - const detail::exception& ex) = 0; - - virtual ~json_sax() = default; -}; - - -namespace detail -{ -/*! -@brief SAX implementation to create a JSON value from SAX events - -This class implements the @ref json_sax interface and processes the SAX events -to create a JSON value which makes it basically a DOM parser. The structure or -hierarchy of the JSON value is managed by the stack `ref_stack` which contains -a pointer to the respective array or object for each recursion depth. - -After successful parsing, the value that is passed by reference to the -constructor contains the parsed value. - -@tparam BasicJsonType the JSON type -*/ -template<typename BasicJsonType> -class json_sax_dom_parser -{ - public: - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - - /*! - @param[in, out] r reference to a JSON value that is manipulated while - parsing - @param[in] allow_exceptions_ whether parse errors yield exceptions - */ - explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) - : root(r), allow_exceptions(allow_exceptions_) - {} - - bool null() - { - handle_value(nullptr); - return true; - } - - bool boolean(bool val) - { - handle_value(val); - return true; - } - - bool number_integer(number_integer_t val) - { - handle_value(val); - return true; - } - - bool number_unsigned(number_unsigned_t val) - { - handle_value(val); - return true; - } - - bool number_float(number_float_t val, const string_t& /*unused*/) - { - handle_value(val); - return true; - } - - bool string(string_t& val) - { - handle_value(val); - return true; - } - - bool start_object(std::size_t len) - { - ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); - - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive object size: " + std::to_string(len))); - } - - return true; - } - - bool key(string_t& val) - { - // add null at given key and store the reference for later - object_element = &(ref_stack.back()->m_value.object->operator[](val)); - return true; - } - - bool end_object() - { - ref_stack.pop_back(); - return true; - } - - bool start_array(std::size_t len) - { - ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); - - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive array size: " + std::to_string(len))); - } - - return true; - } - - bool end_array() - { - ref_stack.pop_back(); - return true; - } - - bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, - const detail::exception& ex) - { - errored = true; - if (allow_exceptions) - { - // determine the proper exception type from the id - switch ((ex.id / 100) % 100) - { - case 1: - JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); - case 4: - JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); - // LCOV_EXCL_START - case 2: - JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); - case 3: - JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); - case 5: - JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); - default: - assert(false); - // LCOV_EXCL_STOP - } - } - return false; - } - - constexpr bool is_errored() const - { - return errored; - } - - private: - /*! - @invariant If the ref stack is empty, then the passed value will be the new - root. - @invariant If the ref stack contains a value, then it is an array or an - object to which we can add elements - */ - template<typename Value> - BasicJsonType* handle_value(Value&& v) - { - if (ref_stack.empty()) - { - root = BasicJsonType(std::forward<Value>(v)); - return &root; - } - - assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); - - if (ref_stack.back()->is_array()) - { - ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v)); - return &(ref_stack.back()->m_value.array->back()); - } - else - { - assert(object_element); - *object_element = BasicJsonType(std::forward<Value>(v)); - return object_element; - } - } - - /// the parsed JSON value - BasicJsonType& root; - /// stack to model hierarchy of values - std::vector<BasicJsonType*> ref_stack; - /// helper to hold the reference for the next object element - BasicJsonType* object_element = nullptr; - /// whether a syntax error occurred - bool errored = false; - /// whether to throw exceptions in case of errors - const bool allow_exceptions = true; -}; - -template<typename BasicJsonType> -class json_sax_dom_callback_parser -{ - public: - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using parser_callback_t = typename BasicJsonType::parser_callback_t; - using parse_event_t = typename BasicJsonType::parse_event_t; - - json_sax_dom_callback_parser(BasicJsonType& r, - const parser_callback_t cb, - const bool allow_exceptions_ = true) - : root(r), callback(cb), allow_exceptions(allow_exceptions_) - { - keep_stack.push_back(true); - } - - bool null() - { - handle_value(nullptr); - return true; - } - - bool boolean(bool val) - { - handle_value(val); - return true; - } - - bool number_integer(number_integer_t val) - { - handle_value(val); - return true; - } - - bool number_unsigned(number_unsigned_t val) - { - handle_value(val); - return true; - } - - bool number_float(number_float_t val, const string_t& /*unused*/) - { - handle_value(val); - return true; - } - - bool string(string_t& val) - { - handle_value(val); - return true; - } - - bool start_object(std::size_t len) - { - // check callback for object start - const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded); - keep_stack.push_back(keep); - - auto val = handle_value(BasicJsonType::value_t::object, true); - ref_stack.push_back(val.second); - - // check object limit - if (ref_stack.back()) - { - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive object size: " + std::to_string(len))); - } - } - - return true; - } - - bool key(string_t& val) - { - BasicJsonType k = BasicJsonType(val); - - // check callback for key - const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k); - key_keep_stack.push_back(keep); - - // add discarded value at given key and store the reference for later - if (keep and ref_stack.back()) - { - object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); - } - - return true; - } - - bool end_object() - { - if (ref_stack.back()) - { - if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) - { - // discard object - *ref_stack.back() = discarded; - } - } - - assert(not ref_stack.empty()); - assert(not keep_stack.empty()); - ref_stack.pop_back(); - keep_stack.pop_back(); - - if (not ref_stack.empty() and ref_stack.back()) - { - // remove discarded value - if (ref_stack.back()->is_object()) - { - for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) - { - if (it->is_discarded()) - { - ref_stack.back()->erase(it); - break; - } - } - } - } - - return true; - } - - bool start_array(std::size_t len) - { - const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded); - keep_stack.push_back(keep); - - auto val = handle_value(BasicJsonType::value_t::array, true); - ref_stack.push_back(val.second); - - // check array limit - if (ref_stack.back()) - { - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive array size: " + std::to_string(len))); - } - } - - return true; - } - - bool end_array() - { - bool keep = true; - - if (ref_stack.back()) - { - keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); - if (not keep) - { - // discard array - *ref_stack.back() = discarded; - } - } - - assert(not ref_stack.empty()); - assert(not keep_stack.empty()); - ref_stack.pop_back(); - keep_stack.pop_back(); - - // remove discarded value - if (not keep and not ref_stack.empty()) - { - if (ref_stack.back()->is_array()) - { - ref_stack.back()->m_value.array->pop_back(); - } - } - - return true; - } - - bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, - const detail::exception& ex) - { - errored = true; - if (allow_exceptions) - { - // determine the proper exception type from the id - switch ((ex.id / 100) % 100) - { - case 1: - JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); - case 4: - JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); - // LCOV_EXCL_START - case 2: - JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); - case 3: - JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); - case 5: - JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); - default: - assert(false); - // LCOV_EXCL_STOP - } - } - return false; - } - - constexpr bool is_errored() const - { - return errored; - } - - private: - /*! - @param[in] v value to add to the JSON value we build during parsing - @param[in] skip_callback whether we should skip calling the callback - function; this is required after start_array() and - start_object() SAX events, because otherwise we would call the - callback function with an empty array or object, respectively. - - @invariant If the ref stack is empty, then the passed value will be the new - root. - @invariant If the ref stack contains a value, then it is an array or an - object to which we can add elements - - @return pair of boolean (whether value should be kept) and pointer (to the - passed value in the ref_stack hierarchy; nullptr if not kept) - */ - template<typename Value> - std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false) - { - assert(not keep_stack.empty()); - - // do not handle this value if we know it would be added to a discarded - // container - if (not keep_stack.back()) - { - return {false, nullptr}; - } - - // create value - auto value = BasicJsonType(std::forward<Value>(v)); - - // check callback - const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value); - - // do not handle this value if we just learnt it shall be discarded - if (not keep) - { - return {false, nullptr}; - } - - if (ref_stack.empty()) - { - root = std::move(value); - return {true, &root}; - } - - // skip this value if we already decided to skip the parent - // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) - if (not ref_stack.back()) - { - return {false, nullptr}; - } - - // we now only expect arrays and objects - assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); - - if (ref_stack.back()->is_array()) - { - ref_stack.back()->m_value.array->push_back(std::move(value)); - return {true, &(ref_stack.back()->m_value.array->back())}; - } - else - { - // check if we should store an element for the current key - assert(not key_keep_stack.empty()); - const bool store_element = key_keep_stack.back(); - key_keep_stack.pop_back(); - - if (not store_element) - { - return {false, nullptr}; - } - - assert(object_element); - *object_element = std::move(value); - return {true, object_element}; - } - } - - /// the parsed JSON value - BasicJsonType& root; - /// stack to model hierarchy of values - std::vector<BasicJsonType*> ref_stack; - /// stack to manage which values to keep - std::vector<bool> keep_stack; - /// stack to manage which object keys to keep - std::vector<bool> key_keep_stack; - /// helper to hold the reference for the next object element - BasicJsonType* object_element = nullptr; - /// whether a syntax error occurred - bool errored = false; - /// callback function - const parser_callback_t callback = nullptr; - /// whether to throw exceptions in case of errors - const bool allow_exceptions = true; - /// a discarded value for the callback - BasicJsonType discarded = BasicJsonType::value_t::discarded; -}; - -template<typename BasicJsonType> -class json_sax_acceptor -{ - public: - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - - bool null() - { - return true; - } - - bool boolean(bool /*unused*/) - { - return true; - } - - bool number_integer(number_integer_t /*unused*/) - { - return true; - } - - bool number_unsigned(number_unsigned_t /*unused*/) - { - return true; - } - - bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) - { - return true; - } - - bool string(string_t& /*unused*/) - { - return true; - } - - bool start_object(std::size_t /*unused*/ = std::size_t(-1)) - { - return true; - } - - bool key(string_t& /*unused*/) - { - return true; - } - - bool end_object() - { - return true; - } - - bool start_array(std::size_t /*unused*/ = std::size_t(-1)) - { - return true; - } - - bool end_array() - { - return true; - } - - bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) - { - return false; - } -}; -} // namespace detail - -} // namespace nlohmann +// #include <nlohmann/detail/macro_scope.hpp> -// #include <nlohmann/detail/input/lexer.hpp> +// #include <nlohmann/detail/meta/is_sax.hpp> // #include <nlohmann/detail/value_t.hpp> @@ -5189,14 +7276,13 @@ class parser m_lexer.get_token_string(), out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'")); } - else + + if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) { - if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) - { - return false; - } - break; + return false; } + + break; } case token_type::literal_false: @@ -5282,103 +7368,95 @@ class parser // empty stack: we reached the end of the hierarchy: done return true; } - else + + if (states.back()) // array { - if (states.back()) // array + // comma -> next value + if (get_token() == token_type::value_separator) { - // comma -> next value - if (get_token() == token_type::value_separator) + // parse a new value + get_token(); + continue; + } + + // closing ] + if (JSON_LIKELY(last_token == token_type::end_array)) + { + if (JSON_UNLIKELY(not sax->end_array())) { - // parse a new value - get_token(); - continue; + return false; } - // closing ] - if (JSON_LIKELY(last_token == token_type::end_array)) - { - if (JSON_UNLIKELY(not sax->end_array())) - { - return false; - } + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } - // We are done with this array. Before we can parse a - // new value, we need to evaluate the new state first. - // By setting skip_to_state_evaluation to false, we - // are effectively jumping to the beginning of this if. - assert(not states.empty()); - states.pop_back(); - skip_to_state_evaluation = true; - continue; - } - else + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_array, "array"))); + } + else // object + { + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse key + if (JSON_UNLIKELY(get_token() != token_type::value_string)) { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::end_array, "array"))); - } - } - else // object - { - // comma -> next value - if (get_token() == token_type::value_separator) - { - // parse key - if (JSON_UNLIKELY(get_token() != token_type::value_string)) - { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::value_string, "object key"))); - } - else - { - if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) - { - return false; - } - } - - // parse separator (:) - if (JSON_UNLIKELY(get_token() != token_type::name_separator)) - { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::name_separator, "object separator"))); - } - - // parse values - get_token(); - continue; + exception_message(token_type::value_string, "object key"))); } - // closing } - if (JSON_LIKELY(last_token == token_type::end_object)) + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) { - if (JSON_UNLIKELY(not sax->end_object())) - { - return false; - } - - // We are done with this object. Before we can parse a - // new value, we need to evaluate the new state first. - // By setting skip_to_state_evaluation to false, we - // are effectively jumping to the beginning of this if. - assert(not states.empty()); - states.pop_back(); - skip_to_state_evaluation = true; - continue; + return false; } - else + + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::end_object, "object"))); + exception_message(token_type::name_separator, "object separator"))); } + + // parse values + get_token(); + continue; + } + + // closing } + if (JSON_LIKELY(last_token == token_type::end_object)) + { + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } + + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_object, "object"))); } } } @@ -5386,7 +7464,7 @@ class parser /// get next token from lexer token_type get_token() { - return (last_token = m_lexer.scan()); + return last_token = m_lexer.scan(); } std::string exception_message(const token_type expected, const std::string& context) @@ -5431,6 +7509,9 @@ class parser } // namespace detail } // namespace nlohmann +// #include <nlohmann/detail/iterators/internal_iterator.hpp> + + // #include <nlohmann/detail/iterators/primitive_iterator.hpp> @@ -5553,11 +7634,6 @@ class primitive_iterator_t } // namespace detail } // namespace nlohmann -// #include <nlohmann/detail/iterators/internal_iterator.hpp> - - -// #include <nlohmann/detail/iterators/primitive_iterator.hpp> - namespace nlohmann { @@ -5598,6 +7674,8 @@ template<typename BasicJsonType> struct internal_iterator // #include <nlohmann/detail/meta/cpp_future.hpp> +// #include <nlohmann/detail/meta/type_traits.hpp> + // #include <nlohmann/detail/value_t.hpp> @@ -6196,10 +8274,11 @@ class iter_impl /// associated JSON instance pointer m_object = nullptr; /// the actual iterator of the associated instance - internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it; + internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it {}; }; } // namespace detail } // namespace nlohmann + // #include <nlohmann/detail/iterators/iteration_proxy.hpp> // #include <nlohmann/detail/iterators/json_reverse_iterator.hpp> @@ -6323,2132 +8402,1163 @@ class json_reverse_iterator : public std::reverse_iterator<Base> } // namespace detail } // namespace nlohmann -// #include <nlohmann/detail/output/output_adapters.hpp> +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> +// #include <nlohmann/detail/json_pointer.hpp> -#include <algorithm> // copy -#include <cstddef> // size_t -#include <ios> // streamsize -#include <iterator> // back_inserter -#include <memory> // shared_ptr, make_shared -#include <ostream> // basic_ostream -#include <string> // basic_string + +#include <algorithm> // all_of +#include <cassert> // assert +#include <numeric> // accumulate +#include <string> // string +#include <utility> // move #include <vector> // vector +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + namespace nlohmann { -namespace detail -{ -/// abstract output adapter interface -template<typename CharType> struct output_adapter_protocol +template<typename BasicJsonType> +class json_pointer { - virtual void write_character(CharType c) = 0; - virtual void write_characters(const CharType* s, std::size_t length) = 0; - virtual ~output_adapter_protocol() = default; -}; - -/// a type to simplify interfaces -template<typename CharType> -using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>; + // allow basic_json to access private members + NLOHMANN_BASIC_JSON_TPL_DECLARATION + friend class basic_json; -/// output adapter for byte vectors -template<typename CharType> -class output_vector_adapter : public output_adapter_protocol<CharType> -{ public: - explicit output_vector_adapter(std::vector<CharType>& vec) noexcept - : v(vec) - {} + /*! + @brief create JSON pointer - void write_character(CharType c) override - { - v.push_back(c); - } + Create a JSON pointer according to the syntax described in + [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). - void write_characters(const CharType* s, std::size_t length) override - { - std::copy(s, s + length, std::back_inserter(v)); - } + @param[in] s string representing the JSON pointer; if omitted, the empty + string is assumed which references the whole JSON value - private: - std::vector<CharType>& v; -}; + @throw parse_error.107 if the given JSON pointer @a s is nonempty and does + not begin with a slash (`/`); see example below -/// output adapter for output streams -template<typename CharType> -class output_stream_adapter : public output_adapter_protocol<CharType> -{ - public: - explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept - : stream(s) + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is + not followed by `0` (representing `~`) or `1` (representing `/`); see + example below + + @liveexample{The example shows the construction several valid JSON pointers + as well as the exceptional behavior.,json_pointer} + + @since version 2.0.0 + */ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) {} - void write_character(CharType c) override - { - stream.put(c); - } + /*! + @brief return a string representation of the JSON pointer - void write_characters(const CharType* s, std::size_t length) override - { - stream.write(s, static_cast<std::streamsize>(length)); - } + @invariant For each JSON pointer `ptr`, it holds: + @code {.cpp} + ptr == json_pointer(ptr.to_string()); + @endcode - private: - std::basic_ostream<CharType>& stream; -}; + @return a string representation of the JSON pointer -/// output adapter for basic_string -template<typename CharType, typename StringType = std::basic_string<CharType>> -class output_string_adapter : public output_adapter_protocol<CharType> -{ - public: - explicit output_string_adapter(StringType& s) noexcept - : str(s) - {} + @liveexample{The example shows the result of `to_string`.,json_pointer__to_string} - void write_character(CharType c) override + @since version 2.0.0 + */ + std::string to_string() const { - str.push_back(c); + return std::accumulate(reference_tokens.begin(), reference_tokens.end(), + std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + escape(b); + }); } - void write_characters(const CharType* s, std::size_t length) override + /// @copydoc to_string() + operator std::string() const { - str.append(s, length); + return to_string(); } - private: - StringType& str; -}; + /*! + @brief append another JSON pointer at the end of this JSON pointer -template<typename CharType, typename StringType = std::basic_string<CharType>> -class output_adapter -{ - public: - output_adapter(std::vector<CharType>& vec) - : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {} + @param[in] ptr JSON pointer to append + @return JSON pointer with @a ptr appended - output_adapter(std::basic_ostream<CharType>& s) - : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {} + @liveexample{The example shows the usage of `operator/=`.,json_pointer__operator_add} - output_adapter(StringType& s) - : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {} + @complexity Linear in the length of @a ptr. - operator output_adapter_t<CharType>() + @sa @ref operator/=(std::string) to append a reference token + @sa @ref operator/=(std::size_t) to append an array index + @sa @ref operator/(const json_pointer&, const json_pointer&) for a binary operator + + @since version 3.6.0 + */ + json_pointer& operator/=(const json_pointer& ptr) { - return oa; + reference_tokens.insert(reference_tokens.end(), + ptr.reference_tokens.begin(), + ptr.reference_tokens.end()); + return *this; } - private: - output_adapter_t<CharType> oa = nullptr; -}; -} // namespace detail -} // namespace nlohmann - -// #include <nlohmann/detail/input/binary_reader.hpp> - - -#include <algorithm> // generate_n -#include <array> // array -#include <cassert> // assert -#include <cmath> // ldexp -#include <cstddef> // size_t -#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t -#include <cstdio> // snprintf -#include <cstring> // memcpy -#include <iterator> // back_inserter -#include <limits> // numeric_limits -#include <string> // char_traits, string -#include <utility> // make_pair, move + /*! + @brief append an unescaped reference token at the end of this JSON pointer -// #include <nlohmann/detail/input/input_adapters.hpp> + @param[in] token reference token to append + @return JSON pointer with @a token appended without escaping @a token -// #include <nlohmann/detail/input/json_sax.hpp> + @liveexample{The example shows the usage of `operator/=`.,json_pointer__operator_add} -// #include <nlohmann/detail/exceptions.hpp> + @complexity Amortized constant. -// #include <nlohmann/detail/macro_scope.hpp> + @sa @ref operator/=(const json_pointer&) to append a JSON pointer + @sa @ref operator/=(std::size_t) to append an array index + @sa @ref operator/(const json_pointer&, std::size_t) for a binary operator -// #include <nlohmann/detail/meta/is_sax.hpp> + @since version 3.6.0 + */ + json_pointer& operator/=(std::string token) + { + push_back(std::move(token)); + return *this; + } -// #include <nlohmann/detail/value_t.hpp> + /*! + @brief append an array index at the end of this JSON pointer + @param[in] array_index array index ot append + @return JSON pointer with @a array_index appended -namespace nlohmann -{ -namespace detail -{ -/////////////////// -// binary reader // -/////////////////// + @liveexample{The example shows the usage of `operator/=`.,json_pointer__operator_add} -/*! -@brief deserialization of CBOR, MessagePack, and UBJSON values -*/ -template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>> -class binary_reader -{ - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using json_sax_t = SAX; + @complexity Amortized constant. - public: - /*! - @brief create a binary reader + @sa @ref operator/=(const json_pointer&) to append a JSON pointer + @sa @ref operator/=(std::string) to append a reference token + @sa @ref operator/(const json_pointer&, std::string) for a binary operator - @param[in] adapter input adapter to read from + @since version 3.6.0 */ - explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) + json_pointer& operator/=(std::size_t array_index) { - (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; - assert(ia); + return *this /= std::to_string(array_index); } /*! - @param[in] format the binary format to parse - @param[in] sax_ a SAX event processor - @param[in] strict whether to expect the input to be consumed completed + @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer - @return - */ - bool sax_parse(const input_format_t format, - json_sax_t* sax_, - const bool strict = true) - { - sax = sax_; - bool result = false; + @param[in] lhs JSON pointer + @param[in] rhs JSON pointer + @return a new JSON pointer with @a rhs appended to @a lhs - switch (format) - { - case input_format_t::bson: - result = parse_bson_internal(); - break; + @liveexample{The example shows the usage of `operator/`.,json_pointer__operator_add_binary} - case input_format_t::cbor: - result = parse_cbor_internal(); - break; + @complexity Linear in the length of @a lhs and @a rhs. - case input_format_t::msgpack: - result = parse_msgpack_internal(); - break; + @sa @ref operator/=(const json_pointer&) to append a JSON pointer - case input_format_t::ubjson: - result = parse_ubjson_internal(); - break; - - // LCOV_EXCL_START - default: - assert(false); - // LCOV_EXCL_STOP - } + @since version 3.6.0 + */ + friend json_pointer operator/(const json_pointer& lhs, + const json_pointer& rhs) + { + return json_pointer(lhs) /= rhs; + } - // strict mode: next byte must be EOF - if (result and strict) - { - if (format == input_format_t::ubjson) - { - get_ignore_noop(); - } - else - { - get(); - } + /*! + @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer - if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) - { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); - } - } + @param[in] ptr JSON pointer + @param[in] token reference token + @return a new JSON pointer with unescaped @a token appended to @a ptr - return result; - } + @liveexample{The example shows the usage of `operator/`.,json_pointer__operator_add_binary} - /*! - @brief determine system byte order + @complexity Linear in the length of @a ptr. - @return true if and only if system's byte order is little endian + @sa @ref operator/=(std::string) to append a reference token - @note from http://stackoverflow.com/a/1001328/266378 + @since version 3.6.0 */ - static constexpr bool little_endianess(int num = 1) noexcept + friend json_pointer operator/(const json_pointer& ptr, std::string token) { - return (*reinterpret_cast<char*>(&num) == 1); + return json_pointer(ptr) /= std::move(token); } - private: - ////////// - // BSON // - ////////// - /*! - @brief Reads in a BSON-object and passes it to the SAX-parser. - @return whether a valid BSON-value was passed to the SAX parser - */ - bool parse_bson_internal() - { - std::int32_t document_size; - get_number<std::int32_t, true>(input_format_t::bson, document_size); + @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer - if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) - { - return false; - } + @param[in] ptr JSON pointer + @param[in] array_index array index + @return a new JSON pointer with @a array_index appended to @a ptr - if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) - { - return false; - } + @liveexample{The example shows the usage of `operator/`.,json_pointer__operator_add_binary} - return sax->end_object(); - } + @complexity Linear in the length of @a ptr. - /*! - @brief Parses a C-style string from the BSON input. - @param[in, out] result A reference to the string variable where the read - string is to be stored. - @return `true` if the \x00-byte indicating the end of the string was - encountered before the EOF; false` indicates an unexpected EOF. + @sa @ref operator/=(std::size_t) to append an array index + + @since version 3.6.0 */ - bool get_bson_cstr(string_t& result) + friend json_pointer operator/(const json_pointer& ptr, std::size_t array_index) { - auto out = std::back_inserter(result); - while (true) - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) - { - return false; - } - if (current == 0x00) - { - return true; - } - *out++ = static_cast<char>(current); - } - - return true; + return json_pointer(ptr) /= array_index; } /*! - @brief Parses a zero-terminated string of length @a len from the BSON - input. - @param[in] len The length (including the zero-byte at the end) of the - string to be read. - @param[in, out] result A reference to the string variable where the read - string is to be stored. - @tparam NumberType The type of the length @a len - @pre len >= 1 - @return `true` if the string was successfully parsed + @brief returns the parent of this JSON pointer + + @return parent of this JSON pointer; in case this JSON pointer is the root, + the root itself is returned + + @complexity Linear in the length of the JSON pointer. + + @liveexample{The example shows the result of `parent_pointer` for different + JSON Pointers.,json_pointer__parent_pointer} + + @since version 3.6.0 */ - template<typename NumberType> - bool get_bson_string(const NumberType len, string_t& result) + json_pointer parent_pointer() const { - if (JSON_UNLIKELY(len < 1)) + if (empty()) { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); + return *this; } - return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof(); + json_pointer res = *this; + res.pop_back(); + return res; } /*! - @brief Read a BSON document element of the given @a element_type. - @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html - @param[in] element_type_parse_position The position in the input stream, - where the `element_type` was read. - @warning Not all BSON element types are supported yet. An unsupported - @a element_type will give rise to a parse_error.114: - Unsupported BSON record type 0x... - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_internal(const int element_type, - const std::size_t element_type_parse_position) - { - switch (element_type) - { - case 0x01: // double - { - double number; - return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), ""); - } - - case 0x02: // string - { - std::int32_t len; - string_t value; - return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); - } - - case 0x03: // object - { - return parse_bson_internal(); - } - - case 0x04: // array - { - return parse_bson_array(); - } - - case 0x08: // boolean - { - return sax->boolean(get() != 0); - } + @brief remove last reference token - case 0x0A: // null - { - return sax->null(); - } + @pre not `empty()` - case 0x10: // int32 - { - std::int32_t value; - return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value); - } - - case 0x12: // int64 - { - std::int64_t value; - return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value); - } - - default: // anything else not supported (yet) - { - char cr[3]; - (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type)); - return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr))); - } - } - } + @liveexample{The example shows the usage of `pop_back`.,json_pointer__pop_back} - /*! - @brief Read a BSON element list (as specified in the BSON-spec) + @complexity Constant. - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). + @throw out_of_range.405 if JSON pointer has no parent - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser + @since version 3.6.0 */ - bool parse_bson_element_list(const bool is_array) + void pop_back() { - string_t key; - while (int element_type = get()) + if (JSON_UNLIKELY(empty())) { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_UNLIKELY(not get_bson_cstr(key))) - { - return false; - } - - if (not is_array) - { - if (not sax->key(key)) - { - return false; - } - } - - if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); } - return true; + reference_tokens.pop_back(); } /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - std::int32_t document_size; - get_number<std::int32_t, true>(input_format_t::bson, document_size); + @brief return last reference token - if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) - { - return false; - } + @pre not `empty()` + @return last reference token - if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) - { - return false; - } + @liveexample{The example shows the usage of `back`.,json_pointer__back} - return sax->end_array(); - } - - ////////// - // CBOR // - ////////// + @complexity Constant. - /*! - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead + @throw out_of_range.405 if JSON pointer has no parent - @return whether a valid CBOR value was passed to the SAX parser + @since version 3.6.0 */ - bool parse_cbor_internal(const bool get_char = true) + const std::string& back() { - switch (get_char ? get() : current) + if (JSON_UNLIKELY(empty())) { - // EOF - case std::char_traits<char>::eof(): - return unexpect_eof(input_format_t::cbor, "value"); - - // Integer 0x00..0x17 (0..23) - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - return sax->number_unsigned(static_cast<number_unsigned_t>(current)); - - case 0x18: // Unsigned integer (one-byte uint8_t follows) - { - uint8_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } - - case 0x19: // Unsigned integer (two-byte uint16_t follows) - { - uint16_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); + } - case 0x1A: // Unsigned integer (four-byte uint32_t follows) - { - uint32_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } + return reference_tokens.back(); + } - case 0x1B: // Unsigned integer (eight-byte uint64_t follows) - { - uint64_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } + /*! + @brief append an unescaped token at the end of the reference pointer - // Negative integer -1-0x00..-1-0x17 (-1..-24) - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current)); + @param[in] token token to add - case 0x38: // Negative integer (one-byte uint8_t follows) - { - uint8_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); - } + @complexity Amortized constant. - case 0x39: // Negative integer -1-n (two-byte uint16_t follows) - { - uint16_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); - } + @liveexample{The example shows the result of `push_back` for different + JSON Pointers.,json_pointer__push_back} - case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) - { - uint32_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); - } + @since version 3.6.0 + */ + void push_back(const std::string& token) + { + reference_tokens.push_back(token); + } - case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) - { - uint64_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - - static_cast<number_integer_t>(number)); - } + /// @copydoc push_back(const std::string&) + void push_back(std::string&& token) + { + reference_tokens.push_back(std::move(token)); + } - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) - case 0x7F: // UTF-8 string (indefinite length) - { - string_t s; - return get_cbor_string(s) and sax->string(s); - } + /*! + @brief return whether pointer points to the root document - // array (0x00..0x17 data items follow) - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - return get_cbor_array(static_cast<std::size_t>(current & 0x1F)); + @return true iff the JSON pointer points to the root document - case 0x98: // array (one-byte uint8_t for n follows) - { - uint8_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); - } + @complexity Constant. - case 0x99: // array (two-byte uint16_t for n follow) - { - uint16_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); - } + @exceptionsafety No-throw guarantee: this function never throws exceptions. - case 0x9A: // array (four-byte uint32_t for n follow) - { - uint32_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); - } + @liveexample{The example shows the result of `empty` for different JSON + Pointers.,json_pointer__empty} - case 0x9B: // array (eight-byte uint64_t for n follow) - { - uint64_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); - } + @since version 3.6.0 + */ + bool empty() const noexcept + { + return reference_tokens.empty(); + } - case 0x9F: // array (indefinite length) - return get_cbor_array(std::size_t(-1)); + private: + /*! + @param[in] s reference token to be converted into an array index - // map (0x00..0x17 pairs of data items follow) - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - return get_cbor_object(static_cast<std::size_t>(current & 0x1F)); + @return integer representation of @a s - case 0xB8: // map (one-byte uint8_t for n follows) - { - uint8_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); - } + @throw out_of_range.404 if string @a s could not be converted to an integer + */ + static int array_index(const std::string& s) + { + std::size_t processed_chars = 0; + const int res = std::stoi(s, &processed_chars); - case 0xB9: // map (two-byte uint16_t for n follow) - { - uint16_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); - } + // check if the string was completely read + if (JSON_UNLIKELY(processed_chars != s.size())) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); + } - case 0xBA: // map (four-byte uint32_t for n follow) - { - uint32_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); - } + return res; + } - case 0xBB: // map (eight-byte uint64_t for n follow) - { - uint64_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); - } + json_pointer top() const + { + if (JSON_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); + } - case 0xBF: // map (indefinite length) - return get_cbor_object(std::size_t(-1)); + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } - case 0xF4: // false - return sax->boolean(false); + /*! + @brief create and return a reference to the pointed to value - case 0xF5: // true - return sax->boolean(true); + @complexity Linear in the number of reference tokens. - case 0xF6: // null - return sax->null(); + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened + */ + BasicJsonType& get_and_create(BasicJsonType& j) const + { + using size_type = typename BasicJsonType::size_type; + auto result = &j; - case 0xF9: // Half-Precision Float (two-byte IEEE 754) + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) { - const int byte1_raw = get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + case detail::value_t::null: { - return false; + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; } - const int byte2_raw = get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + + case detail::value_t::object: { - return false; + // create an entry in the object + result = &result->operator[](reference_token); + break; } - const auto byte1 = static_cast<unsigned char>(byte1_raw); - const auto byte2 = static_cast<unsigned char>(byte2_raw); - - // code from RFC 7049, Appendix D, Figure 3: - // As half-precision floating-point numbers were only added - // to IEEE 754 in 2008, today's programming platforms often - // still only have limited support for them. It is very - // easy to include at least decoding support for them even - // without such support. An example of a small decoder for - // half-precision floating-point numbers in the C language - // is shown in Fig. 3. - const int half = (byte1 << 8) + byte2; - const double val = [&half] + case detail::value_t::array: { - const int exp = (half >> 10) & 0x1F; - const int mant = half & 0x3FF; - assert(0 <= exp and exp <= 32); - assert(0 <= mant and mant <= 1024); - switch (exp) + // create an entry in the array + JSON_TRY { - case 0: - return std::ldexp(mant, -24); - case 31: - return (mant == 0) - ? std::numeric_limits<double>::infinity() - : std::numeric_limits<double>::quiet_NaN(); - default: - return std::ldexp(mant + 1024, exp - 25); + result = &result->operator[](static_cast<size_type>(array_index(reference_token))); } - }(); - return sax->number_float((half & 0x8000) != 0 - ? static_cast<number_float_t>(-val) - : static_cast<number_float_t>(val), ""); - } - - case 0xFA: // Single-Precision Float (four-byte IEEE 754) - { - float number; - return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); - } - - case 0xFB: // Double-Precision Float (eight-byte IEEE 754) - { - double number; - return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); - } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } - default: // anything else (0xFF is handled inside the other types) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); } } + + return *result; } /*! - @brief reads a CBOR string + @brief return a reference to the pointed to value - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + @note This version does not throw if a value is not present, but tries to + create nested values instead. For instance, calling this function + with pointer `"/this/that"` on a null value is equivalent to calling + `operator[]("this").operator[]("that")` on that value, effectively + changing the null value to an object. - @param[out] result created string + @param[in] ptr a JSON value - @return whether string creation completed + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved */ - bool get_cbor_string(string_t& result) + BasicJsonType& get_unchecked(BasicJsonType* ptr) const { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) - { - return false; - } - - switch (current) + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) { - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - { - return get_string(input_format_t::cbor, current & 0x1F, result); - } - - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) - { - uint8_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); - } - - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + // convert null values to arrays or objects before continuing + if (ptr->m_type == detail::value_t::null) { - uint16_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); - } + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const char x) + { + return x >= '0' and x <= '9'; + }); - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) - { - uint32_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums or reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; } - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + switch (ptr->m_type) { - uint64_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); - } + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } - case 0x7F: // UTF-8 string (indefinite length) - { - while (get() != 0xFF) + case detail::value_t::array: { - string_t chunk; - if (not get_cbor_string(chunk)) + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) { - return false; + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); } - result.append(chunk); + + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + } + break; } - return true; - } - default: - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } + + return *ptr; } /*! - @param[in] len the length of the array or std::size_t(-1) for an - array of indefinite size - @return whether array creation completed + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved */ - bool get_cbor_array(const std::size_t len) + BasicJsonType& get_checked(BasicJsonType* ptr) const { - if (JSON_UNLIKELY(not sax->start_array(len))) - { - return false; - } - - if (len != std::size_t(-1)) + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) { - for (std::size_t i = 0; i < len; ++i) + switch (ptr->m_type) { - if (JSON_UNLIKELY(not parse_cbor_internal())) + case detail::value_t::object: { - return false; + // note: at performs range check + ptr = &ptr->at(reference_token); + break; } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_UNLIKELY(not parse_cbor_internal(false))) + + case detail::value_t::array: { - return false; + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } - return sax->end_array(); + return *ptr; } /*! - @param[in] len the length of the object or std::size_t(-1) for an - object of indefinite size - @return whether object creation completed + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved */ - bool get_cbor_object(const std::size_t len) + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const { - if (not JSON_UNLIKELY(sax->start_object(len))) - { - return false; - } - - string_t key; - if (len != std::size_t(-1)) + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) { - for (std::size_t i = 0; i < len; ++i) + switch (ptr->m_type) { - get(); - if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + case detail::value_t::object: { - return false; + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; } - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; - } - key.clear(); - } - } - else - { - while (get() != 0xFF) - { - if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + case detail::value_t::array: { - return false; - } + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // use unchecked array access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; } - key.clear(); + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } - return sax->end_object(); + return *ptr; } - ///////////// - // MsgPack // - ///////////// - /*! - @return whether a valid MessagePack value was passed to the SAX parser + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved */ - bool parse_msgpack_internal() + const BasicJsonType& get_checked(const BasicJsonType* ptr) const { - switch (get()) + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) { - // EOF - case std::char_traits<char>::eof(): - return unexpect_eof(input_format_t::msgpack, "value"); - - // positive fixint - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - case 0x38: - case 0x39: - case 0x3A: - case 0x3B: - case 0x3C: - case 0x3D: - case 0x3E: - case 0x3F: - case 0x40: - case 0x41: - case 0x42: - case 0x43: - case 0x44: - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0x49: - case 0x4A: - case 0x4B: - case 0x4C: - case 0x4D: - case 0x4E: - case 0x4F: - case 0x50: - case 0x51: - case 0x52: - case 0x53: - case 0x54: - case 0x55: - case 0x56: - case 0x57: - case 0x58: - case 0x59: - case 0x5A: - case 0x5B: - case 0x5C: - case 0x5D: - case 0x5E: - case 0x5F: - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: - case 0x79: - case 0x7A: - case 0x7B: - case 0x7C: - case 0x7D: - case 0x7E: - case 0x7F: - return sax->number_unsigned(static_cast<number_unsigned_t>(current)); - - // fixmap - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - return get_msgpack_object(static_cast<std::size_t>(current & 0x0F)); - - // fixarray - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - case 0x98: - case 0x99: - case 0x9A: - case 0x9B: - case 0x9C: - case 0x9D: - case 0x9E: - case 0x9F: - return get_msgpack_array(static_cast<std::size_t>(current & 0x0F)); - - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: - { - string_t s; - return get_msgpack_string(s) and sax->string(s); - } - - case 0xC0: // nil - return sax->null(); - - case 0xC2: // false - return sax->boolean(false); - - case 0xC3: // true - return sax->boolean(true); - - case 0xCA: // float 32 - { - float number; - return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); - } - - case 0xCB: // float 64 - { - double number; - return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); - } - - case 0xCC: // uint 8 - { - uint8_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xCD: // uint 16 - { - uint16_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xCE: // uint 32 - { - uint32_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xCF: // uint 64 - { - uint64_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xD0: // int 8 - { - int8_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD1: // int 16 - { - int16_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD2: // int 32 - { - int32_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD3: // int 64 - { - int64_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD9: // str 8 - case 0xDA: // str 16 - case 0xDB: // str 32 - { - string_t s; - return get_msgpack_string(s) and sax->string(s); - } - - case 0xDC: // array 16 - { - uint16_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); - } - - case 0xDD: // array 32 + switch (ptr->m_type) { - uint32_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); - } + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } - case 0xDE: // map 16 - { - uint16_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); - } + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } - case 0xDF: // map 32 - { - uint32_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); - } + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } - // negative fixint - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - case 0xF0: - case 0xF1: - case 0xF2: - case 0xF3: - case 0xF4: - case 0xF5: - case 0xF6: - case 0xF7: - case 0xF8: - case 0xF9: - case 0xFA: - case 0xFB: - case 0xFC: - case 0xFD: - case 0xFE: - case 0xFF: - return sax->number_integer(static_cast<int8_t>(current)); + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } - default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } + + return *ptr; } /*! - @brief reads a MessagePack string - - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. + @brief split the string input to reference tokens - @param[out] result created string + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. - @return whether string creation completed + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' */ - bool get_msgpack_string(string_t& result) + static std::vector<std::string> split(const std::string& reference_string) { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) + std::vector<std::string> result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) { - return false; + return result; } - switch (current) + // check if nonempty reference string begins with slash + if (JSON_UNLIKELY(reference_string[0] != '/')) { - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: - { - return get_string(input_format_t::msgpack, current & 0x1F, result); - } + JSON_THROW(detail::parse_error::create(107, 1, + "JSON pointer must be empty or begin with '/' - was: '" + + reference_string + "'")); + } - case 0xD9: // str 8 - { - uint8_t len; - return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); - } + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + std::size_t slash = reference_string.find_first_of('/', 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == 0 (if slash == std::string::npos) + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = (slash == std::string::npos) ? 0 : slash + 1, + // find next slash + slash = reference_string.find_first_of('/', start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); - case 0xDA: // str 16 + // check reference tokens are properly escaped + for (std::size_t pos = reference_token.find_first_of('~'); + pos != std::string::npos; + pos = reference_token.find_first_of('~', pos + 1)) { - uint16_t len; - return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); - } + assert(reference_token[pos] == '~'); - case 0xDB: // str 32 - { - uint32_t len; - return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + // ~ must be followed by 0 or 1 + if (JSON_UNLIKELY(pos == reference_token.size() - 1 or + (reference_token[pos + 1] != '0' and + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'")); + } } - default: - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); - } + // finally, store the reference token + unescape(reference_token); + result.push_back(reference_token); } + + return result; } /*! - @param[in] len the length of the array - @return whether array creation completed - */ - bool get_msgpack_array(const std::size_t len) - { - if (JSON_UNLIKELY(not sax->start_array(len))) - { - return false; - } + @brief replace all occurrences of a substring by another string - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_UNLIKELY(not parse_msgpack_internal())) - { - return false; - } - } + @param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t + @param[in] f the substring to replace with @a t + @param[in] t the string to replace @a f - return sax->end_array(); - } + @pre The search string @a f must not be empty. **This precondition is + enforced with an assertion.** - /*! - @param[in] len the length of the object - @return whether object creation completed + @since version 2.0.0 */ - bool get_msgpack_object(const std::size_t len) + static void replace_substring(std::string& s, const std::string& f, + const std::string& t) { - if (JSON_UNLIKELY(not sax->start_object(len))) - { - return false; - } - - string_t key; - for (std::size_t i = 0; i < len; ++i) - { - get(); - if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) - { - return false; - } - - if (JSON_UNLIKELY(not parse_msgpack_internal())) - { - return false; - } - key.clear(); - } - - return sax->end_object(); + assert(not f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} } - //////////// - // UBJSON // - //////////// - - /*! - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead - - @return whether a valid UBJSON value was passed to the SAX parser - */ - bool parse_ubjson_internal(const bool get_char = true) + /// escape "~" to "~0" and "/" to "~1" + static std::string escape(std::string s) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; } - /*! - @brief reads a UBJSON string - - This function is either called after reading the 'S' byte explicitly - indicating a string, or in case of an object key where the 'S' byte can be - left out. - - @param[out] result created string - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead - - @return whether string creation completed - */ - bool get_ubjson_string(string_t& result, const bool get_char = true) + /// unescape "~1" to tilde and "~0" to slash (order is important!) + static void unescape(std::string& s) { - if (get_char) - { - get(); // TODO: may we ignore N here? - } - - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) - { - return false; - } - - switch (current) - { - case 'U': - { - uint8_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'i': - { - int8_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'I': - { - int16_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'l': - { - int32_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'L': - { - int64_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - default: - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); - } + replace_substring(s, "~1", "/"); + replace_substring(s, "~0", "~"); } /*! - @param[out] result determined size - @return whether size determination completed + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. */ - bool get_ubjson_size_value(std::size_t& result) + static void flatten(const std::string& reference_string, + const BasicJsonType& value, + BasicJsonType& result) { - switch (get_ignore_noop()) + switch (value.m_type) { - case 'U': - { - uint8_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) - { - return false; - } - result = static_cast<std::size_t>(number); - return true; - } - - case 'i': + case detail::value_t::array: { - int8_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + if (value.m_value.array->empty()) { - return false; + // flatten empty array as null + result[reference_string] = nullptr; } - result = static_cast<std::size_t>(number); - return true; - } - - case 'I': - { - int16_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + else { - return false; + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } } - result = static_cast<std::size_t>(number); - return true; + break; } - case 'l': + case detail::value_t::object: { - int32_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + if (value.m_value.object->empty()) { - return false; + // flatten empty object as null + result[reference_string] = nullptr; } - result = static_cast<std::size_t>(number); - return true; - } - - case 'L': - { - int64_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + else { - return false; + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), element.second, result); + } } - result = static_cast<std::size_t>(number); - return true; + break; } default: { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); + // add primitive value with its reference string + result[reference_string] = value; + break; } } } /*! - @brief determine the type and size for a container - - In the optimized UBJSON format, a type and a size can be provided to allow - for a more compact representation. + @param[in] value flattened JSON - @param[out] result pair of the size and the type + @return unflattened JSON - @return whether pair creation completed + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened */ - bool get_ubjson_size_type(std::pair<std::size_t, int>& result) + static BasicJsonType + unflatten(const BasicJsonType& value) { - result.first = string_t::npos; // size - result.second = 0; // type + if (JSON_UNLIKELY(not value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); + } - get_ignore_noop(); + BasicJsonType result; - if (current == '$') + // iterate the JSON object values + for (const auto& element : *value.m_value.object) { - result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) - { - return false; - } - - get_ignore_noop(); - if (JSON_UNLIKELY(current != '#')) + if (JSON_UNLIKELY(not element.second.is_primitive())) { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) - { - return false; - } - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); + JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); } - return get_ubjson_size_value(result.first); - } - else if (current == '#') - { - return get_ubjson_size_value(result.first); + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; } - return true; + + return result; } /*! - @param prefix the previously read or set type prefix - @return whether value creation completed + @brief compares two JSON pointers for equality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is equal to @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. */ - bool get_ubjson_value(const int prefix) + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept { - switch (prefix) - { - case std::char_traits<char>::eof(): // EOF - return unexpect_eof(input_format_t::ubjson, "value"); + return lhs.reference_tokens == rhs.reference_tokens; + } - case 'T': // true - return sax->boolean(true); - case 'F': // false - return sax->boolean(false); + /*! + @brief compares two JSON pointers for inequality - case 'Z': // null - return sax->null(); + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is not equal @a rhs - case 'U': - { - uint8_t number; - return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); - } + @complexity Linear in the length of the JSON pointer - case 'i': - { - int8_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return not (lhs == rhs); + } - case 'I': - { - int16_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } + /// the reference tokens + std::vector<std::string> reference_tokens; +}; +} // namespace nlohmann - case 'l': - { - int32_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } +// #include <nlohmann/detail/json_ref.hpp> - case 'L': - { - int64_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } - case 'd': - { - float number; - return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); - } +#include <initializer_list> +#include <utility> - case 'D': - { - double number; - return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); - } +// #include <nlohmann/detail/meta/type_traits.hpp> - case 'C': // char - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) - { - return false; - } - if (JSON_UNLIKELY(current > 127)) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); - } - string_t s(1, static_cast<char>(current)); - return sax->string(s); - } - case 'S': // string - { - string_t s; - return get_ubjson_string(s) and sax->string(s); - } +namespace nlohmann +{ +namespace detail +{ +template<typename BasicJsonType> +class json_ref +{ + public: + using value_type = BasicJsonType; - case '[': // array - return get_ubjson_array(); + json_ref(value_type&& value) + : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true) + {} - case '{': // object - return get_ubjson_object(); + json_ref(const value_type& value) + : value_ref(const_cast<value_type*>(&value)), is_rvalue(false) + {} - default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); - } - } - } + json_ref(std::initializer_list<json_ref> init) + : owned_value(init), value_ref(&owned_value), is_rvalue(true) + {} - /*! - @return whether array creation completed - */ - bool get_ubjson_array() - { - std::pair<std::size_t, int> size_and_type; - if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) - { - return false; - } + template < + class... Args, + enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), + is_rvalue(true) {} - if (size_and_type.first != string_t::npos) - { - if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) - { - return false; - } + // class should be movable only + json_ref(json_ref&&) = default; + json_ref(const json_ref&) = delete; + json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; - if (size_and_type.second != 0) - { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not parse_ubjson_internal())) - { - return false; - } - } - } - } - else + value_type moved_or_copied() const + { + if (is_rvalue) { - if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) - { - return false; - } - - while (current != ']') - { - if (JSON_UNLIKELY(not parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return std::move(*value_ref); } - - return sax->end_array(); + return *value_ref; } - /*! - @return whether object creation completed - */ - bool get_ubjson_object() + value_type const& operator*() const { - std::pair<std::size_t, int> size_and_type; - if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) - { - return false; - } + return *static_cast<value_type const*>(value_ref); + } - string_t key; - if (size_and_type.first != string_t::npos) - { - if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) - { - return false; - } + value_type const* operator->() const + { + return static_cast<value_type const*>(value_ref); + } - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) - { - return false; - } - if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) - { - return false; - } - if (JSON_UNLIKELY(not parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) - { - return false; - } + private: + mutable value_type owned_value = nullptr; + value_type* value_ref = nullptr; + const bool is_rvalue; +}; +} // namespace detail +} // namespace nlohmann - while (current != '}') - { - if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) - { - return false; - } - if (JSON_UNLIKELY(not parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } - } +// #include <nlohmann/detail/macro_scope.hpp> - return sax->end_object(); - } +// #include <nlohmann/detail/meta/cpp_future.hpp> - /////////////////////// - // Utility functions // - /////////////////////// +// #include <nlohmann/detail/meta/type_traits.hpp> - /*! - @brief get next character from the input +// #include <nlohmann/detail/output/binary_writer.hpp> - This function provides the interface to the used input adapter. It does - not throw in case the input reached EOF, but returns a -'ve valued - `std::char_traits<char>::eof()` in that case. - @return character read from the input - */ - int get() - { - ++chars_read; - return (current = ia->get_character()); - } +#include <algorithm> // reverse +#include <array> // array +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstring> // memcpy +#include <limits> // numeric_limits +#include <string> // string - /*! - @return character read from the input after ignoring all 'N' entries - */ - int get_ignore_noop() - { - do - { - get(); - } - while (current == 'N'); +// #include <nlohmann/detail/input/binary_reader.hpp> - return current; - } +// #include <nlohmann/detail/output/output_adapters.hpp> - /* - @brief read a number from the input - @tparam NumberType the type of the number - @param[in] format the current format (for diagnostics) - @param[out] result number of type @a NumberType +#include <algorithm> // copy +#include <cstddef> // size_t +#include <ios> // streamsize +#include <iterator> // back_inserter +#include <memory> // shared_ptr, make_shared +#include <ostream> // basic_ostream +#include <string> // basic_string +#include <vector> // vector - @return whether conversion completed +namespace nlohmann +{ +namespace detail +{ +/// abstract output adapter interface +template<typename CharType> struct output_adapter_protocol +{ + virtual void write_character(CharType c) = 0; + virtual void write_characters(const CharType* s, std::size_t length) = 0; + virtual ~output_adapter_protocol() = default; +}; - @note This function needs to respect the system's endianess, because - bytes in CBOR, MessagePack, and UBJSON are stored in network order - (big endian) and therefore need reordering on little endian systems. - */ - template<typename NumberType, bool InputIsLittleEndian = false> - bool get_number(const input_format_t format, NumberType& result) - { - // step 1: read input into array with system's byte order - std::array<uint8_t, sizeof(NumberType)> vec; - for (std::size_t i = 0; i < sizeof(NumberType); ++i) - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) - { - return false; - } +/// a type to simplify interfaces +template<typename CharType> +using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>; - // reverse byte order prior to conversion if necessary - if (is_little_endian && !InputIsLittleEndian) - { - vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current); - } - else - { - vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE - } - } +/// output adapter for byte vectors +template<typename CharType> +class output_vector_adapter : public output_adapter_protocol<CharType> +{ + public: + explicit output_vector_adapter(std::vector<CharType>& vec) noexcept + : v(vec) + {} - // step 2: convert array into number of type T and return - std::memcpy(&result, vec.data(), sizeof(NumberType)); - return true; + void write_character(CharType c) override + { + v.push_back(c); } - /*! - @brief create a string by reading characters from the input + void write_characters(const CharType* s, std::size_t length) override + { + std::copy(s, s + length, std::back_inserter(v)); + } - @tparam NumberType the type of the number - @param[in] format the current format (for diagnostics) - @param[in] len number of characters to read - @param[out] result string created by reading @a len bytes + private: + std::vector<CharType>& v; +}; - @return whether string creation completed +/// output adapter for output streams +template<typename CharType> +class output_stream_adapter : public output_adapter_protocol<CharType> +{ + public: + explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept + : stream(s) + {} - @note We can not reserve @a len bytes for the result, because @a len - may be too large. Usually, @ref unexpect_eof() detects the end of - the input before we run out of string memory. - */ - template<typename NumberType> - bool get_string(const input_format_t format, - const NumberType len, - string_t& result) + void write_character(CharType c) override { - bool success = true; - std::generate_n(std::back_inserter(result), len, [this, &success, &format]() - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) - { - success = false; - } - return static_cast<char>(current); - }); - return success; + stream.put(c); } - /*! - @param[in] format the current format (for diagnostics) - @param[in] context further context information (for diagnostics) - @return whether the last read character is not EOF - */ - bool unexpect_eof(const input_format_t format, const char* context) const + void write_characters(const CharType* s, std::size_t length) override { - if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) - { - return sax->parse_error(chars_read, "<end of file>", - parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); - } - return true; + stream.write(s, static_cast<std::streamsize>(length)); } - /*! - @return a string representation of the last read byte - */ - std::string get_token_string() const + private: + std::basic_ostream<CharType>& stream; +}; + +/// output adapter for basic_string +template<typename CharType, typename StringType = std::basic_string<CharType>> +class output_string_adapter : public output_adapter_protocol<CharType> +{ + public: + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} + + void write_character(CharType c) override { - char cr[3]; - (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current)); - return std::string{cr}; + str.push_back(c); } - /*! - @param[in] format the current format - @param[in] detail a detailed error message - @param[in] context further contect information - @return a message string to use in the parse_error exceptions - */ - std::string exception_message(const input_format_t format, - const std::string& detail, - const std::string& context) const + void write_characters(const CharType* s, std::size_t length) override { - std::string error_msg = "syntax error while parsing "; - - switch (format) - { - case input_format_t::cbor: - error_msg += "CBOR"; - break; + str.append(s, length); + } - case input_format_t::msgpack: - error_msg += "MessagePack"; - break; + private: + StringType& str; +}; - case input_format_t::ubjson: - error_msg += "UBJSON"; - break; +template<typename CharType, typename StringType = std::basic_string<CharType>> +class output_adapter +{ + public: + output_adapter(std::vector<CharType>& vec) + : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {} - case input_format_t::bson: - error_msg += "BSON"; - break; + output_adapter(std::basic_ostream<CharType>& s) + : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {} - // LCOV_EXCL_START - default: - assert(false); - // LCOV_EXCL_STOP - } + output_adapter(StringType& s) + : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {} - return error_msg + " " + context + ": " + detail; + operator output_adapter_t<CharType>() + { + return oa; } private: - /// input adapter - input_adapter_t ia = nullptr; - - /// the current character - int current = std::char_traits<char>::eof(); - - /// the number of characters read - std::size_t chars_read = 0; - - /// whether we can assume little endianess - const bool is_little_endian = little_endianess(); - - /// the SAX parser - json_sax_t* sax = nullptr; + output_adapter_t<CharType> oa = nullptr; }; } // namespace detail } // namespace nlohmann -// #include <nlohmann/detail/output/binary_writer.hpp> - - -#include <algorithm> // reverse -#include <array> // array -#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t -#include <cstring> // memcpy -#include <limits> // numeric_limits - -// #include <nlohmann/detail/input/binary_reader.hpp> - -// #include <nlohmann/detail/output/output_adapters.hpp> - namespace nlohmann { @@ -8528,27 +9638,27 @@ class binary_writer // code from the value_t::number_unsigned case here. if (j.m_value.number_integer <= 0x17) { - write_number(static_cast<uint8_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) + else if (j.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)()) { oa->write_character(to_char_type(0x18)); - write_number(static_cast<uint8_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)()) + else if (j.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)()) { oa->write_character(to_char_type(0x19)); - write_number(static_cast<uint16_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint16_t>(j.m_value.number_integer)); } - else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)()) + else if (j.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)()) { oa->write_character(to_char_type(0x1A)); - write_number(static_cast<uint32_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint32_t>(j.m_value.number_integer)); } else { oa->write_character(to_char_type(0x1B)); - write_number(static_cast<uint64_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint64_t>(j.m_value.number_integer)); } } else @@ -8558,27 +9668,27 @@ class binary_writer const auto positive_number = -1 - j.m_value.number_integer; if (j.m_value.number_integer >= -24) { - write_number(static_cast<uint8_t>(0x20 + positive_number)); + write_number(static_cast<std::uint8_t>(0x20 + positive_number)); } - else if (positive_number <= (std::numeric_limits<uint8_t>::max)()) + else if (positive_number <= (std::numeric_limits<std::uint8_t>::max)()) { oa->write_character(to_char_type(0x38)); - write_number(static_cast<uint8_t>(positive_number)); + write_number(static_cast<std::uint8_t>(positive_number)); } - else if (positive_number <= (std::numeric_limits<uint16_t>::max)()) + else if (positive_number <= (std::numeric_limits<std::uint16_t>::max)()) { oa->write_character(to_char_type(0x39)); - write_number(static_cast<uint16_t>(positive_number)); + write_number(static_cast<std::uint16_t>(positive_number)); } - else if (positive_number <= (std::numeric_limits<uint32_t>::max)()) + else if (positive_number <= (std::numeric_limits<std::uint32_t>::max)()) { oa->write_character(to_char_type(0x3A)); - write_number(static_cast<uint32_t>(positive_number)); + write_number(static_cast<std::uint32_t>(positive_number)); } else { oa->write_character(to_char_type(0x3B)); - write_number(static_cast<uint64_t>(positive_number)); + write_number(static_cast<std::uint64_t>(positive_number)); } } break; @@ -8588,27 +9698,27 @@ class binary_writer { if (j.m_value.number_unsigned <= 0x17) { - write_number(static_cast<uint8_t>(j.m_value.number_unsigned)); + write_number(static_cast<std::uint8_t>(j.m_value.number_unsigned)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)()) { oa->write_character(to_char_type(0x18)); - write_number(static_cast<uint8_t>(j.m_value.number_unsigned)); + write_number(static_cast<std::uint8_t>(j.m_value.number_unsigned)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)()) { oa->write_character(to_char_type(0x19)); - write_number(static_cast<uint16_t>(j.m_value.number_unsigned)); + write_number(static_cast<std::uint16_t>(j.m_value.number_unsigned)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)()) { oa->write_character(to_char_type(0x1A)); - write_number(static_cast<uint32_t>(j.m_value.number_unsigned)); + write_number(static_cast<std::uint32_t>(j.m_value.number_unsigned)); } else { oa->write_character(to_char_type(0x1B)); - write_number(static_cast<uint64_t>(j.m_value.number_unsigned)); + write_number(static_cast<std::uint64_t>(j.m_value.number_unsigned)); } break; } @@ -8626,28 +9736,28 @@ class binary_writer const auto N = j.m_value.string->size(); if (N <= 0x17) { - write_number(static_cast<uint8_t>(0x60 + N)); + write_number(static_cast<std::uint8_t>(0x60 + N)); } - else if (N <= (std::numeric_limits<uint8_t>::max)()) + else if (N <= (std::numeric_limits<std::uint8_t>::max)()) { oa->write_character(to_char_type(0x78)); - write_number(static_cast<uint8_t>(N)); + write_number(static_cast<std::uint8_t>(N)); } - else if (N <= (std::numeric_limits<uint16_t>::max)()) + else if (N <= (std::numeric_limits<std::uint16_t>::max)()) { oa->write_character(to_char_type(0x79)); - write_number(static_cast<uint16_t>(N)); + write_number(static_cast<std::uint16_t>(N)); } - else if (N <= (std::numeric_limits<uint32_t>::max)()) + else if (N <= (std::numeric_limits<std::uint32_t>::max)()) { oa->write_character(to_char_type(0x7A)); - write_number(static_cast<uint32_t>(N)); + write_number(static_cast<std::uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= (std::numeric_limits<uint64_t>::max)()) + else if (N <= (std::numeric_limits<std::uint64_t>::max)()) { oa->write_character(to_char_type(0x7B)); - write_number(static_cast<uint64_t>(N)); + write_number(static_cast<std::uint64_t>(N)); } // LCOV_EXCL_STOP @@ -8664,28 +9774,28 @@ class binary_writer const auto N = j.m_value.array->size(); if (N <= 0x17) { - write_number(static_cast<uint8_t>(0x80 + N)); + write_number(static_cast<std::uint8_t>(0x80 + N)); } - else if (N <= (std::numeric_limits<uint8_t>::max)()) + else if (N <= (std::numeric_limits<std::uint8_t>::max)()) { oa->write_character(to_char_type(0x98)); - write_number(static_cast<uint8_t>(N)); + write_number(static_cast<std::uint8_t>(N)); } - else if (N <= (std::numeric_limits<uint16_t>::max)()) + else if (N <= (std::numeric_limits<std::uint16_t>::max)()) { oa->write_character(to_char_type(0x99)); - write_number(static_cast<uint16_t>(N)); + write_number(static_cast<std::uint16_t>(N)); } - else if (N <= (std::numeric_limits<uint32_t>::max)()) + else if (N <= (std::numeric_limits<std::uint32_t>::max)()) { oa->write_character(to_char_type(0x9A)); - write_number(static_cast<uint32_t>(N)); + write_number(static_cast<std::uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= (std::numeric_limits<uint64_t>::max)()) + else if (N <= (std::numeric_limits<std::uint64_t>::max)()) { oa->write_character(to_char_type(0x9B)); - write_number(static_cast<uint64_t>(N)); + write_number(static_cast<std::uint64_t>(N)); } // LCOV_EXCL_STOP @@ -8703,28 +9813,28 @@ class binary_writer const auto N = j.m_value.object->size(); if (N <= 0x17) { - write_number(static_cast<uint8_t>(0xA0 + N)); + write_number(static_cast<std::uint8_t>(0xA0 + N)); } - else if (N <= (std::numeric_limits<uint8_t>::max)()) + else if (N <= (std::numeric_limits<std::uint8_t>::max)()) { oa->write_character(to_char_type(0xB8)); - write_number(static_cast<uint8_t>(N)); + write_number(static_cast<std::uint8_t>(N)); } - else if (N <= (std::numeric_limits<uint16_t>::max)()) + else if (N <= (std::numeric_limits<std::uint16_t>::max)()) { oa->write_character(to_char_type(0xB9)); - write_number(static_cast<uint16_t>(N)); + write_number(static_cast<std::uint16_t>(N)); } - else if (N <= (std::numeric_limits<uint32_t>::max)()) + else if (N <= (std::numeric_limits<std::uint32_t>::max)()) { oa->write_character(to_char_type(0xBA)); - write_number(static_cast<uint32_t>(N)); + write_number(static_cast<std::uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= (std::numeric_limits<uint64_t>::max)()) + else if (N <= (std::numeric_limits<std::uint64_t>::max)()) { oa->write_character(to_char_type(0xBB)); - write_number(static_cast<uint64_t>(N)); + write_number(static_cast<std::uint64_t>(N)); } // LCOV_EXCL_STOP @@ -8773,31 +9883,31 @@ class binary_writer if (j.m_value.number_unsigned < 128) { // positive fixnum - write_number(static_cast<uint8_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)()) { // uint 8 oa->write_character(to_char_type(0xCC)); - write_number(static_cast<uint8_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)()) { // uint 16 oa->write_character(to_char_type(0xCD)); - write_number(static_cast<uint16_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint16_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)()) { // uint 32 oa->write_character(to_char_type(0xCE)); - write_number(static_cast<uint32_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint32_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)()) { // uint 64 oa->write_character(to_char_type(0xCF)); - write_number(static_cast<uint64_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint64_t>(j.m_value.number_integer)); } } else @@ -8805,35 +9915,35 @@ class binary_writer if (j.m_value.number_integer >= -32) { // negative fixnum - write_number(static_cast<int8_t>(j.m_value.number_integer)); + write_number(static_cast<std::int8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and - j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) + else if (j.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)()) { // int 8 oa->write_character(to_char_type(0xD0)); - write_number(static_cast<int8_t>(j.m_value.number_integer)); + write_number(static_cast<std::int8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and - j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) + else if (j.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)()) { // int 16 oa->write_character(to_char_type(0xD1)); - write_number(static_cast<int16_t>(j.m_value.number_integer)); + write_number(static_cast<std::int16_t>(j.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and - j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) + else if (j.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)()) { // int 32 oa->write_character(to_char_type(0xD2)); - write_number(static_cast<int32_t>(j.m_value.number_integer)); + write_number(static_cast<std::int32_t>(j.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and - j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)()) + else if (j.m_value.number_integer >= (std::numeric_limits<std::int64_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)()) { // int 64 oa->write_character(to_char_type(0xD3)); - write_number(static_cast<int64_t>(j.m_value.number_integer)); + write_number(static_cast<std::int64_t>(j.m_value.number_integer)); } } break; @@ -8844,31 +9954,31 @@ class binary_writer if (j.m_value.number_unsigned < 128) { // positive fixnum - write_number(static_cast<uint8_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)()) { // uint 8 oa->write_character(to_char_type(0xCC)); - write_number(static_cast<uint8_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint8_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)()) { // uint 16 oa->write_character(to_char_type(0xCD)); - write_number(static_cast<uint16_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint16_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)()) { // uint 32 oa->write_character(to_char_type(0xCE)); - write_number(static_cast<uint32_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint32_t>(j.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)()) { // uint 64 oa->write_character(to_char_type(0xCF)); - write_number(static_cast<uint64_t>(j.m_value.number_integer)); + write_number(static_cast<std::uint64_t>(j.m_value.number_integer)); } break; } @@ -8887,25 +9997,25 @@ class binary_writer if (N <= 31) { // fixstr - write_number(static_cast<uint8_t>(0xA0 | N)); + write_number(static_cast<std::uint8_t>(0xA0 | N)); } - else if (N <= (std::numeric_limits<uint8_t>::max)()) + else if (N <= (std::numeric_limits<std::uint8_t>::max)()) { // str 8 oa->write_character(to_char_type(0xD9)); - write_number(static_cast<uint8_t>(N)); + write_number(static_cast<std::uint8_t>(N)); } - else if (N <= (std::numeric_limits<uint16_t>::max)()) + else if (N <= (std::numeric_limits<std::uint16_t>::max)()) { // str 16 oa->write_character(to_char_type(0xDA)); - write_number(static_cast<uint16_t>(N)); + write_number(static_cast<std::uint16_t>(N)); } - else if (N <= (std::numeric_limits<uint32_t>::max)()) + else if (N <= (std::numeric_limits<std::uint32_t>::max)()) { // str 32 oa->write_character(to_char_type(0xDB)); - write_number(static_cast<uint32_t>(N)); + write_number(static_cast<std::uint32_t>(N)); } // step 2: write the string @@ -8922,19 +10032,19 @@ class binary_writer if (N <= 15) { // fixarray - write_number(static_cast<uint8_t>(0x90 | N)); + write_number(static_cast<std::uint8_t>(0x90 | N)); } - else if (N <= (std::numeric_limits<uint16_t>::max)()) + else if (N <= (std::numeric_limits<std::uint16_t>::max)()) { // array 16 oa->write_character(to_char_type(0xDC)); - write_number(static_cast<uint16_t>(N)); + write_number(static_cast<std::uint16_t>(N)); } - else if (N <= (std::numeric_limits<uint32_t>::max)()) + else if (N <= (std::numeric_limits<std::uint32_t>::max)()) { // array 32 oa->write_character(to_char_type(0xDD)); - write_number(static_cast<uint32_t>(N)); + write_number(static_cast<std::uint32_t>(N)); } // step 2: write each element @@ -8952,19 +10062,19 @@ class binary_writer if (N <= 15) { // fixmap - write_number(static_cast<uint8_t>(0x80 | (N & 0xF))); + write_number(static_cast<std::uint8_t>(0x80 | (N & 0xF))); } - else if (N <= (std::numeric_limits<uint16_t>::max)()) + else if (N <= (std::numeric_limits<std::uint16_t>::max)()) { // map 16 oa->write_character(to_char_type(0xDE)); - write_number(static_cast<uint16_t>(N)); + write_number(static_cast<std::uint16_t>(N)); } - else if (N <= (std::numeric_limits<uint32_t>::max)()) + else if (N <= (std::numeric_limits<std::uint32_t>::max)()) { // map 32 oa->write_character(to_char_type(0xDF)); - write_number(static_cast<uint32_t>(N)); + write_number(static_cast<std::uint32_t>(N)); } // step 2: write each element @@ -9230,14 +10340,9 @@ class binary_writer */ static std::size_t calc_bson_integer_size(const std::int64_t value) { - if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)()) - { - return sizeof(std::int32_t); - } - else - { - return sizeof(std::int64_t); - } + return (std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)() + ? sizeof(std::int32_t) + : sizeof(std::int64_t); } /*! @@ -9374,7 +10479,7 @@ class binary_writer assert(false); return 0ul; // LCOV_EXCL_STOP - }; + } } /*! @@ -9418,7 +10523,7 @@ class binary_writer assert(false); return; // LCOV_EXCL_STOP - }; + } } /*! @@ -9505,45 +10610,45 @@ class binary_writer void write_number_with_ubjson_prefix(const NumberType n, const bool add_prefix) { - if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)())) + if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int8_t>::max)())) { if (add_prefix) { oa->write_character(to_char_type('i')); // int8 } - write_number(static_cast<uint8_t>(n)); + write_number(static_cast<std::uint8_t>(n)); } - else if (n <= (std::numeric_limits<uint8_t>::max)()) + else if (n <= (std::numeric_limits<std::uint8_t>::max)()) { if (add_prefix) { oa->write_character(to_char_type('U')); // uint8 } - write_number(static_cast<uint8_t>(n)); + write_number(static_cast<std::uint8_t>(n)); } - else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)())) + else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int16_t>::max)())) { if (add_prefix) { oa->write_character(to_char_type('I')); // int16 } - write_number(static_cast<int16_t>(n)); + write_number(static_cast<std::int16_t>(n)); } - else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)())) + else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())) { if (add_prefix) { oa->write_character(to_char_type('l')); // int32 } - write_number(static_cast<int32_t>(n)); + write_number(static_cast<std::int32_t>(n)); } - else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)())) + else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)())) { if (add_prefix) { oa->write_character(to_char_type('L')); // int64 } - write_number(static_cast<int64_t>(n)); + write_number(static_cast<std::int64_t>(n)); } else { @@ -9558,45 +10663,45 @@ class binary_writer void write_number_with_ubjson_prefix(const NumberType n, const bool add_prefix) { - if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)()) + if ((std::numeric_limits<std::int8_t>::min)() <= n and n <= (std::numeric_limits<std::int8_t>::max)()) { if (add_prefix) { oa->write_character(to_char_type('i')); // int8 } - write_number(static_cast<int8_t>(n)); + write_number(static_cast<std::int8_t>(n)); } - else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)())) + else if (static_cast<std::int64_t>((std::numeric_limits<std::uint8_t>::min)()) <= n and n <= static_cast<std::int64_t>((std::numeric_limits<std::uint8_t>::max)())) { if (add_prefix) { oa->write_character(to_char_type('U')); // uint8 } - write_number(static_cast<uint8_t>(n)); + write_number(static_cast<std::uint8_t>(n)); } - else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)()) + else if ((std::numeric_limits<std::int16_t>::min)() <= n and n <= (std::numeric_limits<std::int16_t>::max)()) { if (add_prefix) { oa->write_character(to_char_type('I')); // int16 } - write_number(static_cast<int16_t>(n)); + write_number(static_cast<std::int16_t>(n)); } - else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)()) + else if ((std::numeric_limits<std::int32_t>::min)() <= n and n <= (std::numeric_limits<std::int32_t>::max)()) { if (add_prefix) { oa->write_character(to_char_type('l')); // int32 } - write_number(static_cast<int32_t>(n)); + write_number(static_cast<std::int32_t>(n)); } - else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)()) + else if ((std::numeric_limits<std::int64_t>::min)() <= n and n <= (std::numeric_limits<std::int64_t>::max)()) { if (add_prefix) { oa->write_character(to_char_type('L')); // int64 } - write_number(static_cast<int64_t>(n)); + write_number(static_cast<std::int64_t>(n)); } // LCOV_EXCL_START else @@ -9627,19 +10732,19 @@ class binary_writer case value_t::number_integer: { - if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) + if ((std::numeric_limits<std::int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)()) { return 'i'; } - if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) + if ((std::numeric_limits<std::uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)()) { return 'U'; } - if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) + if ((std::numeric_limits<std::int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)()) { return 'I'; } - if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) + if ((std::numeric_limits<std::int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)()) { return 'l'; } @@ -9649,19 +10754,19 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits<std::int8_t>::max)()) { return 'i'; } - if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)()) { return 'U'; } - if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits<std::int16_t>::max)()) { return 'I'; } - if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits<std::int32_t>::max)()) { return 'l'; } @@ -9719,7 +10824,7 @@ class binary_writer std::memcpy(vec.data(), &n, sizeof(NumberType)); // step 2: write array to output (with possible reordering) - if (is_little_endian and not OutputIsLittleEndian) + if (is_little_endian != OutputIsLittleEndian) { // reverse byte order prior to conversion if necessary std::reverse(vec.begin(), vec.end()); @@ -9779,6 +10884,8 @@ class binary_writer } // namespace detail } // namespace nlohmann +// #include <nlohmann/detail/output/output_adapters.hpp> + // #include <nlohmann/detail/output/serializer.hpp> @@ -9794,17 +10901,19 @@ class binary_writer #include <limits> // numeric_limits #include <string> // string #include <type_traits> // is_same - -// #include <nlohmann/detail/exceptions.hpp> +#include <utility> // move // #include <nlohmann/detail/conversions/to_chars.hpp> +#include <array> // array #include <cassert> // assert #include <ciso646> // or, and, not #include <cmath> // signbit, isfinite #include <cstdint> // intN_t, uintN_t #include <cstring> // memcpy, memmove +#include <limits> // numeric_limits +#include <type_traits> // conditional namespace nlohmann { @@ -9847,10 +10956,10 @@ struct diyfp // f * 2^e { static constexpr int kPrecision = 64; // = q - uint64_t f = 0; + std::uint64_t f = 0; int e = 0; - constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {} /*! @brief returns x - y @@ -9895,23 +11004,23 @@ struct diyfp // f * 2^e // // = p_lo + 2^64 p_hi - const uint64_t u_lo = x.f & 0xFFFFFFFF; - const uint64_t u_hi = x.f >> 32; - const uint64_t v_lo = y.f & 0xFFFFFFFF; - const uint64_t v_hi = y.f >> 32; + const std::uint64_t u_lo = x.f & 0xFFFFFFFFu; + const std::uint64_t u_hi = x.f >> 32u; + const std::uint64_t v_lo = y.f & 0xFFFFFFFFu; + const std::uint64_t v_hi = y.f >> 32u; - const uint64_t p0 = u_lo * v_lo; - const uint64_t p1 = u_lo * v_hi; - const uint64_t p2 = u_hi * v_lo; - const uint64_t p3 = u_hi * v_hi; + const std::uint64_t p0 = u_lo * v_lo; + const std::uint64_t p1 = u_lo * v_hi; + const std::uint64_t p2 = u_hi * v_lo; + const std::uint64_t p3 = u_hi * v_hi; - const uint64_t p0_hi = p0 >> 32; - const uint64_t p1_lo = p1 & 0xFFFFFFFF; - const uint64_t p1_hi = p1 >> 32; - const uint64_t p2_lo = p2 & 0xFFFFFFFF; - const uint64_t p2_hi = p2 >> 32; + const std::uint64_t p0_hi = p0 >> 32u; + const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu; + const std::uint64_t p1_hi = p1 >> 32u; + const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu; + const std::uint64_t p2_hi = p2 >> 32u; - uint64_t Q = p0_hi + p1_lo + p2_lo; + std::uint64_t Q = p0_hi + p1_lo + p2_lo; // The full product might now be computed as // @@ -9922,9 +11031,9 @@ struct diyfp // f * 2^e // Effectively we only need to add the highest bit in p_lo to p_hi (and // Q_hi + 1 does not overflow). - Q += uint64_t{1} << (64 - 32 - 1); // round, ties up + Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up - const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32); + const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u); return {h, x.e + y.e + 64}; } @@ -9937,9 +11046,9 @@ struct diyfp // f * 2^e { assert(x.f != 0); - while ((x.f >> 63) == 0) + while ((x.f >> 63u) == 0) { - x.f <<= 1; + x.f <<= 1u; x.e--; } @@ -9993,15 +11102,15 @@ boundaries compute_boundaries(FloatType value) constexpr int kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit) constexpr int kBias = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1); constexpr int kMinExp = 1 - kBias; - constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + constexpr std::uint64_t kHiddenBit = std::uint64_t{1} << (kPrecision - 1); // = 2^(p-1) - using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type; + using bits_type = typename std::conditional<kPrecision == 24, std::uint32_t, std::uint64_t >::type; - const uint64_t bits = reinterpret_bits<bits_type>(value); - const uint64_t E = bits >> (kPrecision - 1); - const uint64_t F = bits & (kHiddenBit - 1); + const std::uint64_t bits = reinterpret_bits<bits_type>(value); + const std::uint64_t E = bits >> (kPrecision - 1); + const std::uint64_t F = bits & (kHiddenBit - 1); - const bool is_denormal = (E == 0); + const bool is_denormal = E == 0; const diyfp v = is_denormal ? diyfp(F, kMinExp) : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias); @@ -10027,7 +11136,7 @@ boundaries compute_boundaries(FloatType value) // -----------------+------+------+-------------+-------------+--- (B) // v- m- v m+ v+ - const bool lower_boundary_is_closer = (F == 0 and E > 1); + const bool lower_boundary_is_closer = F == 0 and E > 1; const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); const diyfp m_minus = lower_boundary_is_closer ? diyfp(4 * v.f - 1, v.e - 2) // (B) @@ -10102,7 +11211,7 @@ constexpr int kGamma = -32; struct cached_power // c = f * 2^e ~= 10^k { - uint64_t f; + std::uint64_t f; int e; int k; }; @@ -10166,91 +11275,92 @@ inline cached_power get_cached_power_for_binary_exponent(int e) // NB: // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. - constexpr int kCachedPowersSize = 79; constexpr int kCachedPowersMinDecExp = -300; constexpr int kCachedPowersDecStep = 8; - static constexpr cached_power kCachedPowers[] = - { - { 0xAB70FE17C79AC6CA, -1060, -300 }, - { 0xFF77B1FCBEBCDC4F, -1034, -292 }, - { 0xBE5691EF416BD60C, -1007, -284 }, - { 0x8DD01FAD907FFC3C, -980, -276 }, - { 0xD3515C2831559A83, -954, -268 }, - { 0x9D71AC8FADA6C9B5, -927, -260 }, - { 0xEA9C227723EE8BCB, -901, -252 }, - { 0xAECC49914078536D, -874, -244 }, - { 0x823C12795DB6CE57, -847, -236 }, - { 0xC21094364DFB5637, -821, -228 }, - { 0x9096EA6F3848984F, -794, -220 }, - { 0xD77485CB25823AC7, -768, -212 }, - { 0xA086CFCD97BF97F4, -741, -204 }, - { 0xEF340A98172AACE5, -715, -196 }, - { 0xB23867FB2A35B28E, -688, -188 }, - { 0x84C8D4DFD2C63F3B, -661, -180 }, - { 0xC5DD44271AD3CDBA, -635, -172 }, - { 0x936B9FCEBB25C996, -608, -164 }, - { 0xDBAC6C247D62A584, -582, -156 }, - { 0xA3AB66580D5FDAF6, -555, -148 }, - { 0xF3E2F893DEC3F126, -529, -140 }, - { 0xB5B5ADA8AAFF80B8, -502, -132 }, - { 0x87625F056C7C4A8B, -475, -124 }, - { 0xC9BCFF6034C13053, -449, -116 }, - { 0x964E858C91BA2655, -422, -108 }, - { 0xDFF9772470297EBD, -396, -100 }, - { 0xA6DFBD9FB8E5B88F, -369, -92 }, - { 0xF8A95FCF88747D94, -343, -84 }, - { 0xB94470938FA89BCF, -316, -76 }, - { 0x8A08F0F8BF0F156B, -289, -68 }, - { 0xCDB02555653131B6, -263, -60 }, - { 0x993FE2C6D07B7FAC, -236, -52 }, - { 0xE45C10C42A2B3B06, -210, -44 }, - { 0xAA242499697392D3, -183, -36 }, - { 0xFD87B5F28300CA0E, -157, -28 }, - { 0xBCE5086492111AEB, -130, -20 }, - { 0x8CBCCC096F5088CC, -103, -12 }, - { 0xD1B71758E219652C, -77, -4 }, - { 0x9C40000000000000, -50, 4 }, - { 0xE8D4A51000000000, -24, 12 }, - { 0xAD78EBC5AC620000, 3, 20 }, - { 0x813F3978F8940984, 30, 28 }, - { 0xC097CE7BC90715B3, 56, 36 }, - { 0x8F7E32CE7BEA5C70, 83, 44 }, - { 0xD5D238A4ABE98068, 109, 52 }, - { 0x9F4F2726179A2245, 136, 60 }, - { 0xED63A231D4C4FB27, 162, 68 }, - { 0xB0DE65388CC8ADA8, 189, 76 }, - { 0x83C7088E1AAB65DB, 216, 84 }, - { 0xC45D1DF942711D9A, 242, 92 }, - { 0x924D692CA61BE758, 269, 100 }, - { 0xDA01EE641A708DEA, 295, 108 }, - { 0xA26DA3999AEF774A, 322, 116 }, - { 0xF209787BB47D6B85, 348, 124 }, - { 0xB454E4A179DD1877, 375, 132 }, - { 0x865B86925B9BC5C2, 402, 140 }, - { 0xC83553C5C8965D3D, 428, 148 }, - { 0x952AB45CFA97A0B3, 455, 156 }, - { 0xDE469FBD99A05FE3, 481, 164 }, - { 0xA59BC234DB398C25, 508, 172 }, - { 0xF6C69A72A3989F5C, 534, 180 }, - { 0xB7DCBF5354E9BECE, 561, 188 }, - { 0x88FCF317F22241E2, 588, 196 }, - { 0xCC20CE9BD35C78A5, 614, 204 }, - { 0x98165AF37B2153DF, 641, 212 }, - { 0xE2A0B5DC971F303A, 667, 220 }, - { 0xA8D9D1535CE3B396, 694, 228 }, - { 0xFB9B7CD9A4A7443C, 720, 236 }, - { 0xBB764C4CA7A44410, 747, 244 }, - { 0x8BAB8EEFB6409C1A, 774, 252 }, - { 0xD01FEF10A657842C, 800, 260 }, - { 0x9B10A4E5E9913129, 827, 268 }, - { 0xE7109BFBA19C0C9D, 853, 276 }, - { 0xAC2820D9623BF429, 880, 284 }, - { 0x80444B5E7AA7CF85, 907, 292 }, - { 0xBF21E44003ACDD2D, 933, 300 }, - { 0x8E679C2F5E44FF8F, 960, 308 }, - { 0xD433179D9C8CB841, 986, 316 }, - { 0x9E19DB92B4E31BA9, 1013, 324 }, + static constexpr std::array<cached_power, 79> kCachedPowers = + { + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + } }; // This computation gives exactly the same results for k as @@ -10264,10 +11374,9 @@ inline cached_power get_cached_power_for_binary_exponent(int e) const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; assert(index >= 0); - assert(index < kCachedPowersSize); - static_cast<void>(kCachedPowersSize); // Fix warning. + assert(static_cast<std::size_t>(index) < kCachedPowers.size()); - const cached_power cached = kCachedPowers[index]; + const cached_power cached = kCachedPowers[static_cast<std::size_t>(index)]; assert(kAlpha <= cached.e + e + 64); assert(kGamma >= cached.e + e + 64); @@ -10278,7 +11387,7 @@ inline cached_power get_cached_power_for_binary_exponent(int e) For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. For n == 0, returns 1 and sets pow10 := 1. */ -inline int find_largest_pow10(const uint32_t n, uint32_t& pow10) +inline int find_largest_pow10(const std::uint32_t n, std::uint32_t& pow10) { // LCOV_EXCL_START if (n >= 1000000000) @@ -10334,8 +11443,8 @@ inline int find_largest_pow10(const uint32_t n, uint32_t& pow10) } } -inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta, - uint64_t rest, uint64_t ten_k) +inline void grisu2_round(char* buf, int len, std::uint64_t dist, std::uint64_t delta, + std::uint64_t rest, std::uint64_t ten_k) { assert(len >= 1); assert(dist <= delta); @@ -10396,8 +11505,8 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, assert(M_plus.e >= kAlpha); assert(M_plus.e <= kGamma); - uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) - uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + std::uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + std::uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): // @@ -10406,10 +11515,10 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, // = ((p1 ) * 2^-e + (p2 )) * 2^e // = p1 + p2 * 2^e - const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e); + const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e); - auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) - uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + auto p1 = static_cast<std::uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e // 1) // @@ -10417,7 +11526,7 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, assert(p1 > 0); - uint32_t pow10; + std::uint32_t pow10; const int k = find_largest_pow10(p1, pow10); // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) @@ -10445,8 +11554,8 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) // pow10 = 10^(n-1) <= p1 < 10^n // - const uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) - const uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) // // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) @@ -10471,7 +11580,7 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, // Note: // Since rest and delta share the same exponent e, it suffices to // compare the significands. - const uint64_t rest = (uint64_t{p1} << -one.e) + p2; + const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2; if (rest <= delta) { // V = buffer * 10^n, with M- <= V <= M+. @@ -10487,7 +11596,7 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, // // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e // - const uint64_t ten_n = uint64_t{pow10} << -one.e; + const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e; grisu2_round(buffer, length, dist, delta, rest, ten_n); return; @@ -10549,10 +11658,10 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e // - assert(p2 <= UINT64_MAX / 10); + assert(p2 <= (std::numeric_limits<std::uint64_t>::max)() / 10); p2 *= 10; - const uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e - const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e // // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) @@ -10592,7 +11701,7 @@ inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, // // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e // - const uint64_t ten_m = one.f; + const std::uint64_t ten_m = one.f; grisu2_round(buffer, length, dist, delta, p2, ten_m); // By construction this algorithm generates the shortest possible decimal @@ -10727,7 +11836,7 @@ inline char* append_exponent(char* buf, int e) *buf++ = '+'; } - auto k = static_cast<uint32_t>(e); + auto k = static_cast<std::uint32_t>(e); if (k < 10) { // Always print at least two digits in the exponent. @@ -10893,6 +12002,8 @@ char* to_chars(char* first, const char* last, FloatType value) } // namespace detail } // namespace nlohmann +// #include <nlohmann/detail/exceptions.hpp> + // #include <nlohmann/detail/macro_scope.hpp> // #include <nlohmann/detail/meta/cpp_future.hpp> @@ -10927,8 +12038,8 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - static constexpr uint8_t UTF8_ACCEPT = 0; - static constexpr uint8_t UTF8_REJECT = 1; + static constexpr std::uint8_t UTF8_ACCEPT = 0; + static constexpr std::uint8_t UTF8_REJECT = 1; public: /*! @@ -11160,6 +12271,9 @@ class serializer o->write_characters("null", 4); return; } + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } } @@ -11180,8 +12294,8 @@ class serializer */ void dump_escaped(const string_t& s, const bool ensure_ascii) { - uint32_t codepoint; - uint8_t state = UTF8_ACCEPT; + std::uint32_t codepoint; + std::uint8_t state = UTF8_ACCEPT; std::size_t bytes = 0; // number of bytes written to string_buffer // number of bytes written at the point of the last valid byte @@ -11256,14 +12370,14 @@ class serializer if (codepoint <= 0xFFFF) { (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", - static_cast<uint16_t>(codepoint)); + static_cast<std::uint16_t>(codepoint)); bytes += 6; } else { (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", - static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)), - static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF))); + static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)), + static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu))); bytes += 12; } } @@ -11337,6 +12451,16 @@ class serializer string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF'); string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD'); } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + bytes_after_last_accept = bytes; } @@ -11346,6 +12470,9 @@ class serializer state = UTF8_ACCEPT; break; } + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } break; } @@ -11380,7 +12507,7 @@ class serializer case error_handler_t::strict: { std::string sn(3, '\0'); - (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back())); + (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<std::uint8_t>(s.back())); JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); } @@ -11406,7 +12533,44 @@ class serializer } break; } + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + } + } + + /*! + @brief count digits + + Count the number of decimal (base 10) digits for an input unsigned integer. + + @param[in] x unsigned integer number to count its digits + @return number of decimal digits + */ + inline unsigned int count_digits(number_unsigned_t x) noexcept + { + unsigned int n_digits = 1; + for (;;) + { + if (x < 10) + { + return n_digits; + } + if (x < 100) + { + return n_digits + 1; + } + if (x < 1000) + { + return n_digits + 2; } + if (x < 10000) + { + return n_digits + 3; + } + x = x / 10000u; + n_digits += 4; } } @@ -11425,6 +12589,22 @@ class serializer int> = 0> void dump_integer(NumberType x) { + static constexpr std::array<std::array<char, 2>, 100> digits_to_99 + { + { + {{'0', '0'}}, {{'0', '1'}}, {{'0', '2'}}, {{'0', '3'}}, {{'0', '4'}}, {{'0', '5'}}, {{'0', '6'}}, {{'0', '7'}}, {{'0', '8'}}, {{'0', '9'}}, + {{'1', '0'}}, {{'1', '1'}}, {{'1', '2'}}, {{'1', '3'}}, {{'1', '4'}}, {{'1', '5'}}, {{'1', '6'}}, {{'1', '7'}}, {{'1', '8'}}, {{'1', '9'}}, + {{'2', '0'}}, {{'2', '1'}}, {{'2', '2'}}, {{'2', '3'}}, {{'2', '4'}}, {{'2', '5'}}, {{'2', '6'}}, {{'2', '7'}}, {{'2', '8'}}, {{'2', '9'}}, + {{'3', '0'}}, {{'3', '1'}}, {{'3', '2'}}, {{'3', '3'}}, {{'3', '4'}}, {{'3', '5'}}, {{'3', '6'}}, {{'3', '7'}}, {{'3', '8'}}, {{'3', '9'}}, + {{'4', '0'}}, {{'4', '1'}}, {{'4', '2'}}, {{'4', '3'}}, {{'4', '4'}}, {{'4', '5'}}, {{'4', '6'}}, {{'4', '7'}}, {{'4', '8'}}, {{'4', '9'}}, + {{'5', '0'}}, {{'5', '1'}}, {{'5', '2'}}, {{'5', '3'}}, {{'5', '4'}}, {{'5', '5'}}, {{'5', '6'}}, {{'5', '7'}}, {{'5', '8'}}, {{'5', '9'}}, + {{'6', '0'}}, {{'6', '1'}}, {{'6', '2'}}, {{'6', '3'}}, {{'6', '4'}}, {{'6', '5'}}, {{'6', '6'}}, {{'6', '7'}}, {{'6', '8'}}, {{'6', '9'}}, + {{'7', '0'}}, {{'7', '1'}}, {{'7', '2'}}, {{'7', '3'}}, {{'7', '4'}}, {{'7', '5'}}, {{'7', '6'}}, {{'7', '7'}}, {{'7', '8'}}, {{'7', '9'}}, + {{'8', '0'}}, {{'8', '1'}}, {{'8', '2'}}, {{'8', '3'}}, {{'8', '4'}}, {{'8', '5'}}, {{'8', '6'}}, {{'8', '7'}}, {{'8', '8'}}, {{'8', '9'}}, + {{'9', '0'}}, {{'9', '1'}}, {{'9', '2'}}, {{'9', '3'}}, {{'9', '4'}}, {{'9', '5'}}, {{'9', '6'}}, {{'9', '7'}}, {{'9', '8'}}, {{'9', '9'}}, + } + }; + // special case for "0" if (x == 0) { @@ -11432,28 +12612,57 @@ class serializer return; } - const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0); // see issue #755 - std::size_t i = 0; + // use a pointer to fill the buffer + auto buffer_ptr = number_buffer.begin(); + + const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not(x >= 0); // see issue #755 + number_unsigned_t abs_value; - while (x != 0) + unsigned int n_chars; + + if (is_negative) { - // spare 1 byte for '\0' - assert(i < number_buffer.size() - 1); + *buffer_ptr = '-'; + abs_value = static_cast<number_unsigned_t>(std::abs(static_cast<std::intmax_t>(x))); - const auto digit = std::labs(static_cast<long>(x % 10)); - number_buffer[i++] = static_cast<char>('0' + digit); - x /= 10; + // account one more byte for the minus sign + n_chars = 1 + count_digits(abs_value); + } + else + { + abs_value = static_cast<number_unsigned_t>(x); + n_chars = count_digits(abs_value); } - if (is_negative) + // spare 1 byte for '\0' + assert(n_chars < number_buffer.size() - 1); + + // jump to the end to generate the string from backward + // so we later avoid reversing the result + buffer_ptr += n_chars; + + // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu + // See: https://www.youtube.com/watch?v=o4-CwDo2zpg + while (abs_value >= 100) { - // make sure there is capacity for the '-' - assert(i < number_buffer.size() - 2); - number_buffer[i++] = '-'; + const auto digits_index = static_cast<unsigned>((abs_value % 100)); + abs_value /= 100; + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; } - std::reverse(number_buffer.begin(), number_buffer.begin() + i); - o->write_characters(number_buffer.data(), i); + if (abs_value >= 10) + { + const auto digits_index = static_cast<unsigned>(abs_value); + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + else + { + *(--buffer_ptr) = static_cast<char>('0' + abs_value); + } + + o->write_characters(number_buffer.data(), n_chars); } /*! @@ -11533,7 +12742,7 @@ class serializer std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1, [](char c) { - return (c == '.' or c == 'e'); + return c == '.' or c == 'e'; }); if (value_is_int_like) @@ -11563,9 +12772,9 @@ class serializer @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ */ - static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept + static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept { - static const std::array<uint8_t, 400> utf8d = + static const std::array<std::uint8_t, 400> utf8d = { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F @@ -11585,11 +12794,11 @@ class serializer } }; - const uint8_t type = utf8d[byte]; + const std::uint8_t type = utf8d[byte]; codep = (state != UTF8_ACCEPT) - ? (byte & 0x3fu) | (codep << 6) - : static_cast<uint32_t>(0xff >> type) & (byte); + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); state = utf8d[256u + state * 16u + type]; return state; @@ -11623,831 +12832,9 @@ class serializer } // namespace detail } // namespace nlohmann -// #include <nlohmann/detail/json_ref.hpp> - - -#include <initializer_list> -#include <utility> - -// #include <nlohmann/detail/meta/type_traits.hpp> - - -namespace nlohmann -{ -namespace detail -{ -template<typename BasicJsonType> -class json_ref -{ - public: - using value_type = BasicJsonType; - - json_ref(value_type&& value) - : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true) - {} - - json_ref(const value_type& value) - : value_ref(const_cast<value_type*>(&value)), is_rvalue(false) - {} - - json_ref(std::initializer_list<json_ref> init) - : owned_value(init), value_ref(&owned_value), is_rvalue(true) - {} - - template < - class... Args, - enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 > - json_ref(Args && ... args) - : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), - is_rvalue(true) {} - - // class should be movable only - json_ref(json_ref&&) = default; - json_ref(const json_ref&) = delete; - json_ref& operator=(const json_ref&) = delete; - json_ref& operator=(json_ref&&) = delete; - ~json_ref() = default; - - value_type moved_or_copied() const - { - if (is_rvalue) - { - return std::move(*value_ref); - } - return *value_ref; - } - - value_type const& operator*() const - { - return *static_cast<value_type const*>(value_ref); - } - - value_type const* operator->() const - { - return static_cast<value_type const*>(value_ref); - } - - private: - mutable value_type owned_value = nullptr; - value_type* value_ref = nullptr; - const bool is_rvalue; -}; -} // namespace detail -} // namespace nlohmann - -// #include <nlohmann/detail/json_pointer.hpp> - - -#include <cassert> // assert -#include <numeric> // accumulate -#include <string> // string -#include <vector> // vector - -// #include <nlohmann/detail/macro_scope.hpp> - -// #include <nlohmann/detail/exceptions.hpp> - // #include <nlohmann/detail/value_t.hpp> - -namespace nlohmann -{ -template<typename BasicJsonType> -class json_pointer -{ - // allow basic_json to access private members - NLOHMANN_BASIC_JSON_TPL_DECLARATION - friend class basic_json; - - public: - /*! - @brief create JSON pointer - - Create a JSON pointer according to the syntax described in - [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). - - @param[in] s string representing the JSON pointer; if omitted, the empty - string is assumed which references the whole JSON value - - @throw parse_error.107 if the given JSON pointer @a s is nonempty and does - not begin with a slash (`/`); see example below - - @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is - not followed by `0` (representing `~`) or `1` (representing `/`); see - example below - - @liveexample{The example shows the construction several valid JSON pointers - as well as the exceptional behavior.,json_pointer} - - @since version 2.0.0 - */ - explicit json_pointer(const std::string& s = "") - : reference_tokens(split(s)) - {} - - /*! - @brief return a string representation of the JSON pointer - - @invariant For each JSON pointer `ptr`, it holds: - @code {.cpp} - ptr == json_pointer(ptr.to_string()); - @endcode - - @return a string representation of the JSON pointer - - @liveexample{The example shows the result of `to_string`., - json_pointer__to_string} - - @since version 2.0.0 - */ - std::string to_string() const - { - return std::accumulate(reference_tokens.begin(), reference_tokens.end(), - std::string{}, - [](const std::string & a, const std::string & b) - { - return a + "/" + escape(b); - }); - } - - /// @copydoc to_string() - operator std::string() const - { - return to_string(); - } - - /*! - @param[in] s reference token to be converted into an array index - - @return integer representation of @a s - - @throw out_of_range.404 if string @a s could not be converted to an integer - */ - static int array_index(const std::string& s) - { - std::size_t processed_chars = 0; - const int res = std::stoi(s, &processed_chars); - - // check if the string was completely read - if (JSON_UNLIKELY(processed_chars != s.size())) - { - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); - } - - return res; - } - - private: - /*! - @brief remove and return last reference pointer - @throw out_of_range.405 if JSON pointer has no parent - */ - std::string pop_back() - { - if (JSON_UNLIKELY(is_root())) - { - JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); - } - - auto last = reference_tokens.back(); - reference_tokens.pop_back(); - return last; - } - - /// return whether pointer points to the root document - bool is_root() const noexcept - { - return reference_tokens.empty(); - } - - json_pointer top() const - { - if (JSON_UNLIKELY(is_root())) - { - JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); - } - - json_pointer result = *this; - result.reference_tokens = {reference_tokens[0]}; - return result; - } - - /*! - @brief create and return a reference to the pointed to value - - @complexity Linear in the number of reference tokens. - - @throw parse_error.109 if array index is not a number - @throw type_error.313 if value cannot be unflattened - */ - BasicJsonType& get_and_create(BasicJsonType& j) const - { - using size_type = typename BasicJsonType::size_type; - auto result = &j; - - // in case no reference tokens exist, return a reference to the JSON value - // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) - { - switch (result->m_type) - { - case detail::value_t::null: - { - if (reference_token == "0") - { - // start a new array if reference token is 0 - result = &result->operator[](0); - } - else - { - // start a new object otherwise - result = &result->operator[](reference_token); - } - break; - } - - case detail::value_t::object: - { - // create an entry in the object - result = &result->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - // create an entry in the array - JSON_TRY - { - result = &result->operator[](static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - /* - The following code is only reached if there exists a reference - token _and_ the current value is primitive. In this case, we have - an error situation, because primitive values may only occur as - single value; that is, with an empty list of reference tokens. - */ - default: - JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); - } - } - - return *result; - } - - /*! - @brief return a reference to the pointed to value - - @note This version does not throw if a value is not present, but tries to - create nested values instead. For instance, calling this function - with pointer `"/this/that"` on a null value is equivalent to calling - `operator[]("this").operator[]("that")` on that value, effectively - changing the null value to an object. - - @param[in] ptr a JSON value - - @return reference to the JSON value pointed to by the JSON pointer - - @complexity Linear in the length of the JSON pointer. - - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number - @throw out_of_range.404 if the JSON pointer can not be resolved - */ - BasicJsonType& get_unchecked(BasicJsonType* ptr) const - { - using size_type = typename BasicJsonType::size_type; - for (const auto& reference_token : reference_tokens) - { - // convert null values to arrays or objects before continuing - if (ptr->m_type == detail::value_t::null) - { - // check if reference token is a number - const bool nums = - std::all_of(reference_token.begin(), reference_token.end(), - [](const char x) - { - return (x >= '0' and x <= '9'); - }); - - // change value to array for numbers or "-" or to object otherwise - *ptr = (nums or reference_token == "-") - ? detail::value_t::array - : detail::value_t::object; - } - - switch (ptr->m_type) - { - case detail::value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - if (reference_token == "-") - { - // explicitly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); - } - else - { - // convert array index to number; unchecked access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; - } - - /*! - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number - @throw out_of_range.402 if the array index '-' is used - @throw out_of_range.404 if the JSON pointer can not be resolved - */ - BasicJsonType& get_checked(BasicJsonType* ptr) const - { - using size_type = typename BasicJsonType::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; - } - - /*! - @brief return a const reference to the pointed to value - - @param[in] ptr a JSON value - - @return const reference to the JSON value pointed to by the JSON - pointer - - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number - @throw out_of_range.402 if the array index '-' is used - @throw out_of_range.404 if the JSON pointer can not be resolved - */ - const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const - { - using size_type = typename BasicJsonType::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" cannot be used for const access - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // use unchecked array access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; - } - - /*! - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number - @throw out_of_range.402 if the array index '-' is used - @throw out_of_range.404 if the JSON pointer can not be resolved - */ - const BasicJsonType& get_checked(const BasicJsonType* ptr) const - { - using size_type = typename BasicJsonType::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; - } - - /*! - @brief split the string input to reference tokens - - @note This function is only called by the json_pointer constructor. - All exceptions below are documented there. - - @throw parse_error.107 if the pointer is not empty or begins with '/' - @throw parse_error.108 if character '~' is not followed by '0' or '1' - */ - static std::vector<std::string> split(const std::string& reference_string) - { - std::vector<std::string> result; - - // special case: empty reference string -> no reference tokens - if (reference_string.empty()) - { - return result; - } - - // check if nonempty reference string begins with slash - if (JSON_UNLIKELY(reference_string[0] != '/')) - { - JSON_THROW(detail::parse_error::create(107, 1, - "JSON pointer must be empty or begin with '/' - was: '" + - reference_string + "'")); - } - - // extract the reference tokens: - // - slash: position of the last read slash (or end of string) - // - start: position after the previous slash - for ( - // search for the first slash after the first character - std::size_t slash = reference_string.find_first_of('/', 1), - // set the beginning of the first reference token - start = 1; - // we can stop if start == 0 (if slash == std::string::npos) - start != 0; - // set the beginning of the next reference token - // (will eventually be 0 if slash == std::string::npos) - start = (slash == std::string::npos) ? 0 : slash + 1, - // find next slash - slash = reference_string.find_first_of('/', start)) - { - // use the text between the beginning of the reference token - // (start) and the last slash (slash). - auto reference_token = reference_string.substr(start, slash - start); - - // check reference tokens are properly escaped - for (std::size_t pos = reference_token.find_first_of('~'); - pos != std::string::npos; - pos = reference_token.find_first_of('~', pos + 1)) - { - assert(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (JSON_UNLIKELY(pos == reference_token.size() - 1 or - (reference_token[pos + 1] != '0' and - reference_token[pos + 1] != '1'))) - { - JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'")); - } - } - - // finally, store the reference token - unescape(reference_token); - result.push_back(reference_token); - } - - return result; - } - - /*! - @brief replace all occurrences of a substring by another string - - @param[in,out] s the string to manipulate; changed so that all - occurrences of @a f are replaced with @a t - @param[in] f the substring to replace with @a t - @param[in] t the string to replace @a f - - @pre The search string @a f must not be empty. **This precondition is - enforced with an assertion.** - - @since version 2.0.0 - */ - static void replace_substring(std::string& s, const std::string& f, - const std::string& t) - { - assert(not f.empty()); - for (auto pos = s.find(f); // find first occurrence of f - pos != std::string::npos; // make sure f was found - s.replace(pos, f.size(), t), // replace with t, and - pos = s.find(f, pos + t.size())) // find next occurrence of f - {} - } - - /// escape "~" to "~0" and "/" to "~1" - static std::string escape(std::string s) - { - replace_substring(s, "~", "~0"); - replace_substring(s, "/", "~1"); - return s; - } - - /// unescape "~1" to tilde and "~0" to slash (order is important!) - static void unescape(std::string& s) - { - replace_substring(s, "~1", "/"); - replace_substring(s, "~0", "~"); - } - - /*! - @param[in] reference_string the reference string to the current value - @param[in] value the value to consider - @param[in,out] result the result object to insert values to - - @note Empty objects or arrays are flattened to `null`. - */ - static void flatten(const std::string& reference_string, - const BasicJsonType& value, - BasicJsonType& result) - { - switch (value.m_type) - { - case detail::value_t::array: - { - if (value.m_value.array->empty()) - { - // flatten empty array as null - result[reference_string] = nullptr; - } - else - { - // iterate array and use index as reference string - for (std::size_t i = 0; i < value.m_value.array->size(); ++i) - { - flatten(reference_string + "/" + std::to_string(i), - value.m_value.array->operator[](i), result); - } - } - break; - } - - case detail::value_t::object: - { - if (value.m_value.object->empty()) - { - // flatten empty object as null - result[reference_string] = nullptr; - } - else - { - // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) - { - flatten(reference_string + "/" + escape(element.first), element.second, result); - } - } - break; - } - - default: - { - // add primitive value with its reference string - result[reference_string] = value; - break; - } - } - } - - /*! - @param[in] value flattened JSON - - @return unflattened JSON - - @throw parse_error.109 if array index is not a number - @throw type_error.314 if value is not an object - @throw type_error.315 if object values are not primitive - @throw type_error.313 if value cannot be unflattened - */ - static BasicJsonType - unflatten(const BasicJsonType& value) - { - if (JSON_UNLIKELY(not value.is_object())) - { - JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); - } - - BasicJsonType result; - - // iterate the JSON object values - for (const auto& element : *value.m_value.object) - { - if (JSON_UNLIKELY(not element.second.is_primitive())) - { - JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); - } - - // assign value to reference pointed to by JSON pointer; Note that if - // the JSON pointer is "" (i.e., points to the whole value), function - // get_and_create returns a reference to result itself. An assignment - // will then create a primitive value. - json_pointer(element.first).get_and_create(result) = element.second; - } - - return result; - } - - friend bool operator==(json_pointer const& lhs, - json_pointer const& rhs) noexcept - { - return (lhs.reference_tokens == rhs.reference_tokens); - } - - friend bool operator!=(json_pointer const& lhs, - json_pointer const& rhs) noexcept - { - return not (lhs == rhs); - } - - /// the reference tokens - std::vector<std::string> reference_tokens; -}; -} // namespace nlohmann - -// #include <nlohmann/adl_serializer.hpp> - - -#include <utility> - -// #include <nlohmann/detail/conversions/from_json.hpp> - -// #include <nlohmann/detail/conversions/to_json.hpp> - - -namespace nlohmann -{ - -template<typename, typename> -struct adl_serializer -{ - /*! - @brief convert a JSON value to any value type - - This function is usually called by the `get()` function of the - @ref basic_json class (either explicit or via conversion operators). - - @param[in] j JSON value to read from - @param[in,out] val value to write to - */ - template<typename BasicJsonType, typename ValueType> - static auto from_json(BasicJsonType&& j, ValueType& val) noexcept( - noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) - -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void()) - { - ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); - } - - /*! - @brief convert any value type to a JSON value - - This function is usually called by the constructors of the @ref basic_json - class. - - @param[in,out] j JSON value to write to - @param[in] val value to read from - */ - template <typename BasicJsonType, typename ValueType> - static auto to_json(BasicJsonType& j, ValueType&& val) noexcept( - noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) - -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void()) - { - ::nlohmann::to_json(j, std::forward<ValueType>(val)); - } -}; - -} // namespace nlohmann +// #include <nlohmann/json_fwd.hpp> /*! @@ -12697,6 +13084,7 @@ class basic_json @since 2.1.0 */ + JSON_NODISCARD static basic_json meta() { basic_json result; @@ -13329,7 +13717,7 @@ class basic_json object = nullptr; // silence warning, see #821 if (JSON_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.6.1")); // LCOV_EXCL_LINE } break; } @@ -13702,6 +14090,8 @@ class basic_json case value_t::discarded: m_type = value_t::discarded; break; + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } assert_invariant(); } @@ -13789,7 +14179,7 @@ class basic_json bool is_an_object = std::all_of(init.begin(), init.end(), [](const detail::json_ref<basic_json>& element_ref) { - return (element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string()); + return element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string(); }); // adjust type if type deduction is not wanted @@ -13869,6 +14259,7 @@ class basic_json @since version 1.0.0 */ + JSON_NODISCARD static basic_json array(initializer_list_t init = {}) { return basic_json(init, false, value_t::array); @@ -13912,6 +14303,7 @@ class basic_json @since version 1.0.0 */ + JSON_NODISCARD static basic_json object(initializer_list_t init = {}) { return basic_json(init, false, value_t::object); @@ -14472,7 +14864,7 @@ class basic_json */ constexpr bool is_null() const noexcept { - return (m_type == value_t::null); + return m_type == value_t::null; } /*! @@ -14494,7 +14886,7 @@ class basic_json */ constexpr bool is_boolean() const noexcept { - return (m_type == value_t::boolean); + return m_type == value_t::boolean; } /*! @@ -14553,7 +14945,7 @@ class basic_json */ constexpr bool is_number_integer() const noexcept { - return (m_type == value_t::number_integer or m_type == value_t::number_unsigned); + return m_type == value_t::number_integer or m_type == value_t::number_unsigned; } /*! @@ -14581,7 +14973,7 @@ class basic_json */ constexpr bool is_number_unsigned() const noexcept { - return (m_type == value_t::number_unsigned); + return m_type == value_t::number_unsigned; } /*! @@ -14609,7 +15001,7 @@ class basic_json */ constexpr bool is_number_float() const noexcept { - return (m_type == value_t::number_float); + return m_type == value_t::number_float; } /*! @@ -14631,7 +15023,7 @@ class basic_json */ constexpr bool is_object() const noexcept { - return (m_type == value_t::object); + return m_type == value_t::object; } /*! @@ -14653,7 +15045,7 @@ class basic_json */ constexpr bool is_array() const noexcept { - return (m_type == value_t::array); + return m_type == value_t::array; } /*! @@ -14675,7 +15067,7 @@ class basic_json */ constexpr bool is_string() const noexcept { - return (m_type == value_t::string); + return m_type == value_t::string; } /*! @@ -14702,7 +15094,7 @@ class basic_json */ constexpr bool is_discarded() const noexcept { - return (m_type == value_t::discarded); + return m_type == value_t::discarded; } /*! @@ -15236,7 +15628,7 @@ class basic_json #ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value -#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914 +#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) and _MSC_VER <= 1914)) and not std::is_same<ValueType, typename std::string_view>::value #endif #endif @@ -16274,6 +16666,8 @@ class basic_json @liveexample{The example shows how `find()` is used.,find__key_type} + @sa @ref contains(KeyT&&) const -- checks whether a key exists + @since version 1.0.0 */ template<typename KeyT> @@ -16334,6 +16728,36 @@ class basic_json return is_object() ? m_value.object->count(std::forward<KeyT>(key)) : 0; } + /*! + @brief check the existence of an element in a JSON object + + Check whether an element exists in a JSON object with key equivalent to + @a key. If the element is not found or the JSON value is not an object, + false is returned. + + @note This method always returns false when executed on a JSON type + that is not an object. + + @param[in] key key value to check its existence. + + @return true if an element with specified @a key exists. If no such + element with such key is found or the JSON value is not an object, + false is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The following code shows an example for `contains()`.,contains} + + @sa @ref find(KeyT&&) -- returns an iterator to an object element + + @since version 3.6.0 + */ + template<typename KeyT> + bool contains(KeyT&& key) const + { + return is_object() and m_value.object->find(std::forward<KeyT>(key)) != m_value.object->end(); + } + /// @} @@ -17118,7 +17542,8 @@ class basic_json // add element to array (move semantics) m_value.array->push_back(std::move(val)); - // invalidate object + // invalidate object: mark it null so we do not call the destructor + // cppcheck-suppress accessMoved val.m_type = value_t::null; } @@ -17910,28 +18335,28 @@ class basic_json switch (lhs_type) { case value_t::array: - return (*lhs.m_value.array == *rhs.m_value.array); + return *lhs.m_value.array == *rhs.m_value.array; case value_t::object: - return (*lhs.m_value.object == *rhs.m_value.object); + return *lhs.m_value.object == *rhs.m_value.object; case value_t::null: return true; case value_t::string: - return (*lhs.m_value.string == *rhs.m_value.string); + return *lhs.m_value.string == *rhs.m_value.string; case value_t::boolean: - return (lhs.m_value.boolean == rhs.m_value.boolean); + return lhs.m_value.boolean == rhs.m_value.boolean; case value_t::number_integer: - return (lhs.m_value.number_integer == rhs.m_value.number_integer); + return lhs.m_value.number_integer == rhs.m_value.number_integer; case value_t::number_unsigned: - return (lhs.m_value.number_unsigned == rhs.m_value.number_unsigned); + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; case value_t::number_float: - return (lhs.m_value.number_float == rhs.m_value.number_float); + return lhs.m_value.number_float == rhs.m_value.number_float; default: return false; @@ -17939,27 +18364,27 @@ class basic_json } else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) { - return (static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float); + return static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) { - return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer)); + return lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer); } else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) { - return (static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float); + return static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) { - return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned)); + return lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned); } else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) { - return (static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer); + return static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; } else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) { - return (lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned)); + return lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned); } return false; @@ -17973,7 +18398,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs == basic_json(rhs)); + return lhs == basic_json(rhs); } /*! @@ -17984,7 +18409,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) == rhs); + return basic_json(lhs) == rhs; } /*! @@ -18018,7 +18443,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs != basic_json(rhs)); + return lhs != basic_json(rhs); } /*! @@ -18029,7 +18454,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) != rhs); + return basic_json(lhs) != rhs; } /*! @@ -18068,6 +18493,8 @@ class basic_json switch (lhs_type) { case value_t::array: + // note parentheses are necessary, see + // https://github.com/nlohmann/json/issues/1530 return (*lhs.m_value.array) < (*rhs.m_value.array); case value_t::object: @@ -18134,7 +18561,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs < basic_json(rhs)); + return lhs < basic_json(rhs); } /*! @@ -18145,7 +18572,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) < rhs); + return basic_json(lhs) < rhs; } /*! @@ -18180,7 +18607,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs <= basic_json(rhs)); + return lhs <= basic_json(rhs); } /*! @@ -18191,7 +18618,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) <= rhs); + return basic_json(lhs) <= rhs; } /*! @@ -18226,7 +18653,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs > basic_json(rhs)); + return lhs > basic_json(rhs); } /*! @@ -18237,7 +18664,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) > rhs); + return basic_json(lhs) > rhs; } /*! @@ -18272,7 +18699,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs >= basic_json(rhs)); + return lhs >= basic_json(rhs); } /*! @@ -18283,7 +18710,7 @@ class basic_json std::is_scalar<ScalarType>::value, int>::type = 0> friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) >= rhs); + return basic_json(lhs) >= rhs; } /// @} @@ -18329,8 +18756,8 @@ class basic_json friend std::ostream& operator<<(std::ostream& o, const basic_json& j) { // read width member and use it as indentation parameter if nonzero - const bool pretty_print = (o.width() > 0); - const auto indentation = (pretty_print ? o.width() : 0); + const bool pretty_print = o.width() > 0; + const auto indentation = pretty_print ? o.width() : 0; // reset width to 0 for subsequent calls to this stream o.width(0); @@ -18386,9 +18813,6 @@ class basic_json @pre The container storage is contiguous. Violating this precondition yields undefined behavior. **This precondition is enforced with an assertion.** - @pre Each element of the container has a size of 1 byte. Violating this - precondition yields undefined behavior. **This precondition is enforced - with a static assertion.** @warning There is no way to enforce all preconditions at compile-time. If the function is called with a noncompliant container and with @@ -18402,7 +18826,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return result of the deserialization + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.101 if a parse error occurs; example: `""unexpected end of input; expected string literal""` @@ -18429,6 +18855,7 @@ class basic_json @since version 2.0.3 (contiguous containers) */ + JSON_NODISCARD static basic_json parse(detail::input_adapter&& i, const parser_callback_t cb = nullptr, const bool allow_exceptions = true) @@ -18466,9 +18893,6 @@ class basic_json @pre The container storage is contiguous. Violating this precondition yields undefined behavior. **This precondition is enforced with an assertion.** - @pre Each element of the container has a size of 1 byte. Violating this - precondition yields undefined behavior. **This precondition is enforced - with a static assertion.** @warning There is no way to enforce all preconditions at compile-time. If the function is called with a noncompliant container and with @@ -18505,13 +18929,9 @@ class basic_json const bool strict = true) { assert(sax); - switch (format) - { - case input_format_t::json: - return parser(std::move(i)).sax_parse(sax, strict); - default: - return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict); - } + return format == input_format_t::json + ? parser(std::move(i)).sax_parse(sax, strict) + : detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict); } /*! @@ -18544,7 +18964,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return result of the deserialization + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.101 in case of an unexpected token @throw parse_error.102 if to_unicode fails or surrogate error @@ -19175,7 +19597,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -19200,6 +19624,7 @@ class basic_json @a strict parameter since 3.0.0; added @a allow_exceptions parameter since 3.2.0 */ + JSON_NODISCARD static basic_json from_cbor(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19215,6 +19640,7 @@ class basic_json */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + JSON_NODISCARD static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19278,7 +19704,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -19305,6 +19733,7 @@ class basic_json @a strict parameter since 3.0.0; added @a allow_exceptions parameter since 3.2.0 */ + JSON_NODISCARD static basic_json from_msgpack(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19320,6 +19749,7 @@ class basic_json */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + JSON_NODISCARD static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19365,7 +19795,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -19389,6 +19821,7 @@ class basic_json @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0 */ + JSON_NODISCARD static basic_json from_ubjson(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19404,6 +19837,7 @@ class basic_json */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + JSON_NODISCARD static basic_json from_ubjson(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19454,7 +19888,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.114 if an unsupported BSON record type is encountered @@ -19472,6 +19908,7 @@ class basic_json @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the related UBJSON format */ + JSON_NODISCARD static basic_json from_bson(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19487,6 +19924,7 @@ class basic_json */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + JSON_NODISCARD static basic_json from_bson(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19822,63 +20260,59 @@ class basic_json const auto operation_add = [&result](json_pointer & ptr, basic_json val) { // adding to the root of the target document means replacing it - if (ptr.is_root()) + if (ptr.empty()) { result = val; + return; } - else + + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) { - // make sure the top element of the pointer exists - json_pointer top_pointer = ptr.top(); - if (top_pointer != ptr) + case value_t::null: + case value_t::object: { - result.at(top_pointer); + // use operator[] to add value + parent[last_path] = val; + break; } - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result[ptr]; - - switch (parent.m_type) + case value_t::array: { - case value_t::null: - case value_t::object: + if (last_path == "-") { - // use operator[] to add value - parent[last_path] = val; - break; + // special case: append to back + parent.push_back(val); } - - case value_t::array: + else { - if (last_path == "-") + const auto idx = json_pointer::array_index(last_path); + if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size())) { - // special case: append to back - parent.push_back(val); + // avoid undefined behavior + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } - else - { - const auto idx = json_pointer::array_index(last_path); - if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size())) - { - // avoid undefined behavior - JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); - } - // default case: insert add offset - parent.insert(parent.begin() + static_cast<difference_type>(idx), val); - } - break; + // default case: insert add offset + parent.insert(parent.begin() + static_cast<difference_type>(idx), val); } - - // LCOV_EXCL_START - default: - { - // if there exists a parent it cannot be primitive - assert(false); - } - // LCOV_EXCL_STOP + break; } + + // if there exists a parent it cannot be primitive + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } }; @@ -19886,7 +20320,8 @@ class basic_json const auto operation_remove = [&result](json_pointer & ptr) { // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); + const auto last_path = ptr.back(); + ptr.pop_back(); basic_json& parent = result.at(ptr); // remove child @@ -20033,7 +20468,7 @@ class basic_json break; } - case patch_operations::invalid: + default: { // op must be "add", "remove", "replace", "move", "copy", or // "test" @@ -20078,6 +20513,7 @@ class basic_json @since version 2.0.0 */ + JSON_NODISCARD static basic_json diff(const basic_json& source, const basic_json& target, const std::string& path = "") { @@ -20097,106 +20533,105 @@ class basic_json { {"op", "replace"}, {"path", path}, {"value", target} }); + return result; } - else + + switch (source.type()) { - switch (source.type()) + case value_t::array: { - case value_t::array: + // first pass: traverse common elements + std::size_t i = 0; + while (i < source.size() and i < target.size()) { - // first pass: traverse common elements - std::size_t i = 0; - while (i < source.size() and i < target.size()) - { - // recursive call to compare array values at index i - auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - ++i; - } + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } - // i now reached the end of at least one array - // in a second pass, traverse the remaining elements + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements - // remove my remaining elements - const auto end_index = static_cast<difference_type>(result.size()); - while (i < source.size()) + // remove my remaining elements + const auto end_index = static_cast<difference_type>(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( { - // add operations in reverse order to avoid invalid - // indices - result.insert(result.begin() + end_index, object( - { - {"op", "remove"}, - {"path", path + "/" + std::to_string(i)} - })); - ++i; - } + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } - // add other remaining elements - while (i < target.size()) + // add other remaining elements + while (i < target.size()) + { + result.push_back( { - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + std::to_string(i)}, - {"value", target[i]} - }); - ++i; - } - - break; + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} + }); + ++i; } - case value_t::object: + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.cbegin(); it != source.cend(); ++it) { - // first pass: traverse this object's elements - for (auto it = source.cbegin(); it != source.cend(); ++it) - { - // escape the key name to be used in a JSON patch - const auto key = json_pointer::escape(it.key()); + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); - if (target.find(it.key()) != target.end()) - { - // recursive call to compare object values at key it - auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - } - else - { - // found a key that is not in o -> remove it - result.push_back(object( - { - {"op", "remove"}, {"path", path + "/" + key} - })); - } + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); } - - // second pass: traverse other object's elements - for (auto it = target.cbegin(); it != target.cend(); ++it) + else { - if (source.find(it.key()) == source.end()) + // found a key that is not in o -> remove it + result.push_back(object( { - // found a key that is not in this -> add it - const auto key = json_pointer::escape(it.key()); - result.push_back( - { - {"op", "add"}, {"path", path + "/" + key}, - {"value", it.value()} - }); - } + {"op", "remove"}, {"path", path + "/" + key} + })); } - - break; } - default: + // second pass: traverse other object's elements + for (auto it = target.cbegin(); it != target.cend(); ++it) { - // both primitive type: replace value - result.push_back( + if (source.find(it.key()) == source.end()) { - {"op", "replace"}, {"path", path}, {"value", target} - }); - break; + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, {"path", path + "/" + key}, + {"value", it.value()} + }); + } } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + break; } } @@ -20397,10 +20832,11 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #undef JSON_LIKELY #undef JSON_UNLIKELY #undef JSON_DEPRECATED +#undef JSON_NODISCARD #undef JSON_HAS_CPP_14 #undef JSON_HAS_CPP_17 #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION #undef NLOHMANN_BASIC_JSON_TPL -#endif +#endif // INCLUDE_NLOHMANN_JSON_HPP_ |