diff options
34 files changed, 917 insertions, 159 deletions
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 1ca734e75..890310b3e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo)); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { Hash h(hash, parseHashType(algo)); - auto path = store()->makeFixedOutputPath(recursive, h, name); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->makeFixedOutputPath(method, h, name); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d04b4bbbf..157e8ddb4 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -31,7 +31,7 @@ readonly NIX_FIRST_BUILD_UID="30001" readonly NIX_ROOT="/nix" readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv") readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" diff --git a/scripts/install.in b/scripts/install.in index 6709f00d4..1d26c4ff0 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -36,7 +36,9 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" require_util tar "unpack the binary tarball" -require_util xz "unpack the binary tarball" +if [ "$(uname -s)" != "Darwin" ]; then + require_util xz "unpack the binary tarball" +fi echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f2c20dd68..82eb1582e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1669,7 +1669,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) else { auto p = settings.readOnlyMode ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first - : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); + : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); dstPath = store->printStorePath(p); srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0a4236da4..d458ab272 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -562,7 +562,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional<std::string> outputHash; std::string outputHashAlgo; - bool outputHashRecursive = false; + auto ingestionMethod = FileIngestionMethod::Flat; StringSet outputs; outputs.insert("out"); @@ -573,8 +573,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; + if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; + else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); }; @@ -721,11 +721,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); Hash h(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath), - (outputHashRecursive ? "r:" : "") + printHashType(h.type), - h.to_string(Base16, false))); + drv.outputs.insert_or_assign("out", DerivationOutput { + std::move(outPath), + (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") + + printHashType(h.type), + h.to_string(Base16, false), + }); } else { @@ -1039,7 +1042,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1070,12 +1073,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first - : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair)); + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else @@ -1096,7 +1099,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args if (args[0]->type != tLambda) throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1105,7 +1108,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Path path; string name; Value * filterFun = nullptr; - auto recursive = true; + auto method = FileIngestionMethod::Recursive; Hash expectedHash; for (auto & attr : *args[0]->attrs) { @@ -1121,7 +1124,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value state.forceValue(*attr.value, pos); filterFun = attr.value; } else if (n == "recursive") - recursive = state.forceBool(*attr.value, *attr.pos); + method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; else if (n == "sha256") expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else @@ -1132,7 +1135,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v); } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7c18cf67f..17cc60228 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -195,7 +195,7 @@ struct GitInput : Input return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); auto tree = Tree { .actualPath = store->printStorePath(storePath), @@ -347,7 +347,7 @@ struct GitInput : Input unpackTarfile(*source, tmpDir); } - auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 1d6571571..2e0d4bf4d 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -114,7 +114,7 @@ struct MercurialInput : Input return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); return {Tree { .actualPath = store->printStorePath(storePath), diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 695525b31..bf2b2a5ff 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -67,10 +67,10 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); store->addToStore(info, sink.s, NoRepair, NoCheckSigs); storePath = std::move(info.path); } @@ -141,7 +141,7 @@ Tree downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); } Attrs infoAttrs({ diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index 5788e94a1..b2d8cfc8d 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,7 +8,7 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); } } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3a2d84861..f13736c58 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { @@ -345,7 +345,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, h, name)); addToStore(info, sink.s, repair, CheckSigs, nullptr); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index aa13c1cb4..4ff5609ed 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -79,7 +79,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index a2b16f95c..f5c132a83 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2734,7 +2734,7 @@ struct RestrictedStore : public LocalFSStore { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2747,9 +2747,9 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { - auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); goal.addDependency(path); return path; } @@ -3692,10 +3692,10 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - bool recursive; Hash h; - i.second.parseHashInfo(recursive, h); + FileIngestionMethod outputHashMode; Hash h; + i.second.parseHashInfo(outputHashMode, h); - if (!recursive) { + if (outputHashMode == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3705,9 +3705,11 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + Hash h2 = outputHashMode == FileIngestionMethod::Recursive + ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); if (h != h2) { @@ -3736,7 +3738,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(recursive, h2); + ca = makeFixedOutputCA(outputHashMode, h2); } /* Get rid of all weird permissions. This also checks that @@ -3957,7 +3959,9 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); - auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); + auto used = recursive + ? cloneStorePathSet(getClosure(info.path).first) + : cloneStorePathSet(info.references); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8e9f9d71b..f1afdff69 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -355,20 +355,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddToStore: { - bool fixed, recursive; std::string s, baseName; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; + FileIngestionMethod method; + { + bool fixed, recursive; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + method = FileIngestionMethod::Recursive; + } } HashType hashAlgo = parseHashType(s); TeeSource savedNAR(from); RetrieveRegularNARSink savedRegular; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ @@ -380,7 +384,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + auto path = store->addToStoreFromDump( + method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + baseName, + method, + hashAlgo); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0067032af..c68e7b16b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,13 +9,13 @@ namespace nix { -void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const { - recursive = false; + recursive = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = true; + recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f010318ce..ea517a78c 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -25,7 +25,7 @@ struct DerivationOutput DerivationOutput(const DerivationOutput &) = default; DerivationOutput(DerivationOutput &&) = default; DerivationOutput & operator = (const DerivationOutput &) = default; - void parseHashInfo(bool & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; typedef std::map<string, DerivationOutput> DerivationOutputs; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 458266be0..af20d389b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -195,7 +195,7 @@ struct LegacySSHStore : public Store { unsupported("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override { unsupported("addToStore"); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ae7513ad8..80851b591 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -557,10 +557,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - bool recursive; Hash h; - out->second.parseHashInfo(recursive, h); + FileIngestionMethod method; Hash h; + out->second.parseHashInfo(method, h); - check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); + check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); } else { @@ -1043,11 +1043,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(method, h, name); addTempRoot(dstPath); @@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (recursive) { + if (method == FileIngestionMethod::Recursive) { StringSource source(dump); restorePath(realPath, source); } else @@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam ValidPathInfo info(dstPath.clone()); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(recursive, h); + info.ca = makeFixedOutputCA(method, h); registerValidPath(info); } @@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1111,12 +1111,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else sink.s = make_ref<std::string>(readFile(srcPath)); - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 16aeab0ad..c1e75390c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -149,7 +149,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; /* Like addToStore(), but the contents of the path are contained @@ -157,7 +157,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 186976855..f3ed6ca39 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -87,6 +87,11 @@ const size_t storePathHashLen = 32; // i.e. 160 bits /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; +enum struct FileIngestionMethod : uint8_t { + Flat = false, + Recursive = true +}; + struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8c55da268..5c36693e6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -484,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -492,10 +492,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, Path srcPath(absPath(_srcPath)); - conn->to << wopAddToStore << name - << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ - << (recursive ? 1 : 0) - << printHashType(hashAlgo); + conn->to + << wopAddToStore + << name + << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (method == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashAlgo); try { conn->to.written = 0; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f301a97d8..3c86b4524 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -65,7 +65,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 91ba64cfc..472595b5a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -172,19 +172,22 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - bool recursive, + FileIngestionMethod recursive, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive) { + if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); - return makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), name); + return makeStorePath("output:out", + hashString(htSHA256, + "fixed:out:" + + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + hash.to_string(Base16) + ":"), + name); } } @@ -201,10 +204,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const + const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(recursive, h, name), h); + Hash h = method == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); + return std::make_pair(makeFixedOutputPath(method, h, name), h); } @@ -767,8 +772,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - bool recursive = ca.compare(6, 2, "r:") == 0; - Hash hash(std::string(ca, recursive ? 8 : 6)); + FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { @@ -812,9 +817,11 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(bool recursive, const Hash & hash) +std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { - return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); + return "fixed:" + + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") + + hash.to_string(); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7832aed23..cd9e97ed9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -44,7 +44,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; - /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; @@ -348,7 +347,7 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(bool recursive, + StorePath makeFixedOutputPath(FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references = {}, bool hasSelfReference = false) const; @@ -360,7 +359,7 @@ public: store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive = true, + const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -464,12 +463,12 @@ public: The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); } @@ -852,7 +851,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo( /* Compute the content-addressability assertion (ValidPathInfo::ca) for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(bool recursive, const Hash & hash); +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); /* Split URI into protocol+hierarchy part and its parameter set. */ diff --git a/src/libutil/config.cc b/src/libutil/config.cc index f03e444ec..8fc700a2b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -65,60 +65,63 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool override res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - string contents = readFile(path); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector<string> tokens = tokenizeString<vector<string> >(line); - if (tokens.empty()) continue; +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } - if (tokens.size() < 2) + if (include) { + if (tokens.size() != 2) throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - - auto include = false; - auto ignoreMissing = false; - if (tokens[0] == "include") - include = true; - else if (tokens[0] == "!include") { - include = true; - ignoreMissing = true; + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); } + continue; + } - if (include) { - if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - auto p = absPath(tokens[1], dirOf(path)); - if (pathExists(p)) { - applyConfigFile(p); - } else if (!ignoreMissing) { - throw Error("file '%1%' included from '%2%' not found", p, path); - } - continue; - } + if (tokens[1] != "=") + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + string name = tokens[0]; - string name = tokens[0]; + vector<string>::iterator i = tokens.begin(); + advance(i, 2); - vector<string>::iterator i = tokens.begin(); - advance(i, 2); + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; +} - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow - }; +void AbstractConfig::applyConfigFile(const Path & path) +{ + try { + string contents = readFile(path); + applyConfig(contents, path); } catch (SysError &) { } } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 7ea78fdaf..5c7a70a2e 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -7,6 +7,38 @@ namespace nix { +/** + * The Config class provides Nix runtime configurations. + * + * What is a Configuration? + * A collection of uniquely named Settings. + * + * What is a Setting? + * Each property that you can set in a configuration corresponds to a + * `Setting`. A setting records value and description of a property + * with a default and optional aliases. + * + * A valid configuration consists of settings that are registered to a + * `Config` object instance: + * + * Config config; + * Setting<std::string> systemSetting{&config, "x86_64-linux", "system", "the current system"}; + * + * The above creates a `Config` object and registers a setting called "system" + * via the variable `systemSetting` with it. The setting defaults to the string + * "x86_64-linux", it's description is "the current system". All of the + * registered settings can then be accessed as shown below: + * + * std::map<std::string, Config::SettingInfo> settings; + * config.getSettings(settings); + * config["system"].description == "the current system" + * config["system"].value == "x86_64-linux" + * + * + * The above retrieves all currently known settings from the `Config` object + * and adds them to the `settings` map. + */ + class Args; class AbstractSetting; class JSONPlaceholder; @@ -23,6 +55,10 @@ protected: public: + /** + * Sets the value referenced by `name` to `value`. Returns true if the + * setting is known, false otherwise. + */ virtual bool set(const std::string & name, const std::string & value) = 0; struct SettingInfo @@ -31,18 +67,52 @@ public: std::string description; }; + /** + * Adds the currently known settings to the given result map `res`. + * - res: map to store settings in + * - overridenOnly: when set to true only overridden settings will be added to `res` + */ virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0; + /** + * Parses the configuration in `contents` and applies it + * - contents: configuration contents to be parsed and applied + * - path: location of the configuration file + */ + void applyConfig(const std::string & contents, const std::string & path = "<unknown>"); + + /** + * Applies a nix configuration file + * - path: the location of the config file to apply + */ void applyConfigFile(const Path & path); + /** + * Resets the `overridden` flag of all Settings + */ virtual void resetOverriden() = 0; + /** + * Outputs all settings to JSON + * - out: JSONObject to write the configuration to + */ virtual void toJSON(JSONObject & out) = 0; + /** + * Converts settings to `Args` to be used on the command line interface + * - args: args to write to + * - category: category of the settings + */ virtual void convertToArgs(Args & args, const std::string & category) = 0; + /** + * Logs a warning for each unregistered setting + */ void warnUnknownSettings(); + /** + * Re-applies all previously attempted changes to unknown settings + */ void reapplyUnknownSettings(); }; diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc new file mode 100644 index 000000000..74c59fd31 --- /dev/null +++ b/src/libutil/tests/config.cc @@ -0,0 +1,264 @@ +#include "json.hh" +#include "config.hh" +#include "args.hh" + +#include <sstream> +#include <gtest/gtest.h> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * Config + * --------------------------------------------------------------------------*/ + + TEST(Config, setUndefinedSetting) { + Config config; + ASSERT_EQ(config.set("undefined-key", "value"), false); + } + + TEST(Config, setDefinedSetting) { + Config config; + std::string value; + Setting<std::string> foo{&config, value, "name-of-the-setting", "description"}; + ASSERT_EQ(config.set("name-of-the-setting", "value"), true); + } + + TEST(Config, getDefinedSetting) { + Config config; + std::string value; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, ""); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedOverridenSettingNotSet) { + Config config; + std::string value; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ true); + const auto e = settings.find("name-of-the-setting"); + ASSERT_EQ(e, settings.end()); + } + + TEST(Config, getDefinedSettingSet1) { + Config config; + std::string value; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, value, "name-of-the-setting", "description"}; + + setting.assign("value"); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, "value"); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedSettingSet2) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_TRUE(config.set("name-of-the-setting", "value")); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto e = settings.find("name-of-the-setting"); + ASSERT_NE(e, settings.end()); + ASSERT_EQ(e->second.value, "value"); + ASSERT_EQ(e->second.description, "description"); + } + + TEST(Config, addSetting) { + class TestSetting : public AbstractSetting { + public: + TestSetting() : AbstractSetting("test", "test", {}) {} + void set(const std::string & value) {} + std::string to_string() const { return {}; } + }; + + Config config; + TestSetting setting; + + ASSERT_FALSE(config.set("test", "value")); + config.addSetting(&setting); + ASSERT_TRUE(config.set("test", "value")); + } + + TEST(Config, withInitialValue) { + const StringMap initials = { + { "key", "value" }, + }; + Config config(initials); + + { + std::map<std::string, Config::SettingInfo> settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings.find("key"), settings.end()); + } + + Setting<std::string> setting{&config, "default-value", "key", "description"}; + + { + std::map<std::string, Config::SettingInfo> settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings["key"].value, "value"); + } + } + + TEST(Config, resetOverriden) { + Config config; + config.resetOverriden(); + } + + TEST(Config, resetOverridenWithSetting) { + Config config; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + + { + std::map<std::string, Config::SettingInfo> settings; + + setting.set("foo"); + ASSERT_EQ(setting.get(), "foo"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + + { + std::map<std::string, Config::SettingInfo> settings; + + setting.override("bar"); + ASSERT_TRUE(setting.overriden); + ASSERT_EQ(setting.get(), "bar"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_FALSE(settings.empty()); + } + + { + std::map<std::string, Config::SettingInfo> settings; + + config.resetOverriden(); + ASSERT_FALSE(setting.overriden); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + } + + TEST(Config, toJSONOnEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + Config config; + config.toJSON(obj); + } + + ASSERT_EQ(out.str(), "{}"); + } + + TEST(Config, toJSONOnNonEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + setting.assign("value"); + + config.toJSON(obj); + } + ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#"); + } + + TEST(Config, setSettingAlias) { + Config config; + Setting<std::string> setting{&config, "", "some-int", "best number", { "another-int" }}; + ASSERT_TRUE(config.set("some-int", "1")); + ASSERT_EQ(setting.get(), "1"); + ASSERT_TRUE(config.set("another-int", "2")); + ASSERT_EQ(setting.get(), "2"); + ASSERT_TRUE(config.set("some-int", "3")); + ASSERT_EQ(setting.get(), "3"); + } + + /* FIXME: The reapplyUnknownSettings method doesn't seem to do anything + * useful (these days). Whenever we add a new setting to Config the + * unknown settings are always considered. In which case is this function + * actually useful? Is there some way to register a Setting without calling + * addSetting? */ + TEST(Config, DISABLED_reapplyUnknownSettings) { + Config config; + ASSERT_FALSE(config.set("name-of-the-setting", "unknownvalue")); + Setting<std::string> setting{&config, "default", "name-of-the-setting", "description"}; + ASSERT_EQ(setting.get(), "default"); + config.reapplyUnknownSettings(); + ASSERT_EQ(setting.get(), "unknownvalue"); + } + + TEST(Config, applyConfigEmpty) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + config.applyConfig(""); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigEmptyWithComment) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + config.applyConfig("# just a comment"); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigAssignment) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = value-from-file #useful comment\n" + "# name-of-the-setting = foo\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "value-from-file"); + } + + TEST(Config, applyConfigWithReassignedSetting) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = first-value\n" + "name-of-the-setting = second-value\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "second-value"); + } + + TEST(Config, applyConfigFailsOnMissingIncludes) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_THROW(config.applyConfig( + "name-of-the-setting = value-from-file\n" + "# name-of-the-setting = foo\n" + "include /nix/store/does/not/exist.nix" + ), Error); + } + + TEST(Config, applyConfigInvalidThrows) { + Config config; + ASSERT_THROW(config.applyConfig("value == key"), UsageError); + ASSERT_THROW(config.applyConfig("value "), UsageError); + } +} diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc new file mode 100644 index 000000000..7cb439817 --- /dev/null +++ b/src/libutil/tests/hash.cc @@ -0,0 +1,80 @@ +#include "hash.hh" +#include <gtest/gtest.h> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * hashString + * --------------------------------------------------------------------------*/ + + TEST(hashString, testKnownMD5Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s1 = ""; + auto hash = hashString(HashType::htMD5, s1); + ASSERT_EQ(hash.to_string(Base::Base16), "md5:d41d8cd98f00b204e9800998ecf8427e"); + } + + TEST(hashString, testKnownMD5Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s2 = "abc"; + auto hash = hashString(HashType::htMD5, s2); + ASSERT_EQ(hash.to_string(Base::Base16), "md5:900150983cd24fb0d6963f7d28e17f72"); + } + + TEST(hashString, testKnownSHA1Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abc"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + } + + TEST(hashString, testKnownSHA1Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + } + + TEST(hashString, testKnownSHA256Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + } + + TEST(hashString, testKnownSHA256Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + } + + TEST(hashString, testKnownSHA512Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" + "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" + "454d4423643ce80e2a9ac94fa54ca49f"); + } + + TEST(hashString, testKnownSHA512Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" + "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" + "c7d329eeb6dd26545e96e55b874be909"); + } + + TEST(hashString, hashingWithUnknownAlgoExits) { + auto s = "unknown"; + ASSERT_DEATH(hashString(HashType::htUnknown, s), ""); + } +} diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc new file mode 100644 index 000000000..dea73f53a --- /dev/null +++ b/src/libutil/tests/json.cc @@ -0,0 +1,193 @@ +#include "json.hh" +#include <gtest/gtest.h> +#include <sstream> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * toJSON + * --------------------------------------------------------------------------*/ + + TEST(toJSON, quotesCharPtr) { + const char* input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, quotesStdString) { + std::string input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, convertsNullptrtoNull) { + auto input = nullptr; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + TEST(toJSON, convertsNullToNull) { + const char* input = 0; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + + TEST(toJSON, convertsFloat) { + auto input = 1.024f; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsDouble) { + const double input = 1.024; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsBool) { + auto input = false; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "false"); + } + + TEST(toJSON, quotesTab) { + std::stringstream out; + toJSON(out, "\t"); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + TEST(toJSON, quotesNewline) { + std::stringstream out; + toJSON(out, "\n"); + + ASSERT_EQ(out.str(), "\"\\n\""); + } + + TEST(toJSON, quotesCreturn) { + std::stringstream out; + toJSON(out, "\r"); + + ASSERT_EQ(out.str(), "\"\\r\""); + } + + TEST(toJSON, quotesCreturnNewLine) { + std::stringstream out; + toJSON(out, "\r\n"); + + ASSERT_EQ(out.str(), "\"\\r\\n\""); + } + + TEST(toJSON, quotesDoublequotes) { + std::stringstream out; + toJSON(out, "\""); + + ASSERT_EQ(out.str(), "\"\\\"\""); + } + + TEST(toJSON, substringEscape) { + std::stringstream out; + const char *s = "foo\t"; + toJSON(out, s+3, s + strlen(s)); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + /* ---------------------------------------------------------------------------- + * JSONObject + * --------------------------------------------------------------------------*/ + + TEST(JSONObject, emptyObject) { + std::stringstream out; + { + JSONObject t(out); + } + ASSERT_EQ(out.str(), "{}"); + } + + TEST(JSONObject, objectWithList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithListIndent) { + std::stringstream out; + { + JSONObject t(out, true); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), +R"#({ + "list": [ + "element" + ] +})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("list"); + l.list().elem("element"); + } + + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndObject) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("object"); + l.object().attr("key", "value"); + } + + ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#"); + } + + /* ---------------------------------------------------------------------------- + * JSONList + * --------------------------------------------------------------------------*/ + + TEST(JSONList, empty) { + std::stringstream out; + { + JSONList l(out); + } + ASSERT_EQ(out.str(), R"#([])#"); + } + + TEST(JSONList, withElements) { + std::stringstream out; + { + JSONList l(out); + l.elem("one"); + l.object(); + l.placeholder().write("three"); + } + ASSERT_EQ(out.str(), R"#(["one",{},"three"])#"); + } +} + diff --git a/src/libutil/tests/xml-writer.cc b/src/libutil/tests/xml-writer.cc new file mode 100644 index 000000000..adcde25c9 --- /dev/null +++ b/src/libutil/tests/xml-writer.cc @@ -0,0 +1,105 @@ +#include "xml-writer.hh" +#include <gtest/gtest.h> +#include <sstream> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * XMLWriter + * --------------------------------------------------------------------------*/ + + TEST(XMLWriter, emptyObject) { + std::stringstream out; + { + XMLWriter t(false, out); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n"); + } + + TEST(XMLWriter, objectWithEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.openElement("foobar"); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\"></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithEmptyAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = {}; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithAttrsEscaping) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "<key>", "<value>" } + }; + t.openElement("foobar", attrs); + } + + // XXX: While "<value>" is escaped, "<key>" isn't which I think is a bug. + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar <key>=\"<value>\"></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithAttrsIndented) { + std::stringstream out; + { + XMLWriter t(true, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\">\n</foobar>\n"); + } + + TEST(XMLWriter, writeEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.writeEmptyElement("foobar"); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar />"); + } + + TEST(XMLWriter, writeEmptyElementWithAttributes) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.writeEmptyElement("foobar", attrs); + + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\" />"); + } + +} diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 748554b9c..5a686c8cd 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -159,7 +159,8 @@ static int _main(int argc, char * * argv) std::optional<StorePath> storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); - storePath = store->makeFixedOutputPath(unpack, expectedHash, name); + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(recursive, expectedHash, name); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -208,13 +209,15 @@ static int _main(int argc, char * * argv) if (expectedHash != Hash(ht) && expectedHash != hash) throw Error(format("hash mismatch for '%1%'") % uri); + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + /* Copy the file to the Nix store. FIXME: if RemoteStore implemented addToStoreFromDump() and downloadFile() supported a sink, we could stream the download directly into the Nix store. */ - storePath = store->addToStore(name, tmpFile, unpack, ht); + storePath = store->addToStore(name, tmpFile, recursive, ht); - assert(*storePath == store->makeFixedOutputPath(unpack, hash, name)); + assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); } stopProgressBar(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index fcc00175a..3a3060ad8 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - bool recursive = false; + auto recursive = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = true; + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.empty()) @@ -194,10 +194,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - bool recursive = false; + auto recursive = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = true; + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.size() != 3) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 1d298903b..4b4ba81cb 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,10 +45,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(true, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); if (!dryRun) store->addToStore(info, sink.s); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 366314227..0a7c343d0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,15 +9,14 @@ using namespace nix; struct CmdHash : Command { - enum Mode { mFile, mPath }; - Mode mode; + FileIngestionMethod mode; Base base = SRI; bool truncate = false; HashType ht = htSHA256; std::vector<std::string> paths; std::optional<std::string> modulus; - CmdHash(Mode mode) : mode(mode) + CmdHash(FileIngestionMethod mode) : mode(mode) { mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "base64", "print hash in base-64", &base, Base64); @@ -36,9 +35,14 @@ struct CmdHash : Command std::string description() override { - return mode == mFile - ? "print cryptographic hash of a regular file" - : "print cryptographic hash of the NAR serialisation of a path"; + const char* d; + switch (mode) { + case FileIngestionMethod::Flat: + d = "print cryptographic hash of a regular file"; + case FileIngestionMethod::Recursive: + d = "print cryptographic hash of the NAR serialisation of a path"; + }; + return d; } Category category() override { return catUtility; } @@ -53,10 +57,14 @@ struct CmdHash : Command else hashSink = std::make_unique<HashSink>(ht); - if (mode == mFile) + switch (mode) { + case FileIngestionMethod::Flat: readFile(path, *hashSink); - else + break; + case FileIngestionMethod::Recursive: dumpPath(path, *hashSink); + break; + } Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); @@ -65,8 +73,8 @@ struct CmdHash : Command } }; -static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(CmdHash::mFile); }); -static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(CmdHash::mPath); }); +static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); }); +static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); }); struct CmdToBase : Command { @@ -137,7 +145,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); cmd.ht = ht; cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 8803461f4..3e7ff544d 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -77,12 +77,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); info.references = std::move(references); if (hasSelfReference) info.references.insert(info.path.clone()); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(true, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); if (!json) printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); |