diff options
66 files changed, 751 insertions, 359 deletions
diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml index fb4bf819f..f01ab71b3 100644 --- a/doc/manual/advanced-topics/diff-hook.xml +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -70,7 +70,7 @@ path just built.</para> <screen> $ nix-build ./deterministic.nix -A stable -these derivations will be built: +this derivation will be built: /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable @@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... <screen> $ nix-build ./deterministic.nix -A unstable -these derivations will be built: +this derivation will be built: /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable @@ -193,7 +193,7 @@ repeat = 1 An example output of this configuration: <screen> $ nix-build ./test.nix -A unstable -these derivations will be built: +this derivation will be built: /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index acfe9e3cc..6cc286ee1 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh <screen> $ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' -these derivations will be built: +this derivation will be built: /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 2b95b6819..55f25d959 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -516,7 +516,7 @@ source: $ nix-env -f '<nixpkgs>' -iA hello --dry-run (dry run; not doing anything) installing ‘hello-2.10’ -these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): +this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 <replaceable>...</replaceable></screen> diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index e2b8fc867..cc52f5b4a 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -39,7 +39,7 @@ bundle.</para> <step><para>Set the environment variable and install Nix</para> <screen> $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -$ sh <(curl https://nixos.org/nix/install) +$ sh <(curl -L https://nixos.org/nix/install) </screen></step> <step><para>In the shell profile and rc files (for example, diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 8d548f0ea..64c7a37fb 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -12,7 +12,7 @@ </para> <screen> - $ sh <(curl https://nixos.org/nix/install) + $ sh <(curl -L https://nixos.org/nix/install) </screen> <para> @@ -39,7 +39,7 @@ To explicitly select a single-user installation on your system: <screen> - sh <(curl https://nixos.org/nix/install) --no-daemon + sh <(curl -L https://nixos.org/nix/install) --no-daemon </screen> </para> @@ -97,7 +97,7 @@ $ rm -rf /nix installation on your system: </para> - <screen>sh <(curl https://nixos.org/nix/install) --daemon</screen> + <screen>sh <(curl -L https://nixos.org/nix/install) --daemon</screen> <para> The multi-user installation of Nix will create build users between @@ -178,7 +178,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist is a bit of a misnomer). To use this approach, just install Nix with: </para> - <screen>$ sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen> + <screen>$ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen> <para> If you don't like the sound of this, you'll want to weigh the @@ -429,7 +429,7 @@ LABEL=Nix\040Store /nix apfs rw,nobrowse NixOS.org installation script: <screen> - sh <(curl https://nixos.org/nix/install) + sh <(curl -L https://nixos.org/nix/install) </screen> </para> diff --git a/doc/manual/introduction/quick-start.xml b/doc/manual/introduction/quick-start.xml index 1ce6c8d50..1992c14ed 100644 --- a/doc/manual/introduction/quick-start.xml +++ b/doc/manual/introduction/quick-start.xml @@ -15,7 +15,7 @@ to subsequent chapters.</para> <step><para>Install single-user Nix by running the following: <screen> -$ bash <(curl https://nixos.org/nix/install) +$ bash <(curl -L https://nixos.org/nix/install) </screen> This will install Nix in <filename>/nix</filename>. The install script diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 157e8ddb4..00c9d540b 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -526,7 +526,7 @@ This script is going to call sudo a lot. Normally, it would show you exactly what commands it is running and why. However, the script is run in a headless fashion, like this: - $ curl https://nixos.org/nix/install | sh + $ curl -L https://nixos.org/nix/install | sh or maybe in a CI pipeline. Because of that, we're going to skip the verbose output in the interest of brevity. @@ -534,7 +534,7 @@ verbose output in the interest of brevity. If you would like to see the output, try like this: - $ curl -o install-nix https://nixos.org/nix/install + $ curl -L -o install-nix https://nixos.org/nix/install $ sh ./install-nix EOF diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 826ca8b8c..5824c2217 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -113,7 +113,7 @@ if [ "$(uname -s)" = "Darwin" ]; then ( echo "" echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." - echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." + echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." echo "See https://nixos.org/nix/manual/#sect-macos-installation" echo "" ) >&2 diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 330df0ab6..000000000 --- a/shell.nix +++ /dev/null @@ -1,3 +0,0 @@ -(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) { - src = ./.; -}).shellNix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 06e06c011..5db380f88 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -348,10 +348,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) + , sRecurseForDerivations(symbols.create("recurseForDerivations")) , sDescription(symbols.create("description")) , sSelf(symbols.create("self")) , sEpsilon(symbols.create("")) - , sRecurseForDerivations(symbols.create("recurseForDerivations")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9cf0030f9..be34f3f32 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,7 +18,7 @@ namespace nix { class Store; class EvalState; -struct StorePath; +class StorePath; enum RepairFlag : bool; @@ -75,7 +75,8 @@ public: sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sOutputHash, sOutputHashAlgo, sOutputHashMode, - sDescription, sSelf, sEpsilon, sRecurseForDerivations; + sRecurseForDerivations, + sDescription, sSelf, sEpsilon; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index a4937e722..9055f59a1 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,7 +1,7 @@ #include "get-drvs.hh" #include "util.hh" #include "eval-inline.hh" -#include "derivations.hh" +#include "store-api.hh" #include <cstring> #include <regex> diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2f1a41a64..3830d8107 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -769,17 +769,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .nixCode = NixCode { .errPos = posDrvName } }); - HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); - + std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - std::move(outPath), - (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") - + printHashType(h.type), - h.to_string(Base16, false), + .path = std::move(outPath), + .hash = FixedOutputHash { + .method = ingestionMethod, + .hash = std::move(h), + }, }); } @@ -793,7 +793,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, - DerivationOutput { StorePath::dummy, "", "" }); + DerivationOutput { + .path = StorePath::dummy, + .hash = std::optional<FixedOutputHash> {}, + }); } Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); @@ -802,7 +805,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeOutputPath(i, h, drvName); if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, - DerivationOutput { std::move(outPath), "", "" }); + DerivationOutput { + .path = std::move(outPath), + .hash = std::optional<FixedOutputHash>(), + }); } } @@ -999,8 +1005,8 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { string type = state.forceStringNoCtx(*args[0], pos); - HashType ht = parseHashType(type); - if (ht == htUnknown) + std::optional<HashType> ht = parseHashType(type); + if (!ht) throw Error({ .hint = hintfmt("unknown hash type '%1%'", type), .nixCode = NixCode { .errPos = pos } @@ -1009,7 +1015,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va PathSet context; // discarded Path p = state.coerceToPath(pos, *args[1], context); - mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); + mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context); } /* Read a directory (without . or ..) */ @@ -1936,8 +1942,8 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) { string type = state.forceStringNoCtx(*args[0], pos); - HashType ht = parseHashType(type); - if (ht == htUnknown) + std::optional<HashType> ht = parseHashType(type); + if (!ht) throw Error({ .hint = hintfmt("unknown hash type '%1%'", type), .nixCode = NixCode { .errPos = pos } @@ -1946,7 +1952,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, PathSet context; // discarded string s = state.forceString(*args[1], context, pos); - mkString(v, hashString(ht, s).to_string(Base16, false), context); + mkString(v, hashString(*ht, s).to_string(Base16, false), context); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index efa2e9576..301e8c5dd 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -1,6 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" -#include "derivations.hh" +#include "store-api.hh" namespace nix { diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 20c1e1093..55158cece 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -70,7 +70,10 @@ DownloadFileResult downloadFile( ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Flat, + .hash = hash, + }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index dc6d5e413..1cb422967 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -48,7 +48,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild, unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) { if (!willBuild.empty()) { - printMsg(lvl, "these derivations will be built:"); + if (willBuild.size() == 1) + printMsg(lvl, fmt("this derivation will be built:")); + else + printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size())); auto sorted = store->topoSortPaths(willBuild); reverse(sorted.begin(), sorted.end()); for (auto & i : sorted) @@ -56,9 +59,18 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild, } if (!willSubstitute.empty()) { - printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", - downloadSize / (1024.0 * 1024.0), - narSize / (1024.0 * 1024.0))); + const float downloadSizeMiB = downloadSize / (1024.f * 1024.f); + const float narSizeMiB = narSize / (1024.f * 1024.f); + if (willSubstitute.size() == 1) { + printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", + downloadSizeMiB, + narSizeMiB)); + } else { + printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", + willSubstitute.size(), + downloadSizeMiB, + narSizeMiB)); + } for (auto & i : willSubstitute) printMsg(lvl, fmt(" %s", store->printStorePath(i))); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f8eff508c..9f52ddafa 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -388,8 +388,6 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe narInfo->sigs.insert(sigs.begin(), sigs.end()); - auto narInfoFile = narInfoFileFor(narInfo->path); - writeNarInfo(narInfo); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 53a0958aa..0c25897f8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -86,7 +86,7 @@ struct HookInstance; /* A pointer to a goal. */ -class Goal; +struct Goal; class DerivationGoal; typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; @@ -1195,6 +1195,12 @@ void DerivationGoal::haveDerivation() parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + if (parsedDrv->contentAddressed()) { + settings.requireExperimentalFeature("ca-derivations"); + throw Error("ca-derivations isn't implemented yet"); + } + + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -3708,14 +3714,11 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::string ca; + std::optional<ContentAddress> ca; if (fixedOutput) { - FileIngestionMethod outputHashMode; Hash h; - i.second.parseHashInfo(outputHashMode, h); - - if (outputHashMode == FileIngestionMethod::Flat) { + if (i.second.hash->method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3726,20 +3729,22 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = outputHashMode == FileIngestionMethod::Recursive - ? hashPath(h.type, actualPath).first - : hashFile(h.type, actualPath); + Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive + ? hashPath(*i.second.hash->hash.type, actualPath).first + : hashFile(*i.second.hash->hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); - if (h != h2) { + if (i.second.hash->hash != h2) { /* Throw an error after registering the path as valid. */ worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); + worker.store.printStorePath(dest), + i.second.hash->hash.to_string(SRI, true), + h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3759,7 +3764,10 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(outputHashMode, h2); + ca = FixedOutputHash { + .method = i.second.hash->method, + .hash = h2, + }; } /* Get rid of all weird permissions. This also checks that @@ -3832,7 +3840,10 @@ void DerivationGoal::registerOutputs() info.ca = ca; worker.store.signPathInfo(info); - if (!info.references.empty()) info.ca.clear(); + if (!info.references.empty()) { + // FIXME don't we have an experimental feature for fixed output with references? + info.ca = {}; + } infos.emplace(i.first, std::move(info)); } @@ -4992,7 +5003,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 2048f8f87..1cfe4a46a 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto ht = parseHashType(getAttr("outputHashAlgo")); + auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc new file mode 100644 index 000000000..3d753836f --- /dev/null +++ b/src/libstore/content-address.cc @@ -0,0 +1,85 @@ +#include "content-address.hh" + +namespace nix { + +std::string FixedOutputHash::printMethodAlgo() const { + return makeFileIngestionPrefix(method) + printHashType(*hash.type); +} + +std::string makeFileIngestionPrefix(const FileIngestionMethod m) { + switch (m) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + default: + throw Error("impossible, caught both cases"); + } +} + +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) +{ + return "fixed:" + + makeFileIngestionPrefix(method) + + hash.to_string(Base32, true); +} + +// FIXME Put this somewhere? +template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; +template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; + +std::string renderContentAddress(ContentAddress ca) { + return std::visit(overloaded { + [](TextHash th) { + return "text:" + th.hash.to_string(Base32, true); + }, + [](FixedOutputHash fsh) { + return makeFixedOutputCA(fsh.method, fsh.hash); + } + }, ca); +} + +ContentAddress parseContentAddress(std::string_view rawCa) { + auto prefixSeparator = rawCa.find(':'); + if (prefixSeparator != string::npos) { + auto prefix = string(rawCa, 0, prefixSeparator); + if (prefix == "text") { + auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); + Hash hash = Hash(string(hashTypeAndHash)); + if (*hash.type != htSHA256) { + throw Error("parseContentAddress: the text hash should have type SHA256"); + } + return TextHash { hash }; + } else if (prefix == "fixed") { + // This has to be an inverse of makeFixedOutputCA + auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); + if (methodAndHash.substr(0,2) == "r:") { + std::string_view hashRaw = methodAndHash.substr(2,string::npos); + return FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = Hash(string(hashRaw)), + }; + } else { + std::string_view hashRaw = methodAndHash; + return FixedOutputHash { + .method = FileIngestionMethod::Flat, + .hash = Hash(string(hashRaw)), + }; + } + } else { + throw Error("parseContentAddress: format not recognized; has to be text or fixed"); + } + } else { + throw Error("Not a content address because it lacks an appropriate prefix"); + } +}; + +std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt); +}; + +std::string renderContentAddress(std::optional<ContentAddress> ca) { + return ca ? renderContentAddress(*ca) : ""; +} + +} diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh new file mode 100644 index 000000000..ba4797f5b --- /dev/null +++ b/src/libstore/content-address.hh @@ -0,0 +1,56 @@ +#pragma once + +#include <variant> +#include "hash.hh" + +namespace nix { + +enum struct FileIngestionMethod : uint8_t { + Flat = false, + Recursive = true +}; + +struct TextHash { + Hash hash; +}; + +/// Pair of a hash, and how the file system was ingested +struct FixedOutputHash { + FileIngestionMethod method; + Hash hash; + std::string printMethodAlgo() const; +}; + +/* + We've accumulated several types of content-addressed paths over the years; + fixed-output derivations support multiple hash algorithms and serialisation + methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: + + * ‘text:sha256:<sha256 hash of file contents>’: For paths + computed by makeTextPath() / addTextToStore(). + + * ‘fixed:<r?>:<ht>:<h>’: For paths computed by + makeFixedOutputPath() / addToStore(). +*/ +typedef std::variant< + TextHash, // for paths computed by makeTextPath() / addTextToStore + FixedOutputHash // for path computed by makeFixedOutputPath +> ContentAddress; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); + +/* Compute the content-addressability assertion (ValidPathInfo::ca) + for paths created by makeFixedOutputPath() / addToStore(). */ +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); + +std::string renderContentAddress(ContentAddress ca); + +std::string renderContentAddress(std::optional<ContentAddress> ca); + +ContentAddress parseContentAddress(std::string_view rawCa); + +std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); + +} diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..842aef20c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -652,7 +652,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << info->ca; + << renderContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -710,7 +710,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, info.references = readStorePaths<StorePathSet>(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); - from >> info.ca >> repair >> dontCheckSigs; + info.ca = parseContentAddressOpt(readString(from)); + from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b95f7bfdc..42551ef6b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,25 +8,6 @@ namespace nix { - -void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const -{ - recursive = FileIngestionMethod::Flat; - string algo = hashAlgo; - - if (string(algo, 0, 2) == "r:") { - recursive = FileIngestionMethod::Recursive; - algo = string(algo, 2); - } - - HashType hashType = parseHashType(algo); - if (hashType == htUnknown) - throw Error("unknown hash algorithm '%s'", algo); - - hash = Hash(this->hash, hashType); -} - - const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); @@ -120,6 +101,34 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } +static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) +{ + expect(str, ","); auto path = store.parseStorePath(parsePath(str)); + expect(str, ","); auto hashAlgo = parseString(str); + expect(str, ","); const auto hash = parseString(str); + expect(str, ")"); + + std::optional<FixedOutputHash> fsh; + if (hashAlgo != "") { + auto method = FileIngestionMethod::Flat; + if (string(hashAlgo, 0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = string(hashAlgo, 2); + } + const HashType hashType = parseHashType(hashAlgo); + fsh = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }; + } + + return DerivationOutput { + .path = std::move(path), + .hash = std::move(fsh), + }; +} + + static Derivation parseDerivation(const Store & store, const string & s) { Derivation drv; @@ -129,15 +138,8 @@ static Derivation parseDerivation(const Store & store, const string & s) /* Parse the list of outputs. */ while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - expect(str, ","); auto path = store.parseStorePath(parsePath(str)); - expect(str, ","); auto hashAlgo = parseString(str); - expect(str, ","); auto hash = parseString(str); - expect(str, ")"); - drv.outputs.emplace(id, DerivationOutput { - .path = std::move(path), - .hashAlgo = std::move(hashAlgo), - .hash = std::move(hash) - }); + auto output = parseDerivationOutput(store, str); + drv.outputs.emplace(std::move(id), std::move(output)); } /* Parse the list of input derivations. */ @@ -263,8 +265,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs, if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hashAlgo); - s += ','; printUnquotedString(s, i.second.hash); + s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); + s += ','; printUnquotedString(s, + i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); s += ')'; } @@ -320,7 +323,7 @@ bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash != ""; + outputs.begin()->second.hash; } @@ -353,8 +356,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" - + i->second.hashAlgo + ":" - + i->second.hash + ":" + + i->second.hash->printMethodAlgo() + ":" + + i->second.hash->hash.to_string(Base16, false) + ":" + store.printStorePath(i->second.path)); } @@ -397,6 +400,31 @@ StorePathSet BasicDerivation::outputPaths() const return paths; } +static DerivationOutput readDerivationOutput(Source & in, const Store & store) +{ + auto path = store.parseStorePath(readString(in)); + auto hashAlgo = readString(in); + auto hash = readString(in); + + std::optional<FixedOutputHash> fsh; + if (hashAlgo != "") { + auto method = FileIngestionMethod::Flat; + if (string(hashAlgo, 0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = string(hashAlgo, 2); + } + auto hashType = parseHashType(hashAlgo); + fsh = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }; + } + + return DerivationOutput { + .path = std::move(path), + .hash = std::move(fsh), + }; +} StringSet BasicDerivation::outputNames() const { @@ -413,14 +441,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) auto nr = readNum<size_t>(in); for (size_t n = 0; n < nr; n++) { auto name = readString(in); - auto path = store.parseStorePath(readString(in)); - auto hashAlgo = readString(in); - auto hash = readString(in); - drv.outputs.emplace(name, DerivationOutput { - .path = std::move(path), - .hashAlgo = std::move(hashAlgo), - .hash = std::move(hash) - }); + auto output = readDerivationOutput(in, store); + drv.outputs.emplace(std::move(name), std::move(output)); } drv.inputSrcs = readStorePaths<StorePathSet>(store, in); @@ -441,8 +463,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) { out << drv.outputs.size(); - for (auto & i : drv.outputs) - out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash; + for (auto & i : drv.outputs) { + out << i.first + << store.printStorePath(i.second.path); + if (i.second.hash) { + out << i.second.hash->printMethodAlgo() + << i.second.hash->hash.to_string(Base16, false); + } else { + out << "" << ""; + } + } writeStorePaths(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d349c6d4d..68c53c1ff 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,8 +1,9 @@ #pragma once +#include "path.hh" #include "types.hh" #include "hash.hh" -#include "store-api.hh" +#include "content-address.hh" #include <map> @@ -15,9 +16,7 @@ namespace nix { struct DerivationOutput { StorePath path; - std::string hashAlgo; /* hash used for expected hash computation */ - std::string hash; /* expected hash, may be null */ - void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; + std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */ }; typedef std::map<string, DerivationOutput> DerivationOutputs; @@ -70,6 +69,7 @@ struct Derivation : BasicDerivation class Store; +enum RepairFlag : bool { NoRepair = false, Repair = true }; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref<Store> store, diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cb9da027d..57b7e9590 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -55,7 +55,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cd15cde62..beb508e67 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -73,6 +73,7 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -99,7 +100,7 @@ struct curlFileTransfer : public FileTransfer /* Only write data to the sink if this is a successful response. */ - if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + if (successfulStatuses.count(httpStatus)) { writtenToSink += len; this->request.dataCallback((char *) data, len); } @@ -356,8 +357,7 @@ struct curlFileTransfer : public FileTransfer if (writeException) failEx(writeException); - else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */)) + else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { result.cached = httpStatus == 304; act.progress(result.bodySize, result.bodySize); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 45c70fad6..5657aa593 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -114,7 +114,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - conn->from >> info->ca; + info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings<StringSet>(conn->from); } @@ -146,7 +146,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << info.ca; + << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2ddb43cb7..5b462c5b3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - FileIngestionMethod method; Hash h; - out->second.parseHashInfo(method, h); - - check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); + check( + makeFixedOutputPath( + out->second.hash->method, + out->second.hash->hash, + drvName), + out->second.path, "out"); } else { @@ -578,7 +580,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { - if (info.ca != "" && !info.isContentAddressed(*this)) + if (info.ca.has_value() && !info.isContentAddressed(*this)) throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", printStorePath(info.path)); @@ -590,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) + (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = state.db.getLastInsertedRowId(); @@ -664,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString<StringSet>(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = s; + if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -687,7 +689,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) + (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } @@ -983,15 +985,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); - if (info.ca != "" && - !((hasPrefix(info.ca, "text:") && !info.references.count(info.path)) - || info.references.empty())) + // text hashing has long been allowed to have non-self-references because it is used for drv files. + bool refersToSelf = info.references.count(info.path) > 0; + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr<AbstractHashSink> hashSink; - if (info.ca == "" || !info.references.count(info.path)) + if (!info.ca.has_value() || !info.references.count(info.path)) hashSink = std::make_unique<HashSink>(htSHA256); else hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); @@ -1077,7 +1079,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam ValidPathInfo info(dstPath); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(method, h); + info.ca = FixedOutputHash { .method = method, .hash = h }; registerValidPath(info); } @@ -1141,7 +1143,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = references; - info.ca = "text:" + hash.to_string(Base32, true); + info.ca = TextHash { .hash = hash }; registerValidPath(info); } @@ -1252,10 +1254,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr<AbstractHashSink> hashSink; - if (info->ca == "" || !info->references.count(info->path)) - hashSink = std::make_unique<HashSink>(info->narHash.type); + if (!info->ca || !info->references.count(info->path)) + hashSink = std::make_unique<HashSink>(*info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 552970248..012dea6ea 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = queryNAR.getStr(11); + narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ public: (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (info->ca) + (renderContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index bb4448c90..04550ed97 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -67,8 +67,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!ca.empty()) corrupt(); - ca = value; + if (ca) corrupt(); + // FIXME: allow blank ca or require skipping field? + ca = parseContentAddressOpt(value); } pos = eol + 1; @@ -104,8 +105,8 @@ std::string NarInfo::to_string(const Store & store) const for (auto sig : sigs) res += "Sig: " + sig + "\n"; - if (!ca.empty()) - res += "CA: " + ca + "\n"; + if (ca) + res += "CA: " + renderContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 24f848e46..c7797b730 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -117,4 +117,9 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } +bool ParsedDerivation::contentAddressed() const +{ + return getBoolAttr("__contentAddressed", false); +} + } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 7621342d7..0b8e8d031 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,4 @@ -#include "derivations.hh" +#include "store-api.hh" #include <nlohmann/json_fwd.hpp> @@ -34,6 +34,8 @@ public: bool willBuildLocally() const; bool substitutesAllowed() const; + + bool contentAddressed() const; }; } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index aaebd3ec3..4f79843fe 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,5 +1,6 @@ #pragma once +#include "content-address.hh" #include "types.hh" namespace nix { @@ -65,11 +66,6 @@ typedef std::vector<StorePath> StorePaths; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true -}; - struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index fc5ab5865..b7cc7a5fc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -228,7 +228,7 @@ struct ConnectionHandle ~ConnectionHandle() { - if (!daemonException && std::uncaught_exception()) { + if (!daemonException && std::uncaught_exceptions()) { handle.markBad(); debug("closing daemon connection because of an exception"); } @@ -381,7 +381,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings<StringSet>(conn->from); - conn->from >> info->ca; + info->ca = parseContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -465,7 +465,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << info.ca + << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index aae227bae..e4a4ae11e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -61,7 +61,7 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p /* Store paths have the following form: - <store>/<h>-<name> + <realized-path> = <store>/<h>-<name> where @@ -85,11 +85,14 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p <type> = one of: "text:<r1>:<r2>:...<rN>" for plain text files written to the store using - addTextToStore(); <r1> ... <rN> are the references of the - path. - "source" + addTextToStore(); <r1> ... <rN> are the store paths referenced + by this path, in the form described by <realized-path> + "source:<r1>:<r2>:...:<rN>:self" for paths copied to the store using addToStore() when recursive - = true and hashAlgo = "sha256" + = true and hashAlgo = "sha256". Just like in the text case, we + can have the store paths referenced by the path. + Additionally, we can have an optional :self label to denote self + reference. "output:<id>" for either the outputs created by derivations, OR paths copied to the store using addToStore() with recursive != true or @@ -117,6 +120,12 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p the contents of the path (or expected contents of the path for fixed-output derivations) + Note that since an output derivation has always type output, while + something added by addToStore can have type output or source depending + on the hash, this means that the same input can be hashed differently + if added to the store via addToStore or via a derivation, in the sha256 + recursive case. + It would have been nicer to handle fixed-output derivations under "source", e.g. have something like "source:<rec><algo>", but we're stuck with this for now... @@ -164,20 +173,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod recursive, + FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + makeFileIngestionPrefix(method) + hash.to_string(Base16, true) + ":"), name); } @@ -462,8 +471,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store jsonRefs.elem(printStorePath(ref)); } - if (info->ca != "") - jsonPath.attr("ca", info->ca); + if (info->ca) + jsonPath.attr("ca", renderContentAddress(info->ca)); std::pair<uint64_t, uint64_t> closureSizes; @@ -748,41 +757,35 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +// FIXME Put this somewhere? +template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; +template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto warn = [&]() { - logWarning( - ErrorInfo{ - .name = "Path not content-addressed", - .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path)) - }); - }; - - if (hasPrefix(ca, "text:")) { - Hash hash(ca.substr(5)); - if (store.makeTextPath(path.name(), hash, references) == path) - return true; - else - warn(); - } + if (! ca) return false; - else if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6)); - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); + auto caPath = std::visit(overloaded { + [&](TextHash th) { + return store.makeTextPath(path.name(), th.hash, references); + }, + [&](FixedOutputHash fsh) { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); } - if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) - return true; - else - warn(); - } + }, *ca); + + bool res = caPath == path; - return false; + if (!res) + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + + return res; } @@ -813,14 +816,6 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) -{ - return "fixed:" - + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(Base32, true); -} - - } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6f4dd959c..25d78c297 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,12 +2,14 @@ #include "path.hh" #include "hash.hh" +#include "content-address.hh" #include "serialise.hh" #include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" #include "config.hh" +#include "derivations.hh" #include <atomic> #include <limits> @@ -17,6 +19,7 @@ #include <memory> #include <string> #include <chrono> +#include <variant> namespace nix { @@ -31,15 +34,12 @@ MakeError(SubstituterDisabled, Error); MakeError(NotInStore, Error); -struct BasicDerivation; -struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; class JSONPlaceholder; -enum RepairFlag : bool { NoRepair = false, Repair = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; @@ -111,7 +111,6 @@ struct SubstitutablePathInfo typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - struct ValidPathInfo { StorePath path; @@ -140,21 +139,11 @@ struct ValidPathInfo that a particular output path was produced by a derivation; the path then implies the contents.) - Ideally, the content-addressability assertion would just be a - Boolean, and the store path would be computed from - the name component, ‘narHash’ and ‘references’. However, - 1) we've accumulated several types of content-addressed paths - over the years; and 2) fixed-output derivations support - multiple hash algorithms and serialisation methods (flat file - vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:<sha256 hash of file contents>’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:<r?>:<ht>:<h>’: For paths computed by - makeFixedOutputPath() / addToStore(). + Ideally, the content-addressability assertion would just be a Boolean, + and the store path would be computed from the name component, ‘narHash’ + and ‘references’. However, we support many types of content addresses. */ - std::string ca; + std::optional<ContentAddress> ca; bool operator == (const ValidPathInfo & i) const { @@ -189,9 +178,10 @@ struct ValidPathInfo Strings shortRefs() const; - ValidPathInfo(const StorePath & path) : path(path) { } + ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path) : path(std::move(path)) { } + ValidPathInfo(StorePath && path) : path(std::move(path)) { }; + ValidPathInfo(const StorePath & path) : path(path) { }; virtual ~ValidPathInfo() { } }; @@ -838,12 +828,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo( std::istream & str, bool hashGiven = false); - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - - /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 8ae07b092..a38c2d798 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -11,5 +11,7 @@ namespace nix { #define ANSI_GREEN "\e[32;1m" #define ANSI_YELLOW "\e[33;1m" #define ANSI_BLUE "\e[34;1m" +#define ANSI_MAGENTA "\e[35m;1m" +#define ANSI_CYAN "\e[36m;1m" } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index e0dfc841f..b16a2e213 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -201,6 +201,13 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } +static void hashTypeCompleter(size_t index, std::string_view prefix) +{ + for (auto & type : hashTypes) + if (hasPrefix(type, prefix)) + completions->insert(type); +} + Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) { return Flag { @@ -209,14 +216,21 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .labels = {"hash-algo"}, .handler = {[ht](std::string s) { *ht = parseHashType(s); - if (*ht == htUnknown) - throw UsageError("unknown hash type '%1%'", s); }}, - .completer = [](size_t index, std::string_view prefix) { - for (auto & type : hashTypes) - if (hasPrefix(type, prefix)) - completions->insert(type); - } + .completer = hashTypeCompleter + }; +} + +Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht) +{ + return Flag { + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", + .labels = {"hash-algo"}, + .handler = {[oht](std::string s) { + *oht = std::optional<HashType> { parseHashType(s) }; + }}, + .completer = hashTypeCompleter }; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 2d49dbf66..97a517344 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -89,6 +89,7 @@ protected: std::function<void(size_t, std::string_view)> completer; static Flag mkHashTypeFlag(std::string && longName, HashType * ht); + static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht); }; std::map<std::string, Flag::ptr> longFlags; diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 12ab9c407..a39de041f 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,6 +1,7 @@ #pragma once #include <boost/format.hpp> +#include <boost/algorithm/string/replace.hpp> #include <string> #include "ansicolor.hh" @@ -103,7 +104,9 @@ class hintformat public: hintformat(const string &format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + fmt.exceptions(boost::io::all_error_bits ^ + boost::io::too_many_args_bit ^ + boost::io::too_few_args_bit); } hintformat(const hintformat &hf) @@ -117,6 +120,13 @@ public: return *this; } + template<class T> + hintformat& operator%(const normaltxt<T> &value) + { + fmt % value.value; + return *this; + } + std::string str() const { return fmt.str(); @@ -136,4 +146,9 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args) return f; } +inline hintformat hintfmt(std::string plain_string) +{ + // we won't be receiving any args in this case, so just print the original string + return hintfmt("%s", normaltxt(plain_string)); +} } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index cb0621a32..baea6412b 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -4,6 +4,7 @@ #include <openssl/md5.h> #include <openssl/sha.h> +#include "args.hh" #include "hash.hh" #include "archive.hh" #include "util.hh" @@ -21,11 +22,13 @@ std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" }; void Hash::init() { - if (type == htMD5) hashSize = md5HashSize; - else if (type == htSHA1) hashSize = sha1HashSize; - else if (type == htSHA256) hashSize = sha256HashSize; - else if (type == htSHA512) hashSize = sha512HashSize; - else abort(); + assert(type); + switch (*type) { + case htMD5: hashSize = md5HashSize; break; + case htSHA1: hashSize = sha1HashSize; break; + case htSHA256: hashSize = sha256HashSize; break; + case htSHA512: hashSize = sha512HashSize; break; + } assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -101,15 +104,22 @@ static string printHash32(const Hash & hash) string printHash16or32(const Hash & hash) { + assert(hash.type); return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); } +HashType assertInitHashType(const Hash & h) +{ + assert(h.type); + return *h.type; +} + std::string Hash::to_string(Base base, bool includeType) const { std::string s; if (base == SRI || includeType) { - s += printHashType(type); + s += printHashType(assertInitHashType(*this)); s += base == SRI ? '-' : ':'; } switch (base) { @@ -127,8 +137,10 @@ std::string Hash::to_string(Base base, bool includeType) const return s; } +Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } +Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { } -Hash::Hash(std::string_view s, HashType type) +Hash::Hash(std::string_view s, std::optional<HashType> type) : type(type) { size_t pos = 0; @@ -139,17 +151,17 @@ Hash::Hash(std::string_view s, HashType type) sep = s.find('-'); if (sep != string::npos) { isSRI = true; - } else if (type == htUnknown) + } else if (! type) throw BadHash("hash '%s' does not include a type", s); } if (sep != string::npos) { string hts = string(s, 0, sep); this->type = parseHashType(hts); - if (this->type == htUnknown) + if (!this->type) throw BadHash("unknown hash type '%s'", hts); - if (type != htUnknown && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); + if (type && type != this->type) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type)); pos = sep + 1; } @@ -205,13 +217,15 @@ Hash::Hash(std::string_view s, HashType type) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type)); } -Hash newHashAllowEmpty(std::string hashStr, HashType ht) +Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht) { if (hashStr.empty()) { - Hash h(ht); + if (!ht) + throw BadHash("empty hash requires explicit hash type"); + Hash h(*ht); warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); return h; } else @@ -331,24 +345,36 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -HashType parseHashType(const string & s) +std::optional<HashType> parseHashTypeOpt(const string & s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; else if (s == "sha256") return htSHA256; else if (s == "sha512") return htSHA512; - else return htUnknown; + else return std::optional<HashType> {}; } +HashType parseHashType(const string & s) +{ + auto opt_h = parseHashTypeOpt(s); + if (opt_h) + return *opt_h; + else + throw UsageError("unknown hash algorithm '%1%'", s); +} string printHashType(HashType ht) { - if (ht == htMD5) return "md5"; - else if (ht == htSHA1) return "sha1"; - else if (ht == htSHA256) return "sha256"; - else if (ht == htSHA512) return "sha512"; - else abort(); + switch (ht) { + case htMD5: return "md5"; + case htSHA1: return "sha1"; + case htSHA256: return "sha256"; + case htSHA512: return "sha512"; + default: + // illegal hash type enum value internally, as opposed to external input + // which should be validated with nice error message. + abort(); + } } - } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index c5724a321..ad6093fca 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -10,7 +10,7 @@ namespace nix { MakeError(BadHash, Error); -enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; +enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; const int md5HashSize = 16; @@ -31,7 +31,7 @@ struct Hash unsigned int hashSize = 0; unsigned char hash[maxHashSize] = {}; - HashType type = htUnknown; + std::optional<HashType> type = {}; /* Create an unset hash object. */ Hash() { }; @@ -42,14 +42,18 @@ struct Hash /* Initialize the hash from a string representation, in the format "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a Subresource Integrity hash expression). If the 'type' argument - is htUnknown, then the hash type must be specified in the + is not present, then the hash type must be specified in the string. */ - Hash(std::string_view s, HashType type = htUnknown); + Hash(std::string_view s, std::optional<HashType> type); + // type must be provided + Hash(std::string_view s, HashType type); + // hash type must be part of string + Hash(std::string_view s); void init(); /* Check whether a hash is set. */ - operator bool () const { return type != htUnknown; } + operator bool () const { return (bool) type; } /* Check whether two hash are equal. */ bool operator == (const Hash & h2) const; @@ -97,7 +101,7 @@ struct Hash }; /* Helper that defaults empty hashes to the 0 hash. */ -Hash newHashAllowEmpty(std::string hashStr, HashType ht); +Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht); /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ string printHash16or32(const Hash & hash); @@ -121,6 +125,9 @@ Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ HashType parseHashType(const string & s); +/* Will return nothing on parse error */ +std::optional<HashType> parseHashTypeOpt(const string & s); + /* And the reverse. */ string printHashType(HashType ht); diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 74e37b4c4..01331947e 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -173,7 +173,7 @@ JSONObject JSONPlaceholder::object() JSONPlaceholder::~JSONPlaceholder() { - assert(!first || std::uncaught_exception()); + assert(!first || std::uncaught_exceptions()); } } diff --git a/src/libutil/tests/compression.cc b/src/libutil/tests/compression.cc new file mode 100644 index 000000000..5b7a2c5b9 --- /dev/null +++ b/src/libutil/tests/compression.cc @@ -0,0 +1,78 @@ +#include "compression.hh" +#include <gtest/gtest.h> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * compress / decompress + * --------------------------------------------------------------------------*/ + + TEST(compress, compressWithUnknownMethod) { + ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod); + } + + TEST(compress, noneMethodDoesNothingToTheInput) { + ref<std::string> o = compress("none", "this-is-a-test"); + + ASSERT_EQ(*o, "this-is-a-test"); + } + + TEST(decompress, decompressXzCompressed) { + auto method = "xz"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref<std::string> o = decompress(method, *compress(method, str)); + + ASSERT_EQ(*o, str); + } + + TEST(decompress, decompressBzip2Compressed) { + auto method = "bzip2"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref<std::string> o = decompress(method, *compress(method, str)); + + ASSERT_EQ(*o, str); + } + + TEST(decompress, decompressBrCompressed) { + auto method = "br"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref<std::string> o = decompress(method, *compress(method, str)); + + ASSERT_EQ(*o, str); + } + + TEST(decompress, decompressInvalidInputThrowsCompressionError) { + auto method = "bzip2"; + auto str = "this is a string that does not qualify as valid bzip2 data"; + + ASSERT_THROW(decompress(method, str), CompressionError); + } + + /* ---------------------------------------------------------------------------- + * compression sinks + * --------------------------------------------------------------------------*/ + + TEST(makeCompressionSink, noneSinkDoesNothingToInput) { + StringSink strSink; + auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto sink = makeCompressionSink("none", strSink); + (*sink)(inputString); + sink->finish(); + + ASSERT_STREQ((*strSink.s).c_str(), inputString); + } + + TEST(makeCompressionSink, compressAndDecompress) { + StringSink strSink; + auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto decompressionSink = makeDecompressionSink("bzip2", strSink); + auto sink = makeCompressionSink("bzip2", *decompressionSink); + + (*sink)(inputString); + sink->finish(); + decompressionSink->finish(); + + ASSERT_STREQ((*strSink.s).c_str(), inputString); + } + +} diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 5334b046e..412c03030 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -72,9 +72,4 @@ namespace nix { "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); } - - TEST(hashString, hashingWithUnknownAlgoExits) { - auto s = "unknown"; - ASSERT_DEATH(hashString(HashType::htUnknown, s), ""); - } } diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 4cb54995b..6a6fb4ac3 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -1,6 +1,7 @@ #include "logging.hh" #include "nixexpr.hh" #include "util.hh" +#include <fstream> #include <gtest/gtest.h> @@ -42,7 +43,7 @@ namespace nix { logger->logEI(ei); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\ninitial error; subsequent error message.\n"); } } @@ -60,8 +61,7 @@ namespace nix { logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n"); - + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n"); } } @@ -69,9 +69,9 @@ namespace nix { testing::internal::CaptureStderr(); logger->logEI({ .level = lvlInfo, - .name = "Info name", - .description = "Info description", - }); + .name = "Info name", + .description = "Info description", + }); auto str = testing::internal::GetCapturedStderr(); ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name --- error-unit-test\x1B[0m\nInfo description\n"); @@ -85,7 +85,7 @@ namespace nix { logger->logEI({ .level = lvlTalkative, .name = "Talkative name", .description = "Talkative description", - }); + }); auto str = testing::internal::GetCapturedStderr(); ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name --- error-unit-test\x1B[0m\nTalkative description\n"); @@ -99,7 +99,7 @@ namespace nix { logger->logEI({ .level = lvlChatty, .name = "Chatty name", .description = "Talkative description", - }); + }); auto str = testing::internal::GetCapturedStderr(); ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name --- error-unit-test\x1B[0m\nTalkative description\n"); @@ -113,7 +113,7 @@ namespace nix { logger->logEI({ .level = lvlDebug, .name = "Debug name", .description = "Debug description", - }); + }); auto str = testing::internal::GetCapturedStderr(); ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name --- error-unit-test\x1B[0m\nDebug description\n"); @@ -127,7 +127,7 @@ namespace nix { logger->logEI({ .level = lvlVomit, .name = "Vomit name", .description = "Vomit description", - }); + }); auto str = testing::internal::GetCapturedStderr(); ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name --- error-unit-test\x1B[0m\nVomit description\n"); @@ -144,7 +144,7 @@ namespace nix { logError({ .name = "name", .description = "error description", - }); + }); auto str = testing::internal::GetCapturedStderr(); ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n"); @@ -160,13 +160,13 @@ namespace nix { .name = "error name", .description = "error with code lines", .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), + "yellow", + "values"), .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = "previous line of code", - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = "next line of code", + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = "previous line of code", + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = "next line of code", }}); @@ -183,10 +183,10 @@ namespace nix { .name = "error name", .description = "error without any code lines.", .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), + "yellow", + "values"), .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13) + .errPos = Pos(problem_file, 40, 13) }}); auto str = testing::internal::GetCapturedStderr(); @@ -202,7 +202,7 @@ namespace nix { .name = "error name", .hint = hintfmt("hint %1%", "only"), .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13) + .errPos = Pos(problem_file, 40, 13) }}); auto str = testing::internal::GetCapturedStderr(); @@ -218,10 +218,10 @@ namespace nix { testing::internal::CaptureStderr(); logWarning({ - .name = "name", - .description = "error description", - .hint = hintfmt("there was a %1%", "warning"), - }); + .name = "name", + .description = "error description", + .hint = hintfmt("there was a %1%", "warning"), + }); auto str = testing::internal::GetCapturedStderr(); ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n"); @@ -238,13 +238,13 @@ namespace nix { .name = "warning name", .description = "warning description", .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), + "yellow", + "values"), .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = std::nullopt, - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = std::nullopt + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::nullopt, + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::nullopt }}); @@ -252,4 +252,41 @@ namespace nix { ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); } + /* ---------------------------------------------------------------------------- + * hintfmt + * --------------------------------------------------------------------------*/ + + TEST(hintfmt, percentStringWithoutArgs) { + + const char *teststr = "this is 100%s correct!"; + + ASSERT_STREQ( + hintfmt(teststr).str().c_str(), + teststr); + + } + + TEST(hintfmt, fmtToHintfmt) { + + ASSERT_STREQ( + hintfmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(), + "the color of this this text is not yellow"); + + } + + TEST(hintfmt, tooFewArguments) { + + ASSERT_STREQ( + hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(), + "only one arg " ANSI_YELLOW "fulfilled" ANSI_NORMAL " "); + + } + + TEST(hintfmt, tooManyArguments) { + + ASSERT_STREQ( + hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), + "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); + + } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index cc0e606a9..93798a765 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -36,7 +36,7 @@ #endif -extern char * * environ; +extern char * * environ __attribute__((weak)); namespace nix { @@ -1215,7 +1215,7 @@ void _interrupted() /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!interruptThrown && !std::uncaught_exception()) { + if (!interruptThrown && !std::uncaught_exceptions()) { interruptThrown = true; throw Interrupted("interrupted by the user"); } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index a224d635d..f77de56ea 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -21,7 +21,7 @@ using namespace nix; using namespace std::string_literals; -extern char * * environ; +extern char * * environ __attribute__((weak)); /* Recreate the effect of the perl shellwords function, breaking up a * string into arguments like a shell word, including escapes diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 55b72bda6..40b05a2f3 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -72,8 +72,6 @@ static int _main(int argc, char * * argv) else if (*arg == "--type") { string s = getArg(*arg, arg, end); ht = parseHashType(s); - if (ht == htUnknown) - throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--print-path") printPath = true; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5c5afd5ec..7d81bf54f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(info->narHash.type); + HashSink sink(*info->narHash.type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -952,7 +952,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.references = readStorePaths<StorePathSet>(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); - in >> info.ca; + info.ca = parseContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1..f9d6de16e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -48,7 +48,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = std::optional { FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = info.narHash, + } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/command.cc b/src/nix/command.cc index f7ebf998d..dbf5e0988 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -4,7 +4,7 @@ #include "nixexpr.hh" #include "profiles.hh" -extern char * * environ; +extern char * * environ __attribute__((weak)); namespace nix { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 06b165e41..9a50fd47e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" }); + drv.outputs.insert_or_assign("out", DerivationOutput { + .path = shellOutPath, + .hash = FixedOutputHash { + .method = FileIngestionMethod::Flat, + .hash = Hash { }, + }, + }); drv.env["out"] = store->printStorePath(shellOutPath); auto shellDrvPath2 = writeDerivation(store, drv, drvName); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 7f646ad53..b94751e45 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "hash.hh" +#include "content-address.hh" #include "legacy.hh" #include "shared.hh" #include "references.hh" @@ -83,12 +84,12 @@ static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngest struct CmdToBase : Command { Base base; - HashType ht = htUnknown; + std::optional<HashType> ht; std::vector<std::string> args; CmdToBase(Base base) : base(base) { - addFlag(Flag::mkHashTypeFlag("type", &ht)); + addFlag(Flag::mkHashTypeOptFlag("type", &ht)); expectArgs("strings", &args); } @@ -136,8 +137,6 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--type") { string s = getArg(*arg, arg, end); ht = parseHashType(s); - if (ht == htUnknown) - throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--to-base16") op = opTo16; else if (*arg == "--to-base32") op = opTo32; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0ebb8f13b..fb36fc410 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -82,7 +82,10 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = info.narHash, + }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index fb7bacc4c..b89a44f83 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca != "") ss.push_back("ca:" + info->ca); + if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 3ba4f6e32..307e236d8 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -132,7 +132,7 @@ struct ProfileManifest info.references = std::move(references); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 73b0ae521..c30ac2f79 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -19,6 +19,7 @@ extern "C" { } #endif +#include "ansicolor.hh" #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" @@ -37,14 +38,6 @@ extern "C" { namespace nix { -#define ESC_RED "\033[31m" -#define ESC_GRE "\033[32m" -#define ESC_YEL "\033[33m" -#define ESC_BLU "\033[34;1m" -#define ESC_MAG "\033[35m" -#define ESC_CYA "\033[36m" -#define ESC_END "\033[0m" - struct NixRepl : gc { string curDir; @@ -645,25 +638,25 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m switch (v.type) { case tInt: - str << ESC_CYA << v.integer << ESC_END; + str << ANSI_CYAN << v.integer << ANSI_NORMAL; break; case tBool: - str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; + str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; break; case tString: - str << ESC_YEL; + str << ANSI_YELLOW; printStringValue(str, v.string.s); - str << ESC_END; + str << ANSI_NORMAL; break; case tPath: - str << ESC_GRE << v.path << ESC_END; // !!! escaping? + str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? break; case tNull: - str << ESC_CYA "null" ESC_END; + str << ANSI_CYAN "null" ANSI_NORMAL; break; case tAttrs: { @@ -699,7 +692,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m try { printValue(str, *i.second, maxDepth - 1, seen); } catch (AssertionError & e) { - str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; } str << "; "; } @@ -725,7 +718,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m try { printValue(str, *v.listElems()[n], maxDepth - 1, seen); } catch (AssertionError & e) { - str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; } str << " "; } @@ -737,16 +730,16 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case tLambda: { std::ostringstream s; s << v.lambda.fun->pos; - str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; + str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; break; } case tPrimOp: - str << ESC_MAG "«primop»" ESC_END; + str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; break; case tPrimOpApp: - str << ESC_BLU "«primop-app»" ESC_END; + str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; break; case tFloat: @@ -754,7 +747,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; default: - str << ESC_RED "«unknown»" ESC_END; + str << ANSI_RED "«unknown»" ANSI_NORMAL; break; } diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 2d31894c2..5d77cfdca 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -70,9 +70,9 @@ struct CmdShowDerivation : InstallablesCommand for (auto & output : drv.outputs) { auto outputObj(outputsObj.object(output.first)); outputObj.attr("path", store->printStorePath(output.second.path)); - if (output.second.hash != "") { - outputObj.attr("hashAlgo", output.second.hashAlgo); - outputObj.attr("hash", output.second.hash); + if (output.second.hash) { + outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); + outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); } } } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index ab83637dc..bb5e4529b 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -87,10 +87,10 @@ struct CmdVerify : StorePathsCommand if (!noContents) { std::unique_ptr<AbstractHashSink> hashSink; - if (info->ca == "") - hashSink = std::make_unique<HashSink>(info->narHash.type); + if (!info->ca) + hashSink = std::make_unique<HashSink>(*info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 8c5ca8cd3..a19c10dde 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -1,23 +1,39 @@ +{ busybox }: + with import ./config.nix; let + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + } // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; + input1 = mkDerivation { - name = "build-hook-input-1"; - buildCommand = "mkdir $out; echo FOO > $out/foo"; + shell = busybox; + name = "build-remote-input-1"; + buildCommand = "echo FOO > $out"; requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { - name = "build-hook-input-2"; - buildCommand = "mkdir $out; echo BAR > $out/bar"; + shell = busybox; + name = "build-remote-input-2"; + buildCommand = "echo BAR > $out"; }; in mkDerivation { - name = "build-hook"; - builder = ./dependencies.builder0.sh; - input1 = " " + input1 + "/."; - input2 = " ${input2}/."; + shell = busybox; + name = "build-remote"; + buildCommand = + '' + read x < ${input1} + read y < ${input2} + echo $x$y > $out + ''; } diff --git a/tests/build-remote.sh b/tests/build-remote.sh index a550f4460..4dfb753e1 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -3,22 +3,29 @@ source common.sh clearStore if ! canUseSandbox; then exit; fi -if [[ ! $SHELL =~ /nix/store ]]; then exit; fi +if ! [[ $busybox =~ busybox ]]; then exit; fi -chmod -R u+w $TEST_ROOT/store0 || true -chmod -R u+w $TEST_ROOT/store1 || true -rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 +chmod -R u+w $TEST_ROOT/machine0 || true +chmod -R u+w $TEST_ROOT/machine1 || true +chmod -R u+w $TEST_ROOT/machine2 || true +rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2 +rm -f $TEST_ROOT/result -nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ - --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ - --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \ +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a +# child process. This allows us to test LegacySSHStore::buildDerivation(). +nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ + --arg busybox $busybox \ + --store $TEST_ROOT/machine0 \ + --builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \ --system-features foo -outPath=$TEST_ROOT/result +outPath=$(readlink -f $TEST_ROOT/result) -cat $outPath/foobar | grep FOOBAR +cat $TEST_ROOT/machine0/$outPath | grep FOOBAR -# Ensure that input1 was built on store1 due to the required feature. -p=$(readlink -f $outPath/input-2) -(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh) -nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh +# Ensure that input1 was built on store2 due to the required feature. +(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh) +nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index dd7e61822..73fe77345 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -35,6 +35,7 @@ export xmllint="@xmllint@" export SHELL="@bash@" export PAGER=cat export HAVE_SODIUM="@HAVE_SODIUM@" +export busybox="@sandbox_shell@" export version=@PACKAGE_VERSION@ export system=@system@ diff --git a/tests/post-hook.sh b/tests/post-hook.sh index a02657215..aa3e6a574 100644 --- a/tests/post-hook.sh +++ b/tests/post-hook.sh @@ -2,6 +2,8 @@ source common.sh clearStore +rm -f $TEST_ROOT/result + export REMOTE_STORE=$TEST_ROOT/remote_store # Build the dependencies and push them to the remote store diff --git a/tests/recursive.sh b/tests/recursive.sh index b255a2883..cf10d55bf 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -5,6 +5,8 @@ if [[ $(uname) != Linux ]]; then exit; fi clearStore +rm -f $TEST_ROOT/result + export unreachable=$(nix add-to-store ./recursive.sh) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index 646bdb876..dcfe6d580 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -2,6 +2,8 @@ source common.sh clearStore +rm -f $TEST_ROOT/result + nix-build structured-attrs.nix -A all -o $TEST_ROOT/result [[ $(cat $TEST_ROOT/result/foo) = bar ]] |