diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/libexpr/primops.cc | 2 | ||||
-rw-r--r-- | src/libexpr/primops/context.cc | 54 | ||||
-rw-r--r-- | src/libexpr/primops/fetchMercurial.cc | 6 | ||||
-rw-r--r-- | src/libexpr/primops/fetchTree.cc | 6 | ||||
-rw-r--r-- | src/libexpr/primops/fromTOML.cc | 20 | ||||
-rw-r--r-- | src/libstore/build/local-derivation-goal.cc | 43 | ||||
-rw-r--r-- | src/libstore/path-references.cc | 73 | ||||
-rw-r--r-- | src/libstore/path-references.hh | 25 | ||||
-rw-r--r-- | src/libstore/uds-remote-store.cc | 8 | ||||
-rw-r--r-- | src/libstore/uds-remote-store.hh | 7 | ||||
-rw-r--r-- | src/libutil/filesystem.cc | 17 | ||||
-rw-r--r-- | src/libutil/references.cc (renamed from src/libstore/references.cc) | 80 | ||||
-rw-r--r-- | src/libutil/references.hh (renamed from src/libstore/references.hh) | 23 | ||||
-rw-r--r-- | src/libutil/tests/references.cc | 46 | ||||
-rw-r--r-- | src/libutil/util.hh | 6 | ||||
-rwxr-xr-x | src/nix-channel/nix-channel.cc | 8 |
16 files changed, 287 insertions, 137 deletions
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 42efca4e7..47d8076b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -6,7 +6,7 @@ #include "globals.hh" #include "json-to-value.hh" #include "names.hh" -#include "references.hh" +#include "path-references.hh" #include "store-api.hh" #include "util.hh" #include "value-to-json.hh" diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 07bf400cf..8b3468009 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -12,7 +12,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, v.mkString(*s); } -static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext({ + .name = "__unsafeDiscardStringContext", + .arity = 1, + .fun = prim_unsafeDiscardStringContext +}); static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -22,7 +26,16 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, v.mkBool(!context.empty()); } -static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext({ + .name = "__hasContext", + .args = {"s"}, + .doc = R"( + Return `true` if string *s* has a non-empty context. The + context can be obtained with + [`getContext`](#builtins-getContext). + )", + .fun = prim_hasContext +}); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -51,7 +64,11 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p v.mkString(*s, context2); } -static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency({ + .name = "__unsafeDiscardOutputDependency", + .arity = 1, + .fun = prim_unsafeDiscardOutputDependency +}); /* Extract the context of a string as a structured Nix value. @@ -119,7 +136,30 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, v.mkAttrs(attrs); } -static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext({ + .name = "__getContext", + .args = {"s"}, + .doc = R"( + Return the string context of *s*. + + The string context tracks references to derivations within a string. + It is represented as an attribute set of [store derivation](@docroot@/glossary.md#gloss-store-derivation) paths mapping to output names. + + Using [string interpolation](@docroot@/language/string-interpolation.md) on a derivation will add that derivation to the string context. + For example, + + ```nix + builtins.getContext "${derivation { name = "a"; builder = "b"; system = "c"; }}" + ``` + + evaluates to + + ``` + { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; } + ``` + )", + .fun = prim_getContext +}); /* Append the given context to a given string. @@ -192,6 +232,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar v.mkString(orig, context); } -static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext({ + .name = "__appendContext", + .arity = 2, + .fun = prim_appendContext +}); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 2c0d98e74..322692b52 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -88,6 +88,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a state.allowPath(tree.storePath); } -static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial({ + .name = "fetchMercurial", + .arity = 1, + .fun = prim_fetchMercurial +}); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fe880aaa8..be8159cc8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -194,7 +194,11 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, } // FIXME: document -static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree({ + .name = "fetchTree", + .arity = 1, + .fun = prim_fetchTree +}); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, const std::string & who, bool unpack, std::string name) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index e2a8b3c3a..2f4d4022e 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -90,6 +90,24 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V } } -static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML({ + .name = "fromTOML", + .args = {"e"}, + .doc = R"( + Convert a TOML string to a Nix value. For example, + + ```nix + builtins.fromTOML '' + x=1 + s="a" + [table] + y=2 + '' + ``` + + returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. + )", + .fun = prim_fromTOML +}); } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 0b0bd3328..7f87cdf55 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -4,7 +4,7 @@ #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "references.hh" +#include "path-references.hh" #include "finally.hh" #include "util.hh" #include "archive.hh" @@ -2394,18 +2394,21 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() continue; auto references = *referencesOpt; - auto rewriteOutput = [&]() { + auto rewriteOutput = [&](const StringMap & rewrites) { /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { + if (!rewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(rewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); deletePath(actualPath); - sink.s = rewriteStrings(sink.s, outputRewrites); - StringSource source(sink.s); - restorePath(actualPath, source); + movePath(tmpPath, actualPath); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -2454,7 +2457,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } - rewriteOutput(); + rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; @@ -2492,16 +2495,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() Hash::dummy, }; if (*scratchPath != newInfo0.path) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - dumpPath(actualPath, rsink2); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); } HashResult narHashAndSize = hashPath(htSHA256, actualPath); @@ -2523,7 +2524,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() outputRewrites.insert_or_assign( std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); + rewriteOutput(outputRewrites); auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc new file mode 100644 index 000000000..33cf66ce3 --- /dev/null +++ b/src/libstore/path-references.cc @@ -0,0 +1,73 @@ +#include "path-references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include <map> +#include <cstdlib> +#include <mutex> +#include <algorithm> + + +namespace nix { + + +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } + +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +{ + StringSet hashes; + std::map<std::string, StorePath> backMap; + + for (auto & i : refs) { + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); + } + + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} + +StorePathSet PathRefScanSink::getResultPaths() +{ + /* Map the hashes found back to their store paths. */ + StorePathSet found; + for (auto & i : getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); + found.insert(j->second); + } + + return found; +} + + +std::pair<StorePathSet, HashResult> scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair<StorePathSet, HashResult>(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + +} diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh new file mode 100644 index 000000000..7b44e3261 --- /dev/null +++ b/src/libstore/path-references.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "references.hh" +#include "path.hh" + +namespace nix { + +std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs); + +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class PathRefScanSink : public RefScanSink +{ + std::map<std::string, StorePath> backMap; + + PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 0fb7c38e9..69dae2da5 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -13,6 +13,14 @@ namespace nix { +std::string UDSRemoteStoreConfig::doc() +{ + return + #include "uds-remote-store.md" + ; +} + + UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index bd1dcb67c..3bbab371c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -17,12 +17,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon const std::string name() override { return "Local Daemon Store"; } - std::string doc() override - { - return - #include "uds-remote-store.md" - ; - } + std::string doc() override; }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 56be76ecc..11cc0c0e7 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -63,30 +63,19 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) return {std::move(fd), tmpl}; } -void createSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) +void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } } -void replaceSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) +void replaceSymlink(const Path & target, const Path & link) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp, mtime); + createSymlink(target, tmp); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libstore/references.cc b/src/libutil/references.cc index 345f4528b..74003584a 100644 --- a/src/libstore/references.cc +++ b/src/libutil/references.cc @@ -6,6 +6,7 @@ #include <map> #include <cstdlib> #include <mutex> +#include <algorithm> namespace nix { @@ -66,69 +67,20 @@ void RefScanSink::operator () (std::string_view data) } -PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap) - : RefScanSink(std::move(hashes)) - , backMap(std::move(backMap)) -{ } - -PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : RewritingSink({{from, to}}, nextSink) { - StringSet hashes; - std::map<std::string, StorePath> backMap; - - for (auto & i : refs) { - std::string hashPart(i.hashPart()); - auto inserted = backMap.emplace(hashPart, i).second; - assert(inserted); - hashes.insert(hashPart); - } - - return PathRefScanSink(std::move(hashes), std::move(backMap)); } -StorePathSet PathRefScanSink::getResultPaths() +RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) + : rewrites(rewrites), nextSink(nextSink) { - /* Map the hashes found back to their store paths. */ - StorePathSet found; - for (auto & i : getResult()) { - auto j = backMap.find(i); - assert(j != backMap.end()); - found.insert(j->second); + long unsigned int maxRewriteSize = 0; + for (auto & [from, to] : rewrites) { + assert(from.size() == to.size()); + maxRewriteSize = std::max(maxRewriteSize, from.size()); } - - return found; -} - - -std::pair<StorePathSet, HashResult> scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair<StorePathSet, HashResult>(found, hash); -} - -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) -{ - PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); - TeeSink sink { refsSink, toTee }; - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - return refsSink.getResultPaths(); -} - - -RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) - : from(from), to(to), nextSink(nextSink) -{ - assert(from.size() == to.size()); + this->maxRewriteSize = maxRewriteSize; } void RewritingSink::operator () (std::string_view data) @@ -136,13 +88,13 @@ void RewritingSink::operator () (std::string_view data) std::string s(prev); s.append(data); - size_t j = 0; - while ((j = s.find(from, j)) != std::string::npos) { - matches.push_back(pos + j); - s.replace(j, from.size(), to); - } + s = rewriteStrings(s, rewrites); - prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + prev = s.size() < maxRewriteSize + ? s + : maxRewriteSize == 0 + ? "" + : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1); auto consumed = s.size() - prev.size(); diff --git a/src/libstore/references.hh b/src/libutil/references.hh index 52d71b333..ffd730e7b 100644 --- a/src/libstore/references.hh +++ b/src/libutil/references.hh @@ -2,14 +2,9 @@ ///@file #include "hash.hh" -#include "path.hh" namespace nix { -std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs); - -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); - class RefScanSink : public Sink { StringSet hashes; @@ -28,28 +23,18 @@ public: void operator () (std::string_view data) override; }; -class PathRefScanSink : public RefScanSink -{ - std::map<std::string, StorePath> backMap; - - PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap); - -public: - - static PathRefScanSink fromPaths(const StorePathSet & refs); - - StorePathSet getResultPaths(); -}; - struct RewritingSink : Sink { - std::string from, to, prev; + const StringMap rewrites; + long unsigned int maxRewriteSize; + std::string prev; Sink & nextSink; uint64_t pos = 0; std::vector<uint64_t> matches; RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + RewritingSink(const StringMap & rewrites, Sink & nextSink); void operator () (std::string_view data) override; diff --git a/src/libutil/tests/references.cc b/src/libutil/tests/references.cc new file mode 100644 index 000000000..a517d9aa1 --- /dev/null +++ b/src/libutil/tests/references.cc @@ -0,0 +1,46 @@ +#include "references.hh" +#include <gtest/gtest.h> + +namespace nix { + +using std::string; + +struct RewriteParams { + string originalString, finalString; + StringMap rewrites; + + friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) { + StringSet strRewrites; + for (auto & [from, to] : bar.rewrites) + strRewrites.insert(from + "->" + to); + return os << + "OriginalString: " << bar.originalString << std::endl << + "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Expected result: " << bar.finalString; + } +}; + +class RewriteTest : public ::testing::TestWithParam<RewriteParams> { +}; + +TEST_P(RewriteTest, IdentityRewriteIsIdentity) { + RewriteParams param = GetParam(); + StringSink rewritten; + auto rewriter = RewritingSink(param.rewrites, rewritten); + rewriter(param.originalString); + rewriter.flush(); + ASSERT_EQ(rewritten.s, param.finalString); +} + +INSTANTIATE_TEST_CASE_P( + references, + RewriteTest, + ::testing::Values( + RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}}, + RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, + RewriteParams{ "foooo", "foooo", {}} + ) +); + +} + diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 00fcb9b79..b302d6f45 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -256,14 +256,12 @@ inline Paths createDirs(PathView path) /** * Create a symlink. */ -void createSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime = {}); +void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime = {}); +void replaceSymlink(const Path & target, const Path & link); void renameFile(const Path & src, const Path & dst); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 740737ffe..c1c8edd1d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -177,6 +177,7 @@ static int main_nix_channel(int argc, char ** argv) cRemove, cList, cUpdate, + cListGenerations, cRollback } cmd = cNone; std::vector<std::string> args; @@ -193,6 +194,8 @@ static int main_nix_channel(int argc, char ** argv) cmd = cList; } else if (*arg == "--update") { cmd = cUpdate; + } else if (*arg == "--list-generations") { + cmd = cListGenerations; } else if (*arg == "--rollback") { cmd = cRollback; } else { @@ -237,6 +240,11 @@ static int main_nix_channel(int argc, char ** argv) case cUpdate: update(StringSet(args.begin(), args.end())); break; + case cListGenerations: + if (!args.empty()) + throw UsageError("'--list-generations' expects no arguments"); + std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + break; case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); |