From fc84c358d9e55e9ba1d939d8974f6deef629848e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 20:58:02 +0200 Subject: Make 'nix copy' to file:// binary caches run in constant memory --- src/libstore/binary-cache-store.cc | 112 +++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 36 deletions(-) (limited to 'src/libstore/binary-cache-store.cc') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..166041b6c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -57,6 +57,14 @@ void BinaryCacheStore::init() } } +void BinaryCacheStore::upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) +{ + StringSource source(data); + upsertFile(path, source, mimeType); +} + void BinaryCacheStore::getFile(const std::string & path, Callback> callback) noexcept { @@ -113,13 +121,70 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr(narInfo)); } +AutoCloseFD openFile(const Path & path) +{ + auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return fd; +} + +struct FileSource : FdSource +{ + AutoCloseFD fd2; + + FileSource(const Path & path) + : fd2(openFile(path)) + { + fd = fd2.get(); + } +}; + void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) { - // FIXME: See if we can use the original source to reduce memory usage. - auto nar = make_ref(narSource.drain()); + assert(info.narHash && info.narSize); + + if (!repair && isValidPath(info.path)) { + // FIXME: copyNAR -> null sink + narSource.drain(); + return; + } + + auto [fdTemp, fnTemp] = createTempFile(); + + auto now1 = std::chrono::steady_clock::now(); + + HashSink fileHashSink(htSHA256); + + { + FdSink fileSink(fdTemp.get()); + TeeSink teeSink(fileSink, fileHashSink); + auto compressionSink = makeCompressionSink(compression, teeSink); + copyNAR(narSource, *compressionSink); + compressionSink->finish(); + } + + auto now2 = std::chrono::steady_clock::now(); + + auto narInfo = make_ref(info); + narInfo->narSize = info.narSize; + narInfo->narHash = info.narHash; + narInfo->compression = compression; + auto [fileHash, fileSize] = fileHashSink.finish(); + narInfo->fileHash = fileHash; + narInfo->fileSize = fileSize; + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); - if (!repair && isValidPath(info.path)) return; + auto duration = std::chrono::duration_cast(now2 - now1).count(); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), info.narSize, + ((1.0 - (double) fileSize / info.narSize) * 100.0), + duration); /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -132,16 +197,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource printStorePath(info.path), printStorePath(ref)); } - assert(nar->compare(0, narMagic.size(), narMagic) == 0); - - auto narInfo = make_ref(info); - - narInfo->narSize = nar->size(); - narInfo->narHash = hashString(htSHA256, *nar); - - if (info.narHash && info.narHash != narInfo->narHash) - throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); - + #if 0 auto accessor_ = std::dynamic_pointer_cast(accessor); auto narAccessor = makeNarAccessor(nar); @@ -166,27 +222,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } + #endif - /* Compress the NAR. */ - narInfo->compression = compression; - auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar, parallelCompression); - auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, *narCompressed); - narInfo->fileSize = narCompressed->size(); - - auto duration = std::chrono::duration_cast(now2 - now1).count(); - printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", - printStorePath(narInfo->path), narInfo->narSize, - ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), - duration); - - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); - + #if 0 /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ @@ -243,16 +281,18 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource threadPool.process(); } } + #endif /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + FileSource source(fnTemp); + upsertFile(narInfo->url, source, "application/x-nix-nar"); } else stats.narWriteAverted++; - stats.narWriteBytes += nar->size(); - stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteBytes += info.narSize; + stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ -- cgit v1.2.3 From 0a9da00a10fa27a3e3b98439cb0a7d5e79135b58 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 17:30:42 +0200 Subject: NarAccessor: Run in constant memory --- src/libstore/binary-cache-store.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src/libstore/binary-cache-store.cc') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 166041b6c..5851722d7 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -155,13 +155,17 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto now1 = std::chrono::steady_clock::now(); + /* Read the NAR simultaneously into a CompressionSink+FileSink (to + write the compressed NAR to disk), into a HashSink (to get the + NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink(htSHA256); - + std::shared_ptr narAccessor; { FdSink fileSink(fdTemp.get()); TeeSink teeSink(fileSink, fileHashSink); auto compressionSink = makeCompressionSink(compression, teeSink); - copyNAR(narSource, *compressionSink); + TeeSource teeSource(narSource, *compressionSink); + narAccessor = makeNarAccessor(teeSource); compressionSink->finish(); } @@ -200,10 +204,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource #if 0 auto accessor_ = std::dynamic_pointer_cast(accessor); - auto narAccessor = makeNarAccessor(nar); - if (accessor_) accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); + #endif /* Optionally write a JSON file containing a listing of the contents of the NAR. */ @@ -216,15 +219,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource { auto res = jsonRoot.placeholder("root"); - listNar(res, narAccessor, "", true); + listNar(res, ref(narAccessor), "", true); } } upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } - #endif - #if 0 /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ @@ -281,7 +282,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource threadPool.process(); } } - #endif /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { -- cgit v1.2.3 From 545bb2ed03001cd7a80a90f73eb500f396c043a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 17:37:44 +0200 Subject: Remove 'accessor' from addToStore() This is only used by hydra-queue-runner and it's better to implement it there. --- src/libstore/binary-cache-store.cc | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'src/libstore/binary-cache-store.cc') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5851722d7..e71240558 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -141,7 +141,7 @@ struct FileSource : FdSource }; void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { assert(info.narHash && info.narSize); @@ -201,13 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource printStorePath(info.path), printStorePath(ref)); } - #if 0 - auto accessor_ = std::dynamic_pointer_cast(accessor); - - if (accessor_) - accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); - #endif - /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -391,7 +384,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath ValidPathInfo info(makeFixedOutputPath(method, h, name)); auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); + addToStore(info, source, repair, CheckSigs); return std::move(info.path); } @@ -406,7 +399,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s StringSink sink; dumpString(s, sink); auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); + addToStore(info, source, repair, CheckSigs); } return std::move(info.path); -- cgit v1.2.3 From 7c2fef0a819481058d49c469c115bb0668b7016b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 20:07:19 +0200 Subject: Make 'nix copy' to s3:// binary caches run in constant memory --- src/libstore/binary-cache-store.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'src/libstore/binary-cache-store.cc') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e71240558..b791c125b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -58,11 +59,10 @@ void BinaryCacheStore::init() } void BinaryCacheStore::upsertFile(const std::string & path, - const std::string & data, + std::string && data, const std::string & mimeType) { - StringSource source(data); - upsertFile(path, source, mimeType); + upsertFile(path, std::make_shared(std::move(data)), mimeType); } void BinaryCacheStore::getFile(const std::string & path, @@ -279,8 +279,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - FileSource source(fnTemp); - upsertFile(narInfo->url, source, "application/x-nix-nar"); + upsertFile(narInfo->url, + std::make_shared(fnTemp, std::ios_base::in), + "application/x-nix-nar"); } else stats.narWriteAverted++; -- cgit v1.2.3