aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWill Dietz <w@wdtz.org>2018-02-11 13:23:31 -0600
committerWill Dietz <w@wdtz.org>2018-02-11 13:50:28 -0600
commitc6209030c424424ebd51283326d5e5df68a48533 (patch)
tree33336fcec0644daaa081d9940a030fe997887b78
parenta0bdc96726b15b7f529156bccd60d0f8dd5544f3 (diff)
compression: make parallel sink separate class
-rw-r--r--src/libutil/compression.cc63
1 files changed, 34 insertions, 29 deletions
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 0b0ff1102..470c925ed 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -190,33 +190,9 @@ struct XzSink : CompressionSink
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
- XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
- {
- lzma_ret ret;
- if (parallel) {
-#ifdef HAVE_LZMA_MT
- lzma_mt mt_options = {};
- mt_options.flags = 0;
- mt_options.timeout = 300; // Using the same setting as the xz cmd line
- mt_options.preset = LZMA_PRESET_DEFAULT;
- mt_options.filters = NULL;
- mt_options.check = LZMA_CHECK_CRC64;
- mt_options.threads = lzma_cputhreads();
- mt_options.block_size = 0;
- if (mt_options.threads == 0)
- mt_options.threads = 1;
- // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
- // number of threads.
- ret = lzma_stream_encoder_mt(
- &strm, &mt_options);
- } else
-#else
- printMsg(lvlError, "Warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
- }
-#endif
- ret = lzma_easy_encoder(
- &strm, 6, LZMA_CHECK_CRC64);
-
+ template <typename F>
+ XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) {
+ lzma_ret ret = initEncoder();
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder");
// FIXME: apply the x86 BCJ filter?
@@ -224,6 +200,9 @@ struct XzSink : CompressionSink
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
+ XzSink(Sink & nextSink) : XzSink(nextSink, [this]() {
+ return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
+ }) {}
~XzSink()
{
@@ -277,6 +256,27 @@ struct XzSink : CompressionSink
}
};
+#ifdef HAVE_LZMA_MT
+struct ParallelXzSink : public XzSink
+{
+ ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() {
+ lzma_mt mt_options = {};
+ mt_options.flags = 0;
+ mt_options.timeout = 300; // Using the same setting as the xz cmd line
+ mt_options.preset = LZMA_PRESET_DEFAULT;
+ mt_options.filters = NULL;
+ mt_options.check = LZMA_CHECK_CRC64;
+ mt_options.threads = lzma_cputhreads();
+ mt_options.block_size = 0;
+ if (mt_options.threads == 0)
+ mt_options.threads = 1;
+ // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+ // number of threads.
+ return lzma_stream_encoder_mt(&strm, &mt_options);
+ }) {}
+};
+#endif
+
struct BzipSink : CompressionSink
{
Sink & nextSink;
@@ -475,13 +475,18 @@ struct BrotliSink : CompressionSink
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
{
- if (parallel && method != "xz")
+ if (parallel) {
+#ifdef HAVE_LZMA_MT
+ if (method == "xz")
+ return make_ref<ParallelXzSink>(nextSink);
+#endif
printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
+ }
if (method == "none")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
- return make_ref<XzSink>(nextSink, parallel);
+ return make_ref<XzSink>(nextSink);
else if (method == "bzip2")
return make_ref<BzipSink>(nextSink);
else if (method == "br")