diff options
| -rw-r--r-- | configure.ac | 2 | ||||
| -rw-r--r-- | src/libutil/compression.cc | 60 |
2 files changed, 39 insertions, 23 deletions
diff --git a/configure.ac b/configure.ac index 14f742cf3..4102f3216 100644 --- a/configure.ac +++ b/configure.ac @@ -175,6 +175,8 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium]) # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) +AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], + [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) # Look for libbrotli{enc,dec}, optional dependencies diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ed15761b3..470c925ed 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,7 @@ #include "compression.hh" #include "util.hh" #include "finally.hh" +#include "logging.hh" #include <lzma.h> #include <bzlib.h> @@ -189,28 +190,9 @@ struct XzSink : CompressionSink lzma_stream strm = LZMA_STREAM_INIT; bool finished = false; - XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink) - { - lzma_ret ret; - if (parallel) { - lzma_mt mt_options = {}; - mt_options.flags = 0; - mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; - mt_options.filters = NULL; - mt_options.check = LZMA_CHECK_CRC64; - mt_options.threads = lzma_cputhreads(); - mt_options.block_size = 0; - if (mt_options.threads == 0) - mt_options.threads = 1; - // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the - // number of threads. - ret = lzma_stream_encoder_mt( - &strm, &mt_options); - } else - ret = lzma_easy_encoder( - &strm, 6, LZMA_CHECK_CRC64); - + template <typename F> + XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) { + lzma_ret ret = initEncoder(); if (ret != LZMA_OK) throw CompressionError("unable to initialise lzma encoder"); // FIXME: apply the x86 BCJ filter? @@ -218,6 +200,9 @@ struct XzSink : CompressionSink strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } + XzSink(Sink & nextSink) : XzSink(nextSink, [this]() { + return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + }) {} ~XzSink() { @@ -271,6 +256,27 @@ struct XzSink : CompressionSink } }; +#ifdef HAVE_LZMA_MT +struct ParallelXzSink : public XzSink +{ + ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() { + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + return lzma_stream_encoder_mt(&strm, &mt_options); + }) {} +}; +#endif + struct BzipSink : CompressionSink { Sink & nextSink; @@ -469,10 +475,18 @@ struct BrotliSink : CompressionSink ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { + if (parallel) { +#ifdef HAVE_LZMA_MT + if (method == "xz") + return make_ref<ParallelXzSink>(nextSink); +#endif + printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); + } + if (method == "none") return make_ref<NoneSink>(nextSink); else if (method == "xz") - return make_ref<XzSink>(nextSink, parallel); + return make_ref<XzSink>(nextSink); else if (method == "bzip2") return make_ref<BzipSink>(nextSink); else if (method == "br") |