Fix bzip2 compression of files > 4 GiB

Bzip2's 'avail_in' parameter is declared as an unsigned int, so
assigning a size_t length to it led to silent truncation.

Fixes #2111.

1 files changed, 14 insertions, 3 deletions

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 81cb5e98c..e1782f8c4 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -369,7 +369,20 @@ struct BzipSink : CompressionSink
 
     void write(const unsigned char * data, size_t len) override
     {
+        /* Bzip2's 'avail_in' parameter is an unsigned int, so we need
+           to split the input into chunks of at most 4 GiB. */
+        while (len) {
+            auto n = std::min((size_t) std::numeric_limits<decltype(strm.avail_in)>::max(), len);
+            writeInternal(data, n);
+            data += n;
+            len -= n;
+        }
+    }
+
+    void writeInternal(const unsigned char * data, size_t len)
+    {
         assert(!finished);
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
         strm.next_in = (char *) data;
         strm.avail_in = len;
@@ -475,8 +488,6 @@ struct BrotliSink : CompressionSink
 
     void write(const unsigned char * data, size_t len) override
     {
-        assert(!finished);
-
         // Don't feed brotli too much at once
         const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
         while (len) {
@@ -486,7 +497,7 @@ struct BrotliSink : CompressionSink
           len -= n;
         }
     }
-  private:
+
     void writeInternal(const unsigned char * data, size_t len)
     {
         assert(!finished);