From e8186085e07104d4b844208613c2d704b5b57dec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Mar 2017 14:40:15 +0100 Subject: Add support for brotli compression Build logs on cache.nixos.org are compressed using Brotli (since this allows them to be decompressed automatically by Chrome and Firefox), so it's handy if "nix log" can decompress them. --- Makefile.config.in | 1 + configure.ac | 1 + release.nix | 8 ++++++-- shell.nix | 3 ++- src/libstore/download.cc | 21 +++++++++++++++++++-- src/libutil/compression.cc | 7 +++++++ src/libutil/local.mk | 2 ++ 7 files changed, 38 insertions(+), 5 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index d4953b521..fccf63b36 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -14,6 +14,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +bro = @bro@ datadir = @datadir@ datarootdir = @datarootdir@ docdir = @docdir@ diff --git a/configure.ac b/configure.ac index 34dcd6b2a..21ca78af0 100644 --- a/configure.ac +++ b/configure.ac @@ -128,6 +128,7 @@ NEED_PROG(gzip, gzip) NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(pv, pv, pv) +NEED_PROG(bro, bro) # Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). diff --git a/release.nix b/release.nix index e61e81bdf..a266af7c2 100644 --- a/release.nix +++ b/release.nix @@ -24,7 +24,8 @@ let inherit officialRelease; buildInputs = - [ curl bison flex perl libxml2 libxslt bzip2 xz + [ curl bison flex perl libxml2 libxslt + bzip2 xz brotli pkgconfig sqlite libsodium boehmgc docbook5 docbook5_xsl autoconf-archive @@ -73,7 +74,10 @@ let src = tarball; buildInputs = - [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ] + [ curl perl + bzip2 xz brotli + openssl pkgconfig sqlite boehmgc + ] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) (aws-sdk-cpp.override { diff --git a/shell.nix b/shell.nix index 4c1608230..df0ad01df 100644 --- a/shell.nix +++ b/shell.nix @@ -6,7 +6,8 @@ with import {}; name = "nix"; buildInputs = - [ curl bison flex perl libxml2 libxslt bzip2 xz + [ curl bison flex perl libxml2 libxslt + bzip2 xz brotli pkgconfig sqlite libsodium boehmgc docbook5 docbook5_xsl autoconf-archive diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 6567a4dc4..d9b8fbc08 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -5,6 +5,8 @@ #include "store-api.hh" #include "archive.hh" #include "s3.hh" +#include "compression.hh" + #ifdef ENABLE_S3 #include #endif @@ -70,6 +72,8 @@ struct CurlDownloader : public Downloader struct curl_slist * requestHeaders = 0; + std::string encoding; + DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) : downloader(downloader), request(request) { @@ -127,6 +131,7 @@ struct CurlDownloader : public Downloader auto ss = tokenizeString>(line, " "); status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared(); + encoding = ""; } else { auto i = line.find(':'); if (i != string::npos) { @@ -142,7 +147,8 @@ struct CurlDownloader : public Downloader debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } - } + } else if (name == "content-encoding") + encoding = trim(string(line, i + 1));; } } return realSize; @@ -268,7 +274,18 @@ struct CurlDownloader : public Downloader { result.cached = httpStatus == 304; done = true; - callSuccess(success, failure, const_cast(result)); + + /* Ad hoc support for brotli, since curl doesn't do + this yet. */ + try { + if (encoding == "br") + result.data = decompress("br", *result.data); + + callSuccess(success, failure, const_cast(result)); + } catch (...) { + done = true; + callFailure(failure, std::current_exception()); + } } else { Error err = (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound : diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a3bbb5170..723b072af 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -89,6 +89,11 @@ static ref decompressBzip2(const std::string & in) } } +static ref decompressBrotli(const std::string & in) +{ + return make_ref(runProgram(BRO, true, {"-d"}, in)); +} + ref compress(const std::string & method, const std::string & in) { StringSink ssink; @@ -106,6 +111,8 @@ ref decompress(const std::string & method, const std::string & in) return decompressXZ(in); else if (method == "bzip2") return decompressBzip2(in); + else if (method == "br") + return decompressBrotli(in); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index cac5c8795..0721b21c2 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat + +libutil_CXXFLAGS = -DBRO=\"$(bro)\" -- cgit v1.2.3