From c4d903ddb009aa6472530699e154d85a24eac51d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 20:55:53 +0100 Subject: Fix memory corruption caused by GC-invisible coroutine stacks Crucially this introduces BoehmGCStackAllocator, but it also adds a bunch of wiring to avoid making libutil depend on bdw-gc. Part of the solutions for #4178, #4200 --- src/libutil/serialise.cc | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'src/libutil/serialise.cc') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5c9f6f901..28f6968d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,6 +171,39 @@ size_t StringSource::read(unsigned char * data, size_t len) #error Coroutines are broken in this version of Boost! #endif +/* A concrete datatype allow virtual dispatch of stack allocation methods. */ +struct VirtualStackAllocator { + StackAllocator *allocator = StackAllocator::defaultAllocator; + + boost::context::stack_context allocate() { + return allocator->allocate(); + } + + void deallocate(boost::context::stack_context sctx) { + allocator->deallocate(sctx); + } +}; + + +/* This class reifies the default boost coroutine stack allocation strategy with + a virtual interface. */ +class DefaultStackAllocator : public StackAllocator { + boost::coroutines2::default_stack stack; + + boost::context::stack_context allocate() { + return stack.allocate(); + } + + void deallocate(boost::context::stack_context sctx) { + deallocate(sctx); + } +}; + +static DefaultStackAllocator defaultAllocatorSingleton; + +StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; + + std::unique_ptr sinkToSource( std::function fun, std::function eof) @@ -195,7 +228,7 @@ std::unique_ptr sinkToSource( size_t read(unsigned char * data, size_t len) override { if (!coro) - coro = coro_t::pull_type([&](coro_t::push_type & yield) { + coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](const unsigned char * data, size_t len) { if (len) yield(std::string((const char *) data, len)); }); -- cgit v1.2.3 From 108a2dab7e460533064b24f5dff05adc453acb27 Mon Sep 17 00:00:00 2001 From: "Ricardo M. Correia" Date: Tue, 10 Nov 2020 04:24:55 +0100 Subject: Fix stack overflow introduced in #4206 --- src/libutil/serialise.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil/serialise.cc') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 28f6968d0..038ede049 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -195,7 +195,7 @@ class DefaultStackAllocator : public StackAllocator { } void deallocate(boost::context::stack_context sctx) { - deallocate(sctx); + stack.deallocate(sctx); } }; -- cgit v1.2.3 From faa31f40846f7a4dbc2487d000b112a6aef69d1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:00:43 +0100 Subject: Sink: Use std::string_view --- src/libutil/serialise.cc | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'src/libutil/serialise.cc') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 038ede049..534ad5cbb 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -11,23 +11,23 @@ namespace nix { -void BufferedSink::operator () (const unsigned char * data, size_t len) +void BufferedSink::operator () (std::string_view data) { - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + if (!buffer) buffer = decltype(buffer)(new char[bufSize]); - while (len) { + while (!data.empty()) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ - if (bufPos + len >= bufSize) { + if (bufPos + data.size() >= bufSize) { flush(); - write(data, len); + write(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ - size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; - memcpy(buffer.get() + bufPos, data, n); - data += n; bufPos += n; len -= n; + size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size(); + memcpy(buffer.get() + bufPos, data.data(), n); + data.remove_prefix(n); bufPos += n; if (bufPos == bufSize) flush(); } } @@ -38,7 +38,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write(buffer.get(), n); + write({buffer.get(), n}); } @@ -59,9 +59,9 @@ static void warnLargeDump() } -void FdSink::write(const unsigned char * data, size_t len) +void FdSink::write(std::string_view data) { - written += len; + written += data.size(); static bool warned = false; if (warn && !warned) { if (written > threshold) { @@ -70,7 +70,7 @@ void FdSink::write(const unsigned char * data, size_t len) } } try { - writeFull(fd, data, len); + writeFull(fd, data); } catch (SysError & e) { _good = false; throw; @@ -101,7 +101,7 @@ void Source::drainInto(Sink & sink) size_t n; try { n = read(buf.data(), buf.size()); - sink(buf.data(), n); + sink({(char *) buf.data(), n}); } catch (EndOfFile &) { break; } @@ -229,9 +229,9 @@ std::unique_ptr sinkToSource( { if (!coro) coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); - }); + LambdaSink sink([&](std::string_view data) { + if (!data.empty()) yield(std::string(data)); + }); fun(sink); }); @@ -260,22 +260,22 @@ void writePadding(size_t len, Sink & sink) if (len % 8) { unsigned char zero[8]; memset(zero, 0, sizeof(zero)); - sink(zero, 8 - (len % 8)); + sink({(char *) zero, 8 - (len % 8)}); } } -void writeString(const unsigned char * buf, size_t len, Sink & sink) +void writeString(std::string_view data, Sink & sink) { - sink << len; - sink(buf, len); - writePadding(len, sink); + sink << data.size(); + sink(data); + writePadding(data.size(), sink); } Sink & operator << (Sink & sink, const string & s) { - writeString((const unsigned char *) s.data(), s.size(), sink); + writeString(s, sink); return sink; } @@ -394,14 +394,14 @@ Error readError(Source & source) } -void StringSink::operator () (const unsigned char * data, size_t len) +void StringSink::operator () (std::string_view data) { static bool warned = false; if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } - s->append((const char *) data, len); + s->append(data); } size_t ChainSource::read(unsigned char * data, size_t len) -- cgit v1.2.3 From 1b79b5b983a6c775766bd0d1c7881042188998b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:10:56 +0100 Subject: read(): Use char * instead of unsigned char * This gets rid of some pointless casts. --- src/libutil/serialise.cc | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'src/libutil/serialise.cc') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 534ad5cbb..87c1099a1 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -84,7 +84,7 @@ bool FdSink::good() } -void Source::operator () (unsigned char * data, size_t len) +void Source::operator () (char * data, size_t len) { while (len) { size_t n = read(data, len); @@ -96,12 +96,12 @@ void Source::operator () (unsigned char * data, size_t len) void Source::drainInto(Sink & sink) { std::string s; - std::vector buf(8192); + std::vector buf(8192); while (true) { size_t n; try { n = read(buf.data(), buf.size()); - sink({(char *) buf.data(), n}); + sink({buf.data(), n}); } catch (EndOfFile &) { break; } @@ -117,9 +117,9 @@ std::string Source::drain() } -size_t BufferedSource::read(unsigned char * data, size_t len) +size_t BufferedSource::read(char * data, size_t len) { - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + if (!buffer) buffer = decltype(buffer)(new char[bufSize]); if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); @@ -138,12 +138,12 @@ bool BufferedSource::hasData() } -size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +size_t FdSource::readUnbuffered(char * data, size_t len) { ssize_t n; do { checkInterrupt(); - n = ::read(fd, (char *) data, len); + n = ::read(fd, data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } @@ -158,10 +158,10 @@ bool FdSource::good() } -size_t StringSource::read(unsigned char * data, size_t len) +size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); - size_t n = s.copy((char *) data, len, pos); + size_t n = s.copy(data, len, pos); pos += n; return n; } @@ -225,7 +225,7 @@ std::unique_ptr sinkToSource( std::string cur; size_t pos = 0; - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { if (!coro) coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { @@ -244,7 +244,7 @@ std::unique_ptr sinkToSource( } auto n = std::min(cur.size() - pos, len); - memcpy(data, (unsigned char *) cur.data() + pos, n); + memcpy(data, cur.data() + pos, n); pos += n; return n; @@ -258,9 +258,9 @@ std::unique_ptr sinkToSource( void writePadding(size_t len, Sink & sink) { if (len % 8) { - unsigned char zero[8]; + char zero[8]; memset(zero, 0, sizeof(zero)); - sink({(char *) zero, 8 - (len % 8)}); + sink({zero, 8 - (len % 8)}); } } @@ -321,7 +321,7 @@ Sink & operator << (Sink & sink, const Error & ex) void readPadding(size_t len, Source & source) { if (len % 8) { - unsigned char zero[8]; + char zero[8]; size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) @@ -330,7 +330,7 @@ void readPadding(size_t len, Source & source) } -size_t readString(unsigned char * buf, size_t max, Source & source) +size_t readString(char * buf, size_t max, Source & source) { auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); @@ -345,7 +345,7 @@ string readString(Source & source, size_t max) auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); - source((unsigned char*) res.data(), len); + source(res.data(), len); readPadding(len, source); return res; } @@ -404,7 +404,7 @@ void StringSink::operator () (std::string_view data) s->append(data); } -size_t ChainSource::read(unsigned char * data, size_t len) +size_t ChainSource::read(char * data, size_t len) { if (useSecond) { return source2.read(data, len); -- cgit v1.2.3