aboutsummaryrefslogtreecommitdiff
path: root/src/libutil
diff options
context:
space:
mode:
Diffstat (limited to 'src/libutil')
-rw-r--r--src/libutil/archive.cc156
-rw-r--r--src/libutil/archive.hh72
-rw-r--r--src/libutil/async-collect.hh101
-rw-r--r--src/libutil/async-semaphore.hh122
-rw-r--r--src/libutil/compression.cc30
-rw-r--r--src/libutil/experimental-features.cc2
-rw-r--r--src/libutil/file-system.hh2
-rw-r--r--src/libutil/fmt.hh14
-rw-r--r--src/libutil/meson.build2
-rw-r--r--src/libutil/serialise.hh5
10 files changed, 405 insertions, 101 deletions
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index d4da18f14..225483804 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -334,7 +334,7 @@ Generator<Entry> parse(Source & source)
}
-static WireFormatGenerator restore(ParseSink & sink, Generator<nar::Entry> nar)
+static WireFormatGenerator restore(NARParseVisitor & sink, Generator<nar::Entry> nar)
{
while (auto entry = nar.next()) {
co_yield std::visit(
@@ -347,16 +347,13 @@ static WireFormatGenerator restore(ParseSink & sink, Generator<nar::Entry> nar)
},
[&](nar::File f) {
return [](auto f, auto & sink) -> WireFormatGenerator {
- sink.createRegularFile(f.path);
- sink.preallocateContents(f.size);
- if (f.executable) {
- sink.isExecutable();
- }
+ auto handle = sink.createRegularFile(f.path, f.size, f.executable);
+
while (auto block = f.contents.next()) {
- sink.receiveContents(std::string_view{block->data(), block->size()});
+ handle->receiveContents(std::string_view{block->data(), block->size()});
co_yield *block;
}
- sink.closeRegularFile();
+ handle->close();
}(std::move(f), sink);
},
[&](nar::Symlink sl) {
@@ -377,12 +374,12 @@ static WireFormatGenerator restore(ParseSink & sink, Generator<nar::Entry> nar)
}
}
-WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source)
+WireFormatGenerator parseAndCopyDump(NARParseVisitor & sink, Source & source)
{
return restore(sink, nar::parse(source));
}
-void parseDump(ParseSink & sink, Source & source)
+void parseDump(NARParseVisitor & sink, Source & source)
{
auto parser = parseAndCopyDump(sink, source);
while (parser.next()) {
@@ -390,11 +387,99 @@ void parseDump(ParseSink & sink, Source & source)
}
}
-struct RestoreSink : ParseSink
+/*
+ * Note [NAR restoration security]:
+ * It's *critical* that NAR restoration will never overwrite anything even if
+ * duplicate filenames are passed in. It is inevitable that not all NARs are
+ * fit to actually successfully restore to the target filesystem; errors may
+ * occur due to collisions, and this *must* cause the NAR to be rejected.
+ *
+ * Although the filenames are blocked from being *the same bytes* by a higher
+ * layer, filesystems have other ideas on every platform:
+ * - The store may be on a case-insensitive filesystem like APFS, ext4 with
+ * casefold directories, zfs with casesensitivity=insensitive
+ * - The store may be on a Unicode normalizing (or normalization-insensitive)
+ * filesystem like APFS (where files are looked up by
+ * hash(normalize(fname))), HFS+ (where file names are always normalized to
+ * approximately NFD), or zfs with normalization=formC, etc.
+ *
+ * It is impossible to know the version of Unicode being used by the underlying
+ * filesystem, thus it is *impossible* to stop these collisions.
+ *
+ * Overwriting files as a result of invalid NARs will cause a security bug like
+ * CppNix's CVE-2024-45593 (GHSA-h4vv-h3jq-v493)
+ */
+
+/**
+ * This code restores NARs from disk.
+ *
+ * See Note [NAR restoration security] for security invariants in this procedure.
+ *
+ */
+struct NARRestoreVisitor : NARParseVisitor
{
Path dstPath;
- AutoCloseFD fd;
+private:
+ class MyFileHandle : public FileHandle
+ {
+ AutoCloseFD fd;
+
+ MyFileHandle(AutoCloseFD && fd, uint64_t size, bool executable) : FileHandle(), fd(std::move(fd))
+ {
+ if (executable) {
+ makeExecutable();
+ }
+
+ maybePreallocateContents(size);
+ }
+
+ void makeExecutable()
+ {
+ struct stat st;
+ if (fstat(fd.get(), &st) == -1)
+ throw SysError("fstat");
+ if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
+ throw SysError("fchmod");
+ }
+
+ void maybePreallocateContents(uint64_t len)
+ {
+ if (!archiveSettings.preallocateContents)
+ return;
+
+#if HAVE_POSIX_FALLOCATE
+ if (len) {
+ errno = posix_fallocate(fd.get(), 0, len);
+ /* Note that EINVAL may indicate that the underlying
+ filesystem doesn't support preallocation (e.g. on
+ OpenSolaris). Since preallocation is just an
+ optimisation, ignore it. */
+ if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
+ throw SysError("preallocating file of %1% bytes", len);
+ }
+#endif
+ }
+
+ public:
+
+ ~MyFileHandle() = default;
+
+ virtual void close() override
+ {
+ /* Call close explicitly to make sure the error is checked */
+ fd.close();
+ }
+
+ void receiveContents(std::string_view data) override
+ {
+ writeFull(fd.get(), data);
+ }
+
+ friend struct NARRestoreVisitor;
+ };
+
+public:
void createDirectory(const Path & path) override
{
Path p = dstPath + path;
@@ -402,49 +487,13 @@ struct RestoreSink : ParseSink
throw SysError("creating directory '%1%'", p);
};
- void createRegularFile(const Path & path) override
+ std::unique_ptr<FileHandle> createRegularFile(const Path & path, uint64_t size, bool executable) override
{
Path p = dstPath + path;
- fd = AutoCloseFD{open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666)};
+ AutoCloseFD fd = AutoCloseFD{open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666)};
if (!fd) throw SysError("creating file '%1%'", p);
- }
-
- void closeRegularFile() override
- {
- /* Call close explicitly to make sure the error is checked */
- fd.close();
- }
-
- void isExecutable() override
- {
- struct stat st;
- if (fstat(fd.get(), &st) == -1)
- throw SysError("fstat");
- if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
- throw SysError("fchmod");
- }
-
- void preallocateContents(uint64_t len) override
- {
- if (!archiveSettings.preallocateContents)
- return;
-
-#if HAVE_POSIX_FALLOCATE
- if (len) {
- errno = posix_fallocate(fd.get(), 0, len);
- /* Note that EINVAL may indicate that the underlying
- filesystem doesn't support preallocation (e.g. on
- OpenSolaris). Since preallocation is just an
- optimisation, ignore it. */
- if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
- throw SysError("preallocating file of %1% bytes", len);
- }
-#endif
- }
- void receiveContents(std::string_view data) override
- {
- writeFull(fd.get(), data);
+ return std::unique_ptr<MyFileHandle>(new MyFileHandle(std::move(fd), size, executable));
}
void createSymlink(const Path & path, const std::string & target) override
@@ -457,7 +506,7 @@ struct RestoreSink : ParseSink
void restorePath(const Path & path, Source & source)
{
- RestoreSink sink;
+ NARRestoreVisitor sink;
sink.dstPath = path;
parseDump(sink, source);
}
@@ -468,10 +517,9 @@ WireFormatGenerator copyNAR(Source & source)
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
- static ParseSink parseSink; /* null sink; just parse the NAR */
+ static NARParseVisitor parseSink; /* null sink; just parse the NAR */
return parseAndCopyDump(parseSink, source);
}
-
}
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index b34d06e3d..c633bee00 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -76,45 +76,47 @@ WireFormatGenerator dumpString(std::string_view s);
/**
* \todo Fix this API, it sucks.
+ * A visitor for NAR parsing that performs filesystem (or virtual-filesystem)
+ * actions to restore a NAR.
+ *
+ * Methods of this may arbitrarily fail due to filename collisions.
*/
-struct ParseSink
-{
- virtual void createDirectory(const Path & path) { };
-
- virtual void createRegularFile(const Path & path) { };
- virtual void closeRegularFile() { };
- virtual void isExecutable() { };
- virtual void preallocateContents(uint64_t size) { };
- virtual void receiveContents(std::string_view data) { };
-
- virtual void createSymlink(const Path & path, const std::string & target) { };
-};
-
-/**
- * If the NAR archive contains a single file at top-level, then save
- * the contents of the file to `s`. Otherwise barf.
- */
-struct RetrieveRegularNARSink : ParseSink
+struct NARParseVisitor
{
- bool regular = true;
- Sink & sink;
-
- RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
-
- void createDirectory(const Path & path) override
+ /**
+ * A type-erased file handle specific to this particular NARParseVisitor.
+ */
+ struct FileHandle
{
- regular = false;
- }
-
- void receiveContents(std::string_view data) override
+ FileHandle() {}
+ FileHandle(FileHandle const &) = delete;
+ FileHandle & operator=(FileHandle &) = delete;
+
+ /** Puts one block of data into the file */
+ virtual void receiveContents(std::string_view data) { }
+
+ /**
+ * Explicitly closes the file. Further operations may throw an assert.
+ * This exists so that closing can fail and throw an exception without doing so in a destructor.
+ */
+ virtual void close() { }
+
+ virtual ~FileHandle() = default;
+ };
+
+ virtual void createDirectory(const Path & path) { }
+
+ /**
+ * Creates a regular file in the extraction output with the given size and executable flag.
+ * The size is guaranteed to be the true size of the file.
+ */
+ [[nodiscard]]
+ virtual std::unique_ptr<FileHandle> createRegularFile(const Path & path, uint64_t size, bool executable)
{
- sink(data);
+ return std::make_unique<FileHandle>();
}
- void createSymlink(const Path & path, const std::string & target) override
- {
- regular = false;
- }
+ virtual void createSymlink(const Path & path, const std::string & target) { }
};
namespace nar {
@@ -160,8 +162,8 @@ Generator<Entry> parse(Source & source);
}
-WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source);
-void parseDump(ParseSink & sink, Source & source);
+WireFormatGenerator parseAndCopyDump(NARParseVisitor & sink, Source & source);
+void parseDump(NARParseVisitor & sink, Source & source);
void restorePath(const Path & path, Source & source);
diff --git a/src/libutil/async-collect.hh b/src/libutil/async-collect.hh
new file mode 100644
index 000000000..9e0b8bad9
--- /dev/null
+++ b/src/libutil/async-collect.hh
@@ -0,0 +1,101 @@
+#pragma once
+/// @file
+
+#include <kj/async.h>
+#include <kj/common.h>
+#include <kj/vector.h>
+#include <list>
+#include <optional>
+#include <type_traits>
+
+namespace nix {
+
+template<typename K, typename V>
+class AsyncCollect
+{
+public:
+ using Item = std::conditional_t<std::is_void_v<V>, K, std::pair<K, V>>;
+
+private:
+ kj::ForkedPromise<void> allPromises;
+ std::list<Item> results;
+ size_t remaining;
+
+ kj::ForkedPromise<void> signal;
+ kj::Maybe<kj::Own<kj::PromiseFulfiller<void>>> notify;
+
+ void oneDone(Item item)
+ {
+ results.emplace_back(std::move(item));
+ remaining -= 1;
+ KJ_IF_MAYBE (n, notify) {
+ (*n)->fulfill();
+ notify = nullptr;
+ }
+ }
+
+ kj::Promise<void> collectorFor(K key, kj::Promise<V> promise)
+ {
+ if constexpr (std::is_void_v<V>) {
+ return promise.then([this, key{std::move(key)}] { oneDone(std::move(key)); });
+ } else {
+ return promise.then([this, key{std::move(key)}](V v) {
+ oneDone(Item{std::move(key), std::move(v)});
+ });
+ }
+ }
+
+ kj::ForkedPromise<void> waitForAll(kj::Array<std::pair<K, kj::Promise<V>>> & promises)
+ {
+ kj::Vector<kj::Promise<void>> wrappers;
+ for (auto & [key, promise] : promises) {
+ wrappers.add(collectorFor(std::move(key), std::move(promise)));
+ }
+
+ return kj::joinPromisesFailFast(wrappers.releaseAsArray()).fork();
+ }
+
+public:
+ AsyncCollect(kj::Array<std::pair<K, kj::Promise<V>>> && promises)
+ : allPromises(waitForAll(promises))
+ , remaining(promises.size())
+ , signal{nullptr}
+ {
+ }
+
+ kj::Promise<std::optional<Item>> next()
+ {
+ if (remaining == 0 && results.empty()) {
+ return {std::nullopt};
+ }
+
+ if (!results.empty()) {
+ auto result = std::move(results.front());
+ results.pop_front();
+ return {{std::move(result)}};
+ }
+
+ if (notify == nullptr) {
+ auto pair = kj::newPromiseAndFulfiller<void>();
+ notify = std::move(pair.fulfiller);
+ signal = pair.promise.fork();
+ }
+
+ return signal.addBranch().exclusiveJoin(allPromises.addBranch()).then([this] {
+ return next();
+ });
+ }
+};
+
+/**
+ * Collect the results of a list of promises, in order of completion.
+ * Once any input promise is rejected all promises that have not been
+ * resolved or rejected will be cancelled and the exception rethrown.
+ */
+template<typename K, typename V>
+AsyncCollect<K, V> asyncCollect(kj::Array<std::pair<K, kj::Promise<V>>> promises)
+{
+ return AsyncCollect<K, V>(std::move(promises));
+}
+
+}
diff --git a/src/libutil/async-semaphore.hh b/src/libutil/async-semaphore.hh
new file mode 100644
index 000000000..f8db31a68
--- /dev/null
+++ b/src/libutil/async-semaphore.hh
@@ -0,0 +1,122 @@
+#pragma once
+/// @file
+/// @brief A semaphore implementation usable from within a KJ event loop.
+
+#include <cassert>
+#include <kj/async.h>
+#include <kj/common.h>
+#include <kj/exception.h>
+#include <kj/list.h>
+#include <kj/source-location.h>
+#include <memory>
+#include <optional>
+
+namespace nix {
+
+class AsyncSemaphore
+{
+public:
+ class [[nodiscard("destroying a semaphore guard releases the semaphore immediately")]] Token
+ {
+ struct Release
+ {
+ void operator()(AsyncSemaphore * sem) const
+ {
+ sem->unsafeRelease();
+ }
+ };
+
+ std::unique_ptr<AsyncSemaphore, Release> parent;
+
+ public:
+ Token() = default;
+ Token(AsyncSemaphore & parent, kj::Badge<AsyncSemaphore>) : parent(&parent) {}
+
+ bool valid() const
+ {
+ return parent != nullptr;
+ }
+ };
+
+private:
+ struct Waiter
+ {
+ kj::PromiseFulfiller<Token> & fulfiller;
+ kj::ListLink<Waiter> link;
+ kj::List<Waiter, &Waiter::link> & list;
+
+ Waiter(kj::PromiseFulfiller<Token> & fulfiller, kj::List<Waiter, &Waiter::link> & list)
+ : fulfiller(fulfiller)
+ , list(list)
+ {
+ list.add(*this);
+ }
+
+ ~Waiter()
+ {
+ if (link.isLinked()) {
+ list.remove(*this);
+ }
+ }
+ };
+
+ const unsigned capacity_;
+ unsigned used_ = 0;
+ kj::List<Waiter, &Waiter::link> waiters;
+
+ void unsafeRelease()
+ {
+ used_ -= 1;
+ while (used_ < capacity_ && !waiters.empty()) {
+ used_ += 1;
+ auto & w = waiters.front();
+ w.fulfiller.fulfill(Token{*this, {}});
+ waiters.remove(w);
+ }
+ }
+
+public:
+ explicit AsyncSemaphore(unsigned capacity) : capacity_(capacity) {}
+
+ KJ_DISALLOW_COPY_AND_MOVE(AsyncSemaphore);
+
+ ~AsyncSemaphore()
+ {
+ assert(waiters.empty() && "destroyed a semaphore with active waiters");
+ }
+
+ std::optional<Token> tryAcquire()
+ {
+ if (used_ < capacity_) {
+ used_ += 1;
+ return Token{*this, {}};
+ } else {
+ return {};
+ }
+ }
+
+ kj::Promise<Token> acquire()
+ {
+ if (auto t = tryAcquire()) {
+ return std::move(*t);
+ } else {
+ return kj::newAdaptedPromise<Token, Waiter>(waiters);
+ }
+ }
+
+ unsigned capacity() const
+ {
+ return capacity_;
+ }
+
+ unsigned used() const
+ {
+ return used_;
+ }
+
+ unsigned available() const
+ {
+ return capacity_ - used_;
+ }
+};
+}
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 5152a2146..51c820d55 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -144,6 +144,7 @@ struct BrotliDecompressionSource : Source
std::unique_ptr<char[]> buf;
size_t avail_in = 0;
const uint8_t * next_in;
+ std::exception_ptr inputEofException = nullptr;
Source * inner;
std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState *)> state;
@@ -167,23 +168,42 @@ struct BrotliDecompressionSource : Source
while (len && !BrotliDecoderIsFinished(state.get())) {
checkInterrupt();
- while (avail_in == 0) {
+ while (avail_in == 0 && inputEofException == nullptr) {
try {
avail_in = inner->read(buf.get(), BUF_SIZE);
} catch (EndOfFile &) {
+ // No more data, but brotli may still have output remaining
+ // from the last call.
+ inputEofException = std::current_exception();
break;
}
next_in = charptr_cast<const uint8_t *>(buf.get());
}
- if (!BrotliDecoderDecompressStream(
- state.get(), &avail_in, &next_in, &len, &out, nullptr
- ))
- {
+ BrotliDecoderResult res = BrotliDecoderDecompressStream(
+ state.get(), &avail_in, &next_in, &len, &out, nullptr
+ );
+
+ switch (res) {
+ case BROTLI_DECODER_RESULT_SUCCESS:
+ // We're done here!
+ goto finish;
+ case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+ // Grab more input. Don't try if we already have exhausted our input stream.
+ if (inputEofException != nullptr) {
+ std::rethrow_exception(inputEofException);
+ } else {
+ continue;
+ }
+ case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+ // Need more output space: we can only get another buffer by someone calling us again, so get out.
+ goto finish;
+ case BROTLI_DECODER_RESULT_ERROR:
throw CompressionError("error while decompressing brotli file");
}
}
+finish:
if (begin != out) {
return out - begin;
} else {
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 35982c28c..bb7a95a24 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -247,7 +247,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.tag = Xp::ReplAutomation,
.name = "repl-automation",
.description = R"(
- Makes the repl not use readline/editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline.
+ Makes the repl not use editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline.
)",
},
}};
diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh
index d95e8eba5..2547c63b5 100644
--- a/src/libutil/file-system.hh
+++ b/src/libutil/file-system.hh
@@ -210,7 +210,7 @@ inline Paths createDirs(PathView path)
}
/**
- * Create a symlink.
+ * Create a symlink. Throws if the symlink exists.
*/
void createSymlink(const Path & target, const Path & link);
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index ee3e1e2e7..5feefdf90 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -136,11 +136,17 @@ inline std::string fmt(const char * s)
template<typename... Args>
inline std::string fmt(const std::string & fs, const Args &... args)
-{
+try {
boost::format f(fs);
fmt_internal::setExceptions(f);
(f % ... % args);
return f.str();
+} catch (boost::io::format_error & fe) {
+ // I don't care who catches this, we do not put up with boost format errors
+ // Give me a stack trace and a core dump
+ std::cerr << "nix::fmt threw format error. Original format string: '";
+ std::cerr << fs << "'; number of arguments: " << sizeof...(args) << "\n";
+ std::terminate();
}
/**
@@ -174,15 +180,13 @@ public:
std::cerr << "HintFmt received incorrect number of format args. Original format string: '";
std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n";
// And regardless of the coredump give me a damn stacktrace.
- printStackTrace();
- abort();
+ std::terminate();
}
} catch (boost::io::format_error & ex) {
// Same thing, but for anything that happens in the member initializers.
std::cerr << "HintFmt received incorrect format string. Original format string: '";
std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n";
- printStackTrace();
- abort();
+ std::terminate();
}
HintFmt(const HintFmt & hf) : fmt(hf.fmt) {}
diff --git a/src/libutil/meson.build b/src/libutil/meson.build
index a3f21de59..afca4e021 100644
--- a/src/libutil/meson.build
+++ b/src/libutil/meson.build
@@ -53,6 +53,8 @@ libutil_headers = files(
'archive.hh',
'args/root.hh',
'args.hh',
+ 'async-collect.hh',
+ 'async-semaphore.hh',
'backed-string-view.hh',
'box_ptr.hh',
'canon-path.hh',
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 612658b2d..3a9685e0e 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -77,6 +77,11 @@ struct Source
* Store up to ‘len’ in the buffer pointed to by ‘data’, and
* return the number of bytes stored. It blocks until at least
* one byte is available.
+ *
+ * Should not return 0 (generally you want to throw EndOfFile), but nothing
+ * stops that.
+ *
+ * \throws EndOfFile if there is no more data.
*/
virtual size_t read(char * data, size_t len) = 0;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 12:54:43 +0000