aboutsummaryrefslogtreecommitdiff
path: root/src/libutil
diff options
context:
space:
mode:
Diffstat (limited to 'src/libutil')
-rw-r--r--src/libutil/ansicolor.hh2
-rw-r--r--src/libutil/args.cc1
-rw-r--r--src/libutil/args.hh16
-rw-r--r--src/libutil/closure.hh69
-rw-r--r--src/libutil/comparator.hh4
-rw-r--r--src/libutil/compression.cc378
-rw-r--r--src/libutil/compression.hh8
-rw-r--r--src/libutil/config.cc25
-rw-r--r--src/libutil/config.hh13
-rw-r--r--src/libutil/error.cc10
-rw-r--r--src/libutil/fmt.hh2
-rw-r--r--src/libutil/local.mk2
-rw-r--r--src/libutil/logging.cc6
-rw-r--r--src/libutil/ref.hh53
-rw-r--r--src/libutil/serialise.cc57
-rw-r--r--src/libutil/serialise.hh8
-rw-r--r--src/libutil/tarfile.cc130
-rw-r--r--src/libutil/tarfile.hh22
-rw-r--r--src/libutil/tests/closure.cc70
-rw-r--r--src/libutil/tests/compression.cc18
-rw-r--r--src/libutil/tests/logging.cc4
-rw-r--r--src/libutil/url.cc2
-rw-r--r--src/libutil/util.cc125
-rw-r--r--src/libutil/util.hh32
24 files changed, 613 insertions, 444 deletions
diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh
index ae741f867..38305e71c 100644
--- a/src/libutil/ansicolor.hh
+++ b/src/libutil/ansicolor.hh
@@ -9,7 +9,7 @@ namespace nix {
#define ANSI_ITALIC "\e[3m"
#define ANSI_RED "\e[31;1m"
#define ANSI_GREEN "\e[32;1m"
-#define ANSI_YELLOW "\e[33;1m"
+#define ANSI_WARNING "\e[35;1m"
#define ANSI_BLUE "\e[34;1m"
#define ANSI_MAGENTA "\e[35;1m"
#define ANSI_CYAN "\e[36;1m"
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index afed0670f..9df279faf 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -331,6 +331,7 @@ MultiCommand::MultiCommand(const Commands & commands_)
if (i == commands.end())
throw UsageError("'%s' is not a recognised command", s);
command = {s, i->second()};
+ command->second->parent = this;
}}
});
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index c08ba8abd..7521b3065 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -12,6 +12,8 @@ namespace nix {
enum HashType : char;
+class MultiCommand;
+
class Args
{
public:
@@ -89,6 +91,14 @@ protected:
})
, arity(1)
{ }
+
+ template<class I>
+ Handler(std::optional<I> * dest)
+ : fun([=](std::vector<std::string> ss) {
+ *dest = string2IntWithUnitPrefix<I>(ss[0]);
+ })
+ , arity(1)
+ { }
};
/* Options. */
@@ -169,11 +179,13 @@ public:
virtual nlohmann::json toJSON();
friend class MultiCommand;
+
+ MultiCommand * parent = nullptr;
};
/* A command is an argument parser that can be executed by calling its
run() method. */
-struct Command : virtual Args
+struct Command : virtual public Args
{
friend class MultiCommand;
@@ -193,7 +205,7 @@ typedef std::map<std::string, std::function<ref<Command>()>> Commands;
/* An argument parser that supports multiple subcommands,
i.e. ‘<command> <subcommand>’. */
-class MultiCommand : virtual Args
+class MultiCommand : virtual public Args
{
public:
Commands commands;
diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh
new file mode 100644
index 000000000..779b9b2d5
--- /dev/null
+++ b/src/libutil/closure.hh
@@ -0,0 +1,69 @@
+#include <set>
+#include <future>
+#include "sync.hh"
+
+using std::set;
+
+namespace nix {
+
+template<typename T>
+using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
+
+template<typename T>
+void computeClosure(
+ const set<T> startElts,
+ set<T> & res,
+ GetEdgesAsync<T> getEdgesAsync
+)
+{
+ struct State
+ {
+ size_t pending;
+ set<T> & res;
+ std::exception_ptr exc;
+ };
+
+ Sync<State> state_(State{0, res, 0});
+
+ std::function<void(const T &)> enqueue;
+
+ std::condition_variable done;
+
+ enqueue = [&](const T & current) -> void {
+ {
+ auto state(state_.lock());
+ if (state->exc) return;
+ if (!state->res.insert(current).second) return;
+ state->pending++;
+ }
+
+ getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
+ try {
+ auto children = prom.get_future().get();
+ for (auto & child : children)
+ enqueue(child);
+ {
+ auto state(state_.lock());
+ assert(state->pending);
+ if (!--state->pending) done.notify_one();
+ }
+ } catch (...) {
+ auto state(state_.lock());
+ if (!state->exc) state->exc = std::current_exception();
+ assert(state->pending);
+ if (!--state->pending) done.notify_one();
+ };
+ });
+ };
+
+ for (auto & startElt : startElts)
+ enqueue(startElt);
+
+ {
+ auto state(state_.lock());
+ while (state->pending) state.wait(done);
+ if (state->exc) std::rethrow_exception(state->exc);
+ }
+}
+
+}
diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh
index 0315dc506..eecd5b819 100644
--- a/src/libutil/comparator.hh
+++ b/src/libutil/comparator.hh
@@ -25,6 +25,8 @@
}
#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
#define GENERATE_CMP(args...) \
GENERATE_EQUAL(args) \
- GENERATE_LEQ(args)
+ GENERATE_LEQ(args) \
+ GENERATE_NEQ(args)
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 986ba2976..d26f68fde 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -1,18 +1,17 @@
#include "compression.hh"
+#include "tarfile.hh"
#include "util.hh"
#include "finally.hh"
#include "logging.hh"
-#include <lzma.h>
-#include <bzlib.h>
+#include <archive.h>
+#include <archive_entry.h>
#include <cstdio>
#include <cstring>
#include <brotli/decode.h>
#include <brotli/encode.h>
-#include <zlib.h>
-
#include <iostream>
namespace nix {
@@ -27,7 +26,7 @@ struct ChunkedCompressionSink : CompressionSink
const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
while (!data.empty()) {
size_t n = std::min(CHUNK_SIZE, data.size());
- writeInternal(data);
+ writeInternal(data.substr(0, n));
data.remove_prefix(n);
}
}
@@ -35,177 +34,101 @@ struct ChunkedCompressionSink : CompressionSink
virtual void writeInternal(std::string_view data) = 0;
};
-struct NoneSink : CompressionSink
-{
- Sink & nextSink;
- NoneSink(Sink & nextSink) : nextSink(nextSink) { }
- void finish() override { flush(); }
- void write(std::string_view data) override { nextSink(data); }
-};
-
-struct GzipDecompressionSink : CompressionSink
+struct ArchiveDecompressionSource : Source
{
- Sink & nextSink;
- z_stream strm;
- bool finished = false;
- uint8_t outbuf[BUFSIZ];
-
- GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
- {
- strm.zalloc = Z_NULL;
- strm.zfree = Z_NULL;
- strm.opaque = Z_NULL;
- strm.avail_in = 0;
- strm.next_in = Z_NULL;
- strm.next_out = outbuf;
- strm.avail_out = sizeof(outbuf);
-
- // Enable gzip and zlib decoding (+32) with 15 windowBits
- int ret = inflateInit2(&strm,15+32);
- if (ret != Z_OK)
- throw CompressionError("unable to initialise gzip encoder");
- }
-
- ~GzipDecompressionSink()
- {
- inflateEnd(&strm);
- }
-
- void finish() override
- {
- CompressionSink::flush();
- write({});
- }
-
- void write(std::string_view data) override
- {
- assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
- strm.next_in = (Bytef *) data.data();
- strm.avail_in = data.size();
-
- while (!finished && (!data.data() || strm.avail_in)) {
- checkInterrupt();
-
- int ret = inflate(&strm,Z_SYNC_FLUSH);
- if (ret != Z_OK && ret != Z_STREAM_END)
- throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
- zError(ret), data.size(), strm.avail_in);
-
- finished = ret == Z_STREAM_END;
-
- if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
- strm.next_out = (Bytef *) outbuf;
- strm.avail_out = sizeof(outbuf);
+ std::unique_ptr<TarArchive> archive = 0;
+ Source & src;
+ ArchiveDecompressionSource(Source & src) : src(src) {}
+ ~ArchiveDecompressionSource() override {}
+ size_t read(char * data, size_t len) override {
+ struct archive_entry * ae;
+ if (!archive) {
+ archive = std::make_unique<TarArchive>(src, true);
+ this->archive->check(archive_read_next_header(this->archive->archive, &ae),
+ "failed to read header (%s)");
+ if (archive_filter_count(this->archive->archive) < 2) {
+ throw CompressionError("input compression not recognized");
}
}
+ ssize_t result = archive_read_data(this->archive->archive, data, len);
+ if (result > 0) return result;
+ if (result == 0) {
+ throw EndOfFile("reached end of compressed file");
+ }
+ this->archive->check(result, "failed to read compressed data (%s)");
+ return result;
}
};
-struct XzDecompressionSink : CompressionSink
+struct ArchiveCompressionSink : CompressionSink
{
Sink & nextSink;
- uint8_t outbuf[BUFSIZ];
- lzma_stream strm = LZMA_STREAM_INIT;
- bool finished = false;
-
- XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
- {
- lzma_ret ret = lzma_stream_decoder(
- &strm, UINT64_MAX, LZMA_CONCATENATED);
- if (ret != LZMA_OK)
- throw CompressionError("unable to initialise lzma decoder");
-
- strm.next_out = outbuf;
- strm.avail_out = sizeof(outbuf);
+ struct archive * archive;
+
+ ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) {
+ archive = archive_write_new();
+ if (!archive) throw Error("failed to initialize libarchive");
+ check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
+ check(archive_write_set_format_raw(archive));
+ if (format == "xz" && parallel) {
+ check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
+ }
+ // disable internal buffering
+ check(archive_write_set_bytes_per_block(archive, 0));
+ // disable output padding
+ check(archive_write_set_bytes_in_last_block(archive, 1));
+ open();
}
- ~XzDecompressionSink()
+ ~ArchiveCompressionSink() override
{
- lzma_end(&strm);
+ if (archive) archive_write_free(archive);
}
void finish() override
{
- CompressionSink::flush();
- write({});
+ flush();
+ check(archive_write_close(archive));
}
- void write(std::string_view data) override
+ void check(int err, const std::string & reason = "failed to compress (%s)")
{
- strm.next_in = (const unsigned char *) data.data();
- strm.avail_in = data.size();
-
- while (!finished && (!data.data() || strm.avail_in)) {
- checkInterrupt();
-
- lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
- if (ret != LZMA_OK && ret != LZMA_STREAM_END)
- throw CompressionError("error %d while decompressing xz file", ret);
-
- finished = ret == LZMA_STREAM_END;
-
- if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
- strm.next_out = outbuf;
- strm.avail_out = sizeof(outbuf);
- }
- }
+ if (err == ARCHIVE_EOF)
+ throw EndOfFile("reached end of archive");
+ else if (err != ARCHIVE_OK)
+ throw Error(reason, archive_error_string(this->archive));
}
-};
-struct BzipDecompressionSink : ChunkedCompressionSink
-{
- Sink & nextSink;
- bz_stream strm;
- bool finished = false;
-
- BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+ void write(std::string_view data) override
{
- memset(&strm, 0, sizeof(strm));
- int ret = BZ2_bzDecompressInit(&strm, 0, 0);
- if (ret != BZ_OK)
- throw CompressionError("unable to initialise bzip2 decoder");
-
- strm.next_out = (char *) outbuf;
- strm.avail_out = sizeof(outbuf);
+ ssize_t result = archive_write_data(archive, data.data(), data.length());
+ if (result <= 0) check(result);
}
- ~BzipDecompressionSink()
+private:
+ void open()
{
- BZ2_bzDecompressEnd(&strm);
+ check(archive_write_open(archive, this, nullptr, ArchiveCompressionSink::callback_write, nullptr));
+ auto ae = archive_entry_new();
+ archive_entry_set_filetype(ae, AE_IFREG);
+ check(archive_write_header(archive, ae));
+ archive_entry_free(ae);
}
- void finish() override
+ static ssize_t callback_write(struct archive * archive, void * _self, const void * buffer, size_t length)
{
- flush();
- write({});
+ auto self = (ArchiveCompressionSink *) _self;
+ self->nextSink({(const char *) buffer, length});
+ return length;
}
+};
- void writeInternal(std::string_view data) override
- {
- assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
- strm.next_in = (char *) data.data();
- strm.avail_in = data.size();
-
- while (strm.avail_in) {
- checkInterrupt();
-
- int ret = BZ2_bzDecompress(&strm);
- if (ret != BZ_OK && ret != BZ_STREAM_END)
- throw CompressionError("error while decompressing bzip2 file");
-
- finished = ret == BZ_STREAM_END;
-
- if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
- strm.next_out = (char *) outbuf;
- strm.avail_out = sizeof(outbuf);
- }
- }
- }
+struct NoneSink : CompressionSink
+{
+ Sink & nextSink;
+ NoneSink(Sink & nextSink) : nextSink(nextSink) { }
+ void finish() override { flush(); }
+ void write(std::string_view data) override { nextSink(data); }
};
struct BrotliDecompressionSink : ChunkedCompressionSink
@@ -268,159 +191,24 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
return ssink.s;
}
-ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
+std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
{
if (method == "none" || method == "")
- return make_ref<NoneSink>(nextSink);
- else if (method == "xz")
- return make_ref<XzDecompressionSink>(nextSink);
- else if (method == "bzip2")
- return make_ref<BzipDecompressionSink>(nextSink);
- else if (method == "gzip")
- return make_ref<GzipDecompressionSink>(nextSink);
+ return std::make_unique<NoneSink>(nextSink);
else if (method == "br")
- return make_ref<BrotliDecompressionSink>(nextSink);
+ return std::make_unique<BrotliDecompressionSink>(nextSink);
else
- throw UnknownCompressionMethod("unknown compression method '%s'", method);
+ return sourceToSink([&](Source & source) {
+ auto decompressionSource = std::make_unique<ArchiveDecompressionSource>(source);
+ decompressionSource->drainInto(nextSink);
+ });
}
-struct XzCompressionSink : CompressionSink
-{
- Sink & nextSink;
- uint8_t outbuf[BUFSIZ];
- lzma_stream strm = LZMA_STREAM_INIT;
- bool finished = false;
-
- XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
- {
- lzma_ret ret;
- bool done = false;
-
- if (parallel) {
-#ifdef HAVE_LZMA_MT
- lzma_mt mt_options = {};
- mt_options.flags = 0;
- mt_options.timeout = 300; // Using the same setting as the xz cmd line
- mt_options.preset = LZMA_PRESET_DEFAULT;
- mt_options.filters = NULL;
- mt_options.check = LZMA_CHECK_CRC64;
- mt_options.threads = lzma_cputhreads();
- mt_options.block_size = 0;
- if (mt_options.threads == 0)
- mt_options.threads = 1;
- // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
- // number of threads.
- ret = lzma_stream_encoder_mt(&strm, &mt_options);
- done = true;
-#else
- printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
-#endif
- }
-
- if (!done)
- ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
-
- if (ret != LZMA_OK)
- throw CompressionError("unable to initialise lzma encoder");
-
- // FIXME: apply the x86 BCJ filter?
-
- strm.next_out = outbuf;
- strm.avail_out = sizeof(outbuf);
- }
-
- ~XzCompressionSink()
- {
- lzma_end(&strm);
- }
-
- void finish() override
- {
- CompressionSink::flush();
- write({});
- }
-
- void write(std::string_view data) override
- {
- strm.next_in = (const unsigned char *) data.data();
- strm.avail_in = data.size();
-
- while (!finished && (!data.data() || strm.avail_in)) {
- checkInterrupt();
-
- lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
- if (ret != LZMA_OK && ret != LZMA_STREAM_END)
- throw CompressionError("error %d while compressing xz file", ret);
-
- finished = ret == LZMA_STREAM_END;
-
- if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
- strm.next_out = outbuf;
- strm.avail_out = sizeof(outbuf);
- }
- }
- }
-};
-
-struct BzipCompressionSink : ChunkedCompressionSink
-{
- Sink & nextSink;
- bz_stream strm;
- bool finished = false;
-
- BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
- {
- memset(&strm, 0, sizeof(strm));
- int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
- if (ret != BZ_OK)
- throw CompressionError("unable to initialise bzip2 encoder");
-
- strm.next_out = (char *) outbuf;
- strm.avail_out = sizeof(outbuf);
- }
-
- ~BzipCompressionSink()
- {
- BZ2_bzCompressEnd(&strm);
- }
-
- void finish() override
- {
- flush();
- writeInternal({});
- }
-
- void writeInternal(std::string_view data) override
- {
- assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
- strm.next_in = (char *) data.data();
- strm.avail_in = data.size();
-
- while (!finished && (!data.data() || strm.avail_in)) {
- checkInterrupt();
-
- int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
- if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
- throw CompressionError("error %d while compressing bzip2 file", ret);
-
- finished = ret == BZ_STREAM_END;
-
- if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
- strm.next_out = (char *) outbuf;
- strm.avail_out = sizeof(outbuf);
- }
- }
- }
-};
-
struct BrotliCompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
- BrotliEncoderState *state;
+ BrotliEncoderState * state;
bool finished = false;
BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
@@ -471,12 +259,14 @@ struct BrotliCompressionSink : ChunkedCompressionSink
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
{
+ std::vector<std::string> la_supports = {
+ "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
+ };
+ if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
+ return make_ref<ArchiveCompressionSink>(nextSink, method, parallel);
+ }
if (method == "none")
return make_ref<NoneSink>(nextSink);
- else if (method == "xz")
- return make_ref<XzCompressionSink>(nextSink, parallel);
- else if (method == "bzip2")
- return make_ref<BzipCompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliCompressionSink>(nextSink);
else
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index dd666a4e1..338a0d9f2 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,14 +8,16 @@
namespace nix {
-struct CompressionSink : BufferedSink
+struct CompressionSink : BufferedSink, FinishSink
{
- virtual void finish() = 0;
+ using BufferedSink::operator ();
+ using BufferedSink::write;
+ using FinishSink::finish;
};
ref<std::string> decompress(const std::string & method, const std::string & in);
-ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
+std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index bda07cd55..c247c7dae 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -152,6 +152,16 @@ nlohmann::json Config::toJSON()
return res;
}
+std::string Config::toKeyValue()
+{
+ auto res = std::string();
+ for (auto & s : _settings)
+ if (!s.second.isAlias) {
+ res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
+ }
+ return res;
+}
+
void Config::convertToArgs(Args & args, const std::string & category)
{
for (auto & s : _settings)
@@ -167,11 +177,6 @@ AbstractSetting::AbstractSetting(
{
}
-void AbstractSetting::setDefault(const std::string & str)
-{
- if (!overridden) set(str);
-}
-
nlohmann::json AbstractSetting::toJSON()
{
return nlohmann::json(toJSONObject());
@@ -385,6 +390,16 @@ nlohmann::json GlobalConfig::toJSON()
return res;
}
+std::string GlobalConfig::toKeyValue()
+{
+ std::string res;
+ std::map<std::string, Config::SettingInfo> settings;
+ globalConfig.getSettings(settings);
+ for (auto & s : settings)
+ res += fmt("%s = %s\n", s.first, s.second.value);
+ return res;
+}
+
void GlobalConfig::convertToArgs(Args & args, const std::string & category)
{
for (auto & config : *configRegistrations)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index bf81b4892..736810bf3 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -100,6 +100,12 @@ public:
virtual nlohmann::json toJSON() = 0;
/**
+ * Outputs all settings in a key-value pair format suitable to be used as
+ * `nix.conf`
+ */
+ virtual std::string toKeyValue() = 0;
+
+ /**
* Converts settings to `Args` to be used on the command line interface
* - args: args to write to
* - category: category of the settings
@@ -169,6 +175,8 @@ public:
nlohmann::json toJSON() override;
+ std::string toKeyValue() override;
+
void convertToArgs(Args & args, const std::string & category) override;
};
@@ -186,8 +194,6 @@ public:
bool overridden = false;
- void setDefault(const std::string & str);
-
protected:
AbstractSetting(
@@ -245,6 +251,7 @@ public:
bool operator !=(const T & v2) const { return value != v2; }
void operator =(const T & v) { assign(v); }
virtual void assign(const T & v) { value = v; }
+ void setDefault(const T & v) { if (!overridden) value = v; }
void set(const std::string & str, bool append = false) override;
@@ -330,6 +337,8 @@ struct GlobalConfig : public AbstractConfig
nlohmann::json toJSON() override;
+ std::string toKeyValue() override;
+
void convertToArgs(Args & args, const std::string & category) override;
struct Register
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 0eea3455d..203d79087 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -185,15 +185,15 @@ void printAtPos(const ErrPos & pos, std::ostream & out)
if (pos) {
switch (pos.origin) {
case foFile: {
- out << fmt(ANSI_BLUE "at " ANSI_YELLOW "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
+ out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
break;
}
case foString: {
- out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
+ out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
break;
}
case foStdin: {
- out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
+ out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
break;
}
default:
@@ -232,7 +232,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
break;
}
case Verbosity::lvlWarn: {
- prefix = ANSI_YELLOW "warning";
+ prefix = ANSI_WARNING "warning";
break;
}
case Verbosity::lvlInfo: {
@@ -252,7 +252,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
break;
}
case Verbosity::lvlDebug: {
- prefix = ANSI_YELLOW "debug";
+ prefix = ANSI_WARNING "debug";
break;
}
default:
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index 85c0e9429..fd335b811 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -82,7 +82,7 @@ struct yellowtxt
template <class T>
std::ostream & operator<<(std::ostream & out, const yellowtxt<T> & y)
{
- return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+ return out << ANSI_WARNING << y.value << ANSI_NORMAL;
}
template <class T>
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 5341c58e6..f880c0fc5 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -6,7 +6,7 @@ libutil_DIR := $(d)
libutil_SOURCES := $(wildcard $(d)/*.cc)
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
ifeq ($(HAVE_LIBCPUID), 1)
libutil_LDFLAGS += -lcpuid
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index d2e801175..f8a121ed1 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -27,7 +27,7 @@ Logger * logger = makeSimpleLogger(true);
void Logger::warn(const std::string & msg)
{
- log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg);
+ log(lvlWarn, ANSI_WARNING "warning:" ANSI_NORMAL " " + msg);
}
void Logger::writeToStdout(std::string_view s)
@@ -46,7 +46,7 @@ public:
: printBuildLogs(printBuildLogs)
{
systemd = getEnv("IN_SYSTEMD") == "1";
- tty = isatty(STDERR_FILENO);
+ tty = shouldANSI();
}
bool isVerbose() override {
@@ -163,7 +163,7 @@ struct JSONLogger : Logger {
void write(const nlohmann::json & json)
{
- prevLogger.log(lvlError, "@nix " + json.dump());
+ prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace));
}
void log(Verbosity lvl, const FormatOrString & fs) override
diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh
index 0be2a7e74..d6bf53bb8 100644
--- a/src/libutil/ref.hh
+++ b/src/libutil/ref.hh
@@ -73,6 +73,16 @@ public:
return ref<T2>((std::shared_ptr<T2>) p);
}
+ bool operator == (const ref<T> & other) const
+ {
+ return p == other.p;
+ }
+
+ bool operator != (const ref<T> & other) const
+ {
+ return p != other.p;
+ }
+
private:
template<typename T2, typename... Args>
@@ -89,4 +99,47 @@ make_ref(Args&&... args)
return ref<T>(p);
}
+
+/* A non-nullable pointer.
+ This is similar to a C++ "& reference", but mutable.
+ This is similar to ref<T> but backed by a regular pointer instead of a smart pointer.
+ */
+template<typename T>
+class ptr {
+private:
+ T * p;
+
+public:
+ ptr<T>(const ptr<T> & r)
+ : p(r.p)
+ { }
+
+ explicit ptr<T>(T * p)
+ : p(p)
+ {
+ if (!p)
+ throw std::invalid_argument("null pointer cast to ptr");
+ }
+
+ T* operator ->() const
+ {
+ return &*p;
+ }
+
+ T& operator *() const
+ {
+ return *p;
+ }
+
+ bool operator == (const ptr<T> & other) const
+ {
+ return p == other.p;
+ }
+
+ bool operator != (const ptr<T> & other) const
+ {
+ return p != other.p;
+ }
+};
+
}
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index d1a16b6ba..16f3476c2 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -201,6 +201,62 @@ static DefaultStackAllocator defaultAllocatorSingleton;
StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
+{
+ struct SourceToSink : FinishSink
+ {
+ typedef boost::coroutines2::coroutine<bool> coro_t;
+
+ std::function<void(Source &)> fun;
+ std::optional<coro_t::push_type> coro;
+
+ SourceToSink(std::function<void(Source &)> fun) : fun(fun)
+ {
+ }
+
+ std::string_view cur;
+
+ void operator () (std::string_view in) override
+ {
+ if (in.empty()) return;
+ cur = in;
+
+ if (!coro)
+ coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
+ LambdaSource source([&](char *out, size_t out_len) {
+ if (cur.empty()) {
+ yield();
+ if (yield.get()) {
+ return (size_t)0;
+ }
+ }
+
+ size_t n = std::min(cur.size(), out_len);
+ memcpy(out, cur.data(), n);
+ cur.remove_prefix(n);
+ return n;
+ });
+ fun(source);
+ });
+
+ if (!*coro) { abort(); }
+
+ if (!cur.empty()) (*coro)(false);
+ }
+
+ void finish() override
+ {
+ if (!coro) return;
+ if (!*coro) abort();
+ (*coro)(true);
+ if (*coro) abort();
+ }
+ };
+
+ return std::make_unique<SourceToSink>(fun);
+}
+
+
std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun,
std::function<void()> eof)
@@ -212,7 +268,6 @@ std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun;
std::function<void()> eof;
std::optional<coro_t::pull_type> coro;
- bool started = false;
SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
: fun(fun), eof(eof)
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5bbbc7ce3..0fe6e8332 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -25,6 +25,13 @@ struct NullSink : Sink
{ }
};
+
+struct FinishSink : virtual Sink
+{
+ virtual void finish() = 0;
+};
+
+
/* A buffered abstract sink. Warning: a BufferedSink should not be
used from multiple threads concurrently. */
struct BufferedSink : virtual Sink
@@ -281,6 +288,7 @@ struct ChainSource : Source
size_t read(char * data, size_t len) override;
};
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun);
/* Convert a function that feeds data into a Sink into a Source. The
Source executes the function as a coroutine. */
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 2da169ba7..50e691a3d 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -2,83 +2,76 @@
#include <archive_entry.h>
#include "serialise.hh"
+#include "tarfile.hh"
namespace nix {
-struct TarArchive {
- struct archive * archive;
- Source * source;
- std::vector<unsigned char> buffer;
-
- void check(int err, const char * reason = "failed to extract archive: %s")
- {
- if (err == ARCHIVE_EOF)
- throw EndOfFile("reached end of archive");
- else if (err != ARCHIVE_OK)
- throw Error(reason, archive_error_string(this->archive));
- }
-
- TarArchive(Source & source) : buffer(4096)
- {
- this->archive = archive_read_new();
- this->source = &source;
-
- archive_read_support_filter_all(archive);
- archive_read_support_format_all(archive);
- check(archive_read_open(archive,
- (void *)this,
- TarArchive::callback_open,
- TarArchive::callback_read,
- TarArchive::callback_close),
- "failed to open archive: %s");
- }
-
- TarArchive(const Path & path)
- {
- this->archive = archive_read_new();
+static int callback_open(struct archive *, void * self)
+{
+ return ARCHIVE_OK;
+}
- archive_read_support_filter_all(archive);
- archive_read_support_format_all(archive);
- check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
+static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
+{
+ auto self = (TarArchive *) _self;
+ *buffer = self->buffer.data();
+
+ try {
+ return self->source->read((char *) self->buffer.data(), 4096);
+ } catch (EndOfFile &) {
+ return 0;
+ } catch (std::exception & err) {
+ archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
+ return -1;
}
+}
- TarArchive(const TarArchive &) = delete;
+static int callback_close(struct archive *, void * self)
+{
+ return ARCHIVE_OK;
+}
- void close()
- {
- check(archive_read_close(archive), "failed to close archive: %s");
- }
+void TarArchive::check(int err, const std::string & reason)
+{
+ if (err == ARCHIVE_EOF)
+ throw EndOfFile("reached end of archive");
+ else if (err != ARCHIVE_OK)
+ throw Error(reason, archive_error_string(this->archive));
+}
- ~TarArchive()
- {
- if (this->archive) archive_read_free(this->archive);
- }
+TarArchive::TarArchive(Source & source, bool raw)
+ : source(&source), buffer(4096)
+{
+ init();
+ if (!raw)
+ archive_read_support_format_all(archive);
+ else
+ archive_read_support_format_raw(archive);
+ check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
+}
-private:
+TarArchive::TarArchive(const Path & path)
+{
+ init();
+ archive_read_support_format_all(archive);
+ check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
+}
- static int callback_open(struct archive *, void * self) {
- return ARCHIVE_OK;
- }
+void TarArchive::init()
+{
+ archive = archive_read_new();
+ archive_read_support_filter_all(archive);
+}
- static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
- {
- auto self = (TarArchive *)_self;
- *buffer = self->buffer.data();
-
- try {
- return self->source->read((char *) self->buffer.data(), 4096);
- } catch (EndOfFile &) {
- return 0;
- } catch (std::exception & err) {
- archive_set_error(archive, EIO, "source threw exception: %s", err.what());
- return -1;
- }
- }
+void TarArchive::close()
+{
+ check(archive_read_close(this->archive), "Failed to close archive (%s)");
+}
- static int callback_close(struct archive *, void * self) {
- return ARCHIVE_OK;
- }
-};
+TarArchive::~TarArchive()
+{
+ if (this->archive) archive_read_free(this->archive);
+}
static void extract_archive(TarArchive & archive, const Path & destDir)
{
@@ -92,13 +85,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
struct archive_entry * entry;
int r = archive_read_next_header(archive.archive, &entry);
if (r == ARCHIVE_EOF) break;
- else if (r == ARCHIVE_WARN)
+ auto name = archive_entry_pathname(entry);
+ if (!name)
+ throw Error("cannot get archive member name: %s", archive_error_string(archive.archive));
+ if (r == ARCHIVE_WARN)
warn(archive_error_string(archive.archive));
else
archive.check(r);
archive_entry_set_pathname(entry,
- (destDir + "/" + archive_entry_pathname(entry)).c_str());
+ (destDir + "/" + name).c_str());
archive.check(archive_read_extract(archive.archive, entry, flags));
}
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
index 89a024f1d..f107a7e2e 100644
--- a/src/libutil/tarfile.hh
+++ b/src/libutil/tarfile.hh
@@ -1,7 +1,29 @@
#include "serialise.hh"
+#include <archive.h>
namespace nix {
+struct TarArchive {
+ struct archive * archive;
+ Source * source;
+ std::vector<unsigned char> buffer;
+
+ void check(int err, const std::string & reason = "failed to extract archive (%s)");
+
+ TarArchive(Source & source, bool raw = false);
+
+ TarArchive(const Path & path);
+
+ // disable copy constructor
+ TarArchive(const TarArchive &) = delete;
+
+ void init();
+
+ void close();
+
+ ~TarArchive();
+};
+
void unpackTarfile(Source & source, const Path & destDir);
void unpackTarfile(const Path & tarFile, const Path & destDir);
diff --git a/src/libutil/tests/closure.cc b/src/libutil/tests/closure.cc
new file mode 100644
index 000000000..7597e7807
--- /dev/null
+++ b/src/libutil/tests/closure.cc
@@ -0,0 +1,70 @@
+#include "closure.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+using namespace std;
+
+map<string, set<string>> testGraph = {
+ { "A", { "B", "C", "G" } },
+ { "B", { "A" } }, // Loops back to A
+ { "C", { "F" } }, // Indirect reference
+ { "D", { "A" } }, // Not reachable, but has backreferences
+ { "E", {} }, // Just not reachable
+ { "F", {} },
+ { "G", { "G" } }, // Self reference
+};
+
+TEST(closure, correctClosure) {
+ set<string> aClosure;
+ set<string> expectedClosure = {"A", "B", "C", "F", "G"};
+ computeClosure<string>(
+ {"A"},
+ aClosure,
+ [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
+ promise<set<string>> promisedNodes;
+ promisedNodes.set_value(testGraph[currentNode]);
+ processEdges(promisedNodes);
+ }
+ );
+
+ ASSERT_EQ(aClosure, expectedClosure);
+}
+
+TEST(closure, properlyHandlesDirectExceptions) {
+ struct TestExn {};
+ set<string> aClosure;
+ EXPECT_THROW(
+ computeClosure<string>(
+ {"A"},
+ aClosure,
+ [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
+ throw TestExn();
+ }
+ ),
+ TestExn
+ );
+}
+
+TEST(closure, properlyHandlesExceptionsInPromise) {
+ struct TestExn {};
+ set<string> aClosure;
+ EXPECT_THROW(
+ computeClosure<string>(
+ {"A"},
+ aClosure,
+ [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
+ promise<set<string>> promise;
+ try {
+ throw TestExn();
+ } catch (...) {
+ promise.set_exception(std::current_exception());
+ }
+ processEdges(promise);
+ }
+ ),
+ TestExn
+ );
+}
+
+}
diff --git a/src/libutil/tests/compression.cc b/src/libutil/tests/compression.cc
index 5b7a2c5b9..2efa3266b 100644
--- a/src/libutil/tests/compression.cc
+++ b/src/libutil/tests/compression.cc
@@ -17,6 +17,24 @@ namespace nix {
ASSERT_EQ(*o, "this-is-a-test");
}
+ TEST(decompress, decompressNoneCompressed) {
+ auto method = "none";
+ auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+ ref<std::string> o = decompress(method, str);
+
+ ASSERT_EQ(*o, str);
+ }
+
+ TEST(decompress, decompressEmptyCompressed) {
+ // Empty-method decompression used e.g. by S3 store
+ // (Content-Encoding == "").
+ auto method = "";
+ auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+ ref<std::string> o = decompress(method, str);
+
+ ASSERT_EQ(*o, str);
+ }
+
TEST(decompress, decompressXzCompressed) {
auto method = "xz";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index 6c649820b..2ffdc2e9b 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -336,7 +336,7 @@ namespace nix {
ASSERT_STREQ(
hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(),
- "only one arg " ANSI_YELLOW "fulfilled" ANSI_NORMAL " ");
+ "only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " ");
}
@@ -344,7 +344,7 @@ namespace nix {
ASSERT_STREQ(
hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(),
- "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
+ "what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
}
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index c1bab866c..f6232d255 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url)
auto isFile = scheme.find("file") != std::string::npos;
if (authority && *authority != "" && isFile)
- throw Error("file:// URL '%s' has unexpected authority '%s'",
+ throw BadURL("file:// URL '%s' has unexpected authority '%s'",
url, *authority);
if (isFile && path.empty())
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index dea9c74b7..bc841f425 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -4,16 +4,18 @@
#include "finally.hh"
#include "serialise.hh"
+#include <array>
#include <cctype>
#include <cerrno>
+#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <climits>
+#include <future>
#include <iostream>
+#include <mutex>
#include <sstream>
#include <thread>
-#include <future>
#include <fcntl.h>
#include <grp.h>
@@ -32,6 +34,7 @@
#ifdef __linux__
#include <sys/prctl.h>
+#include <sys/resource.h>
#endif
@@ -143,16 +146,21 @@ Path canonPath(const Path & path, bool resolveSymlinks)
s += '/';
while (i != end && *i != '/') s += *i++;
- /* If s points to a symlink, resolve it and restart (since
- the symlink target might contain new symlinks). */
+ /* If s points to a symlink, resolve it and continue from there */
if (resolveSymlinks && isLink(s)) {
if (++followCount >= maxFollow)
throw Error("infinite symlink recursion in path '%1%'", path);
- temp = absPath(readLink(s), dirOf(s))
- + string(i, end);
- i = temp.begin(); /* restart */
+ temp = readLink(s) + string(i, end);
+ i = temp.begin();
end = temp.end();
- s = "";
+ if (!temp.empty() && temp[0] == '/') {
+ s.clear(); /* restart for symlinks pointing to absolute path */
+ } else {
+ s = dirOf(s);
+ if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = /
+ s.clear();
+ }
+ }
}
}
}
@@ -407,7 +415,7 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
}
int fd = openat(parentfd, path.c_str(), O_RDONLY);
- if (!fd)
+ if (fd == -1)
throw SysError("opening directory '%1%'", path);
AutoCloseDir dir(fdopendir(fd));
if (!dir)
@@ -429,12 +437,9 @@ static void _deletePath(const Path & path, uint64_t & bytesFreed)
if (dir == "")
dir = "/";
- AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY));
+ AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
if (!dirfd) {
- // This really shouldn't fail silently, but it's left this way
- // for backwards compatibility.
if (errno == ENOENT) return;
-
throw SysError("opening directory '%1%'", path);
}
@@ -752,13 +757,13 @@ AutoCloseFD::AutoCloseFD() : fd{-1} {}
AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
-AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd}
+AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
{
that.fd = -1;
}
-AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that)
+AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
{
close();
fd = that.fd;
@@ -789,6 +794,7 @@ void AutoCloseFD::close()
if (::close(fd) == -1)
/* This should never happen. */
throw SysError("closing file descriptor %1%", fd);
+ fd = -1;
}
}
@@ -822,6 +828,12 @@ void Pipe::create()
}
+void Pipe::close()
+{
+ readSide.close();
+ writeSide.close();
+}
+
//////////////////////////////////////////////////////////////////////
@@ -1022,17 +1034,10 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
return res;
}
-// Output = "standard out" output stream
string runProgram(Path program, bool searchPath, const Strings & args,
const std::optional<std::string> & input)
{
- RunOptions opts(program, args);
- opts.searchPath = searchPath;
- // This allows you to refer to a program with a pathname relative to the
- // PATH variable.
- opts.input = input;
-
- auto res = runProgram(opts);
+ auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
if (!statusOk(res.first))
throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
@@ -1041,9 +1046,8 @@ string runProgram(Path program, bool searchPath, const Strings & args,
}
// Output = error code + "standard out" output stream
-std::pair<int, std::string> runProgram(const RunOptions & options_)
+std::pair<int, std::string> runProgram(RunOptions && options)
{
- RunOptions options(options_);
StringSink sink;
options.standardOut = &sink;
@@ -1109,7 +1113,7 @@ void runProgram2(const RunOptions & options)
Strings args_(options.args);
args_.push_front(options.program);
- restoreSignals();
+ restoreProcessContext();
if (options.searchPath)
execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
@@ -1121,7 +1125,7 @@ void runProgram2(const RunOptions & options)
throw SysError("executing '%1%'", options.program);
}, processOptions);
- out.writeSide = -1;
+ out.writeSide.close();
std::thread writerThread;
@@ -1134,7 +1138,7 @@ void runProgram2(const RunOptions & options)
if (source) {
- in.readSide = -1;
+ in.readSide.close();
writerThread = std::thread([&]() {
try {
std::vector<char> buf(8 * 1024);
@@ -1151,7 +1155,7 @@ void runProgram2(const RunOptions & options)
} catch (...) {
promise.set_exception(std::current_exception());
}
- in.writeSide = -1;
+ in.writeSide.close();
});
}
@@ -1359,6 +1363,12 @@ void ignoreException()
}
}
+bool shouldANSI()
+{
+ return isatty(STDERR_FILENO)
+ && getEnv("TERM").value_or("dumb") != "dumb"
+ && !getEnv("NO_COLOR").has_value();
+}
std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)
{
@@ -1431,7 +1441,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
+static std::array<char, 256> base64DecodeChars;
string base64Encode(std::string_view s)
{
@@ -1456,15 +1466,12 @@ string base64Encode(std::string_view s)
string base64Decode(std::string_view s)
{
- bool init = false;
- char decode[256];
- if (!init) {
- // FIXME: not thread-safe.
- memset(decode, -1, sizeof(decode));
+ static std::once_flag flag;
+ std::call_once(flag, [](){
+ base64DecodeChars = { (char)-1 };
for (int i = 0; i < 64; i++)
- decode[(int) base64Chars[i]] = i;
- init = true;
- }
+ base64DecodeChars[(int) base64Chars[i]] = i;
+ });
string res;
unsigned int d = 0, bits = 0;
@@ -1473,7 +1480,7 @@ string base64Decode(std::string_view s)
if (c == '=') break;
if (c == '\n') continue;
- char digit = decode[(unsigned char) c];
+ char digit = base64DecodeChars[(unsigned char) c];
if (digit == -1)
throw Error("invalid character in Base64 string: '%c'", c);
@@ -1605,12 +1612,45 @@ void startSignalHandlerThread()
std::thread(signalHandlerThread, set).detach();
}
-void restoreSignals()
+static void restoreSignals()
{
if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
throw SysError("restoring signals");
}
+#if __linux__
+rlim_t savedStackSize = 0;
+#endif
+
+void setStackSize(size_t stackSize)
+{
+ #if __linux__
+ struct rlimit limit;
+ if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+ savedStackSize = limit.rlim_cur;
+ limit.rlim_cur = stackSize;
+ setrlimit(RLIMIT_STACK, &limit);
+ }
+ #endif
+}
+
+void restoreProcessContext()
+{
+ restoreSignals();
+
+ restoreAffinity();
+
+ #if __linux__
+ if (savedStackSize) {
+ struct rlimit limit;
+ if (getrlimit(RLIMIT_STACK, &limit) == 0) {
+ limit.rlim_cur = savedStackSize;
+ setrlimit(RLIMIT_STACK, &limit);
+ }
+ }
+ #endif
+}
+
/* RAII helper to automatically deregister a callback. */
struct InterruptCallbackImpl : InterruptCallback
{
@@ -1673,10 +1713,13 @@ string showBytes(uint64_t bytes)
}
+// FIXME: move to libstore/build
void commonChildInit(Pipe & logPipe)
{
+ logger = makeSimpleLogger();
+
const static string pathNullDevice = "/dev/null";
- restoreSignals();
+ restoreProcessContext();
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index ad49c65b3..bee77b53f 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -188,7 +188,6 @@ public:
class AutoCloseFD
{
int fd;
- void close();
public:
AutoCloseFD();
AutoCloseFD(int fd);
@@ -200,6 +199,7 @@ public:
int get() const;
explicit operator bool() const;
int release();
+ void close();
};
@@ -216,6 +216,7 @@ class Pipe
public:
AutoCloseFD readSide, writeSide;
void create();
+ void close();
};
@@ -275,30 +276,33 @@ string runProgram(Path program, bool searchPath = false,
struct RunOptions
{
+ Path program;
+ bool searchPath = true;
+ Strings args;
std::optional<uid_t> uid;
std::optional<uid_t> gid;
std::optional<Path> chdir;
std::optional<std::map<std::string, std::string>> environment;
- Path program;
- bool searchPath = true;
- Strings args;
std::optional<std::string> input;
Source * standardIn = nullptr;
Sink * standardOut = nullptr;
bool mergeStderrToStdout = false;
- bool _killStderr = false;
-
- RunOptions(const Path & program, const Strings & args)
- : program(program), args(args) { };
-
- RunOptions & killStderr(bool v) { _killStderr = true; return *this; }
};
-std::pair<int, std::string> runProgram(const RunOptions & options);
+std::pair<int, std::string> runProgram(RunOptions && options);
void runProgram2(const RunOptions & options);
+/* Change the stack size. */
+void setStackSize(size_t stackSize);
+
+
+/* Restore the original inherited Unix process context (such as signal
+ masks, stack size, CPU affinity). */
+void restoreProcessContext();
+
+
class ExecError : public Error
{
public:
@@ -472,6 +476,9 @@ constexpr char treeLast[] = "└───";
constexpr char treeLine[] = "│ ";
constexpr char treeNull[] = " ";
+/* Determine whether ANSI escape sequences are appropriate for the
+ present output. */
+bool shouldANSI();
/* Truncate a string to 'width' printable characters. If 'filterAll'
is true, all ANSI escape sequences are filtered out. Otherwise,
@@ -512,9 +519,6 @@ class Callback;
on the current thread (and thus any threads created by it). */
void startSignalHandlerThread();
-/* Restore default signal handling. */
-void restoreSignals();
-
struct InterruptCallback
{
virtual ~InterruptCallback() { };
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 10:47:11 +0000