aboutsummaryrefslogtreecommitdiff
path: root/src/libutil
diff options
context:
space:
mode:
Diffstat (limited to 'src/libutil')
-rw-r--r--src/libutil/archive.cc15
-rw-r--r--src/libutil/archive.hh5
-rw-r--r--src/libutil/compression.cc73
-rw-r--r--src/libutil/compression.hh2
-rw-r--r--src/libutil/lru-cache.hh84
-rw-r--r--src/libutil/pool.hh151
-rw-r--r--src/libutil/ref.hh67
-rw-r--r--src/libutil/serialise.cc22
-rw-r--r--src/libutil/serialise.hh23
-rw-r--r--src/libutil/sync.hh78
-rw-r--r--src/libutil/types.hh68
-rw-r--r--src/libutil/util.cc12
-rw-r--r--src/libutil/util.hh12
13 files changed, 515 insertions, 97 deletions
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 6ee798143..5363496c2 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -29,7 +29,7 @@ bool useCaseHack =
false;
#endif
-static string archiveVersion1 = "nix-archive-1";
+const std::string narVersionMagic1 = "nix-archive-1";
static string caseHackSuffix = "~nix~case~hack~";
@@ -113,11 +113,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
{
- sink << archiveVersion1;
+ sink << narVersionMagic1;
dump(path, sink, filter);
}
+void dumpString(const std::string & s, Sink & sink)
+{
+ sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
+}
+
+
static SerialisationError badArchive(string s)
{
return SerialisationError("bad archive: " + s);
@@ -214,7 +220,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
}
else if (s == "executable" && type == tpRegular) {
- readString(source);
+ auto s = readString(source);
+ if (s != "") throw badArchive("executable marker has non-empty value");
sink.isExecutable();
}
@@ -275,7 +282,7 @@ void parseDump(ParseSink & sink, Source & source)
/* This generally means the integer at the start couldn't be
decoded. Ignore and throw the exception below. */
}
- if (version != archiveVersion1)
+ if (version != narVersionMagic1)
throw badArchive("input doesn't look like a Nix archive");
parse(sink, source, "");
}
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index c216e9768..90117f5ff 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -55,6 +55,8 @@ extern PathFilter defaultPathFilter;
void dumpPath(const Path & path, Sink & sink,
PathFilter & filter = defaultPathFilter);
+void dumpString(const std::string & s, Sink & sink);
+
struct ParseSink
{
virtual void createDirectory(const Path & path) { };
@@ -76,4 +78,7 @@ void restorePath(const Path & path, Source & source);
extern bool useCaseHack;
+extern const std::string narVersionMagic1;
+
+
}
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index fb4160669..a3fa0dab7 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -6,34 +6,83 @@
namespace nix {
+/* RAII wrapper around lzma_stream. */
+struct LzmaStream
+{
+ lzma_stream strm;
+ LzmaStream() : strm(LZMA_STREAM_INIT) { };
+ ~LzmaStream() { lzma_end(&strm); };
+ lzma_stream & operator()() { return strm; }
+};
+
+std::string compressXZ(const std::string & in)
+{
+ LzmaStream strm;
+
+ // FIXME: apply the x86 BCJ filter?
+
+ lzma_ret ret = lzma_easy_encoder(
+ &strm(), 6, LZMA_CHECK_CRC64);
+ if (ret != LZMA_OK)
+ throw Error("unable to initialise lzma encoder");
+
+ lzma_action action = LZMA_RUN;
+ uint8_t outbuf[BUFSIZ];
+ string res;
+ strm().next_in = (uint8_t *) in.c_str();
+ strm().avail_in = in.size();
+ strm().next_out = outbuf;
+ strm().avail_out = sizeof(outbuf);
+
+ while (true) {
+
+ if (strm().avail_in == 0)
+ action = LZMA_FINISH;
+
+ lzma_ret ret = lzma_code(&strm(), action);
+
+ if (strm().avail_out == 0 || ret == LZMA_STREAM_END) {
+ res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out);
+ strm().next_out = outbuf;
+ strm().avail_out = sizeof(outbuf);
+ }
+
+ if (ret == LZMA_STREAM_END)
+ return res;
+
+ if (ret != LZMA_OK)
+ throw Error("error while decompressing xz file");
+ }
+}
+
std::string decompressXZ(const std::string & in)
{
- lzma_stream strm = LZMA_STREAM_INIT;
+ LzmaStream strm;
lzma_ret ret = lzma_stream_decoder(
- &strm, UINT64_MAX, LZMA_CONCATENATED);
+ &strm(), UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK)
throw Error("unable to initialise lzma decoder");
lzma_action action = LZMA_RUN;
uint8_t outbuf[BUFSIZ];
string res;
- strm.next_in = (uint8_t *) in.c_str();
- strm.avail_in = in.size();
- strm.next_out = outbuf;
- strm.avail_out = sizeof(outbuf);
+ strm().next_in = (uint8_t *) in.c_str();
+ strm().avail_in = in.size();
+ strm().next_out = outbuf;
+ strm().avail_out = sizeof(outbuf);
while (true) {
- if (strm.avail_in == 0)
+ if (strm().avail_in == 0)
action = LZMA_FINISH;
- lzma_ret ret = lzma_code(&strm, action);
+ lzma_ret ret = lzma_code(&strm(), action);
- if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
- res.append((char *) outbuf, sizeof(outbuf) - strm.avail_out);
- strm.next_out = outbuf;
- strm.avail_out = sizeof(outbuf);
+ if (strm().avail_out == 0 || ret == LZMA_STREAM_END) {
+ res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out);
+ strm().next_out = outbuf;
+ strm().avail_out = sizeof(outbuf);
}
if (ret == LZMA_STREAM_END)
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index 962ce5ac7..eb1697fc4 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -4,6 +4,8 @@
namespace nix {
+std::string compressXZ(const std::string & in);
+
std::string decompressXZ(const std::string & in);
}
diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
new file mode 100644
index 000000000..4344d6601
--- /dev/null
+++ b/src/libutil/lru-cache.hh
@@ -0,0 +1,84 @@
+#pragma once
+
+#include <map>
+#include <list>
+
+namespace nix {
+
+/* A simple least-recently used cache. Not thread-safe. */
+template<typename Key, typename Value>
+class LRUCache
+{
+private:
+
+ size_t maxSize;
+
+ // Stupid wrapper to get around circular dependency between Data
+ // and LRU.
+ struct LRUIterator;
+
+ using Data = std::map<Key, std::pair<LRUIterator, Value>>;
+ using LRU = std::list<typename Data::iterator>;
+
+ struct LRUIterator { typename LRU::iterator it; };
+
+ Data data;
+ LRU lru;
+
+public:
+
+ LRUCache(size_t maxSize) : maxSize(maxSize) { }
+
+ /* Insert or upsert an item in the cache. */
+ void upsert(const Key & key, const Value & value)
+ {
+ erase(key);
+
+ if (data.size() >= maxSize) {
+ /* Retire the oldest item. */
+ auto oldest = lru.begin();
+ data.erase(*oldest);
+ lru.erase(oldest);
+ }
+
+ auto res = data.emplace(key, std::make_pair(LRUIterator(), value));
+ assert(res.second);
+ auto & i(res.first);
+
+ auto j = lru.insert(lru.end(), i);
+
+ i->second.first.it = j;
+ }
+
+ bool erase(const Key & key)
+ {
+ auto i = data.find(key);
+ if (i == data.end()) return false;
+ lru.erase(i->second.first.it);
+ data.erase(i);
+ return true;
+ }
+
+ /* Look up an item in the cache. If it exists, it becomes the most
+ recently used item. */
+ // FIXME: use boost::optional?
+ Value * get(const Key & key)
+ {
+ auto i = data.find(key);
+ if (i == data.end()) return 0;
+
+ /* Move this item to the back of the LRU list. */
+ lru.erase(i->second.first.it);
+ auto j = lru.insert(lru.end(), i);
+ i->second.first.it = j;
+
+ return &i->second.second;
+ }
+
+ size_t size()
+ {
+ return data.size();
+ }
+};
+
+}
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
new file mode 100644
index 000000000..f291cd578
--- /dev/null
+++ b/src/libutil/pool.hh
@@ -0,0 +1,151 @@
+#pragma once
+
+#include <functional>
+#include <limits>
+#include <list>
+#include <memory>
+#include <cassert>
+
+#include "sync.hh"
+#include "ref.hh"
+
+namespace nix {
+
+/* This template class implements a simple pool manager of resources
+ of some type R, such as database connections. It is used as
+ follows:
+
+ class Connection { ... };
+
+ Pool<Connection> pool;
+
+ {
+ auto conn(pool.get());
+ conn->exec("select ...");
+ }
+
+ Here, the Connection object referenced by ‘conn’ is automatically
+ returned to the pool when ‘conn’ goes out of scope.
+*/
+
+template <class R>
+class Pool
+{
+public:
+
+ /* A function that produces new instances of R on demand. */
+ typedef std::function<ref<R>()> Factory;
+
+ /* A function that checks whether an instance of R is still
+ usable. Unusable instances are removed from the pool. */
+ typedef std::function<bool(const ref<R> &)> Validator;
+
+private:
+
+ Factory factory;
+ Validator validator;
+
+ struct State
+ {
+ size_t inUse = 0;
+ size_t max;
+ std::vector<ref<R>> idle;
+ };
+
+ Sync<State> state;
+
+ std::condition_variable wakeup;
+
+public:
+
+ Pool(size_t max = std::numeric_limits<size_t>::max(),
+ const Factory & factory = []() { return make_ref<R>(); },
+ const Validator & validator = [](ref<R> r) { return true; })
+ : factory(factory)
+ , validator(validator)
+ {
+ auto state_(state.lock());
+ state_->max = max;
+ }
+
+ ~Pool()
+ {
+ auto state_(state.lock());
+ assert(!state_->inUse);
+ state_->max = 0;
+ state_->idle.clear();
+ }
+
+ class Handle
+ {
+ private:
+ Pool & pool;
+ std::shared_ptr<R> r;
+
+ friend Pool;
+
+ Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { }
+
+ public:
+ Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); }
+
+ Handle(const Handle & l) = delete;
+
+ ~Handle()
+ {
+ if (!r) return;
+ {
+ auto state_(pool.state.lock());
+ state_->idle.push_back(ref<R>(r));
+ assert(state_->inUse);
+ state_->inUse--;
+ }
+ pool.wakeup.notify_one();
+ }
+
+ R * operator -> () { return &*r; }
+ R & operator * () { return *r; }
+ };
+
+ Handle get()
+ {
+ {
+ auto state_(state.lock());
+
+ /* If we're over the maximum number of instance, we need
+ to wait until a slot becomes available. */
+ while (state_->idle.empty() && state_->inUse >= state_->max)
+ state_.wait(wakeup);
+
+ while (!state_->idle.empty()) {
+ auto p = state_->idle.back();
+ state_->idle.pop_back();
+ if (validator(p)) {
+ state_->inUse++;
+ return Handle(*this, p);
+ }
+ }
+
+ state_->inUse++;
+ }
+
+ /* We need to create a new instance. Because that might take a
+ while, we don't hold the lock in the meantime. */
+ try {
+ Handle h(*this, factory());
+ return h;
+ } catch (...) {
+ auto state_(state.lock());
+ state_->inUse--;
+ throw;
+ }
+ }
+
+ unsigned int count()
+ {
+ auto state_(state.lock());
+ return state_->idle.size() + state_->inUse;
+ }
+};
+
+}
diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh
new file mode 100644
index 000000000..a6d338d79
--- /dev/null
+++ b/src/libutil/ref.hh
@@ -0,0 +1,67 @@
+#pragma once
+
+#include <memory>
+#include <exception>
+
+namespace nix {
+
+/* A simple non-nullable reference-counted pointer. Actually a wrapper
+ around std::shared_ptr that prevents non-null constructions. */
+template<typename T>
+class ref
+{
+private:
+
+ std::shared_ptr<T> p;
+
+public:
+
+ ref<T>(const ref<T> & r)
+ : p(r.p)
+ { }
+
+ explicit ref<T>(const std::shared_ptr<T> & p)
+ : p(p)
+ {
+ if (!p)
+ throw std::invalid_argument("null pointer cast to ref");
+ }
+
+ T* operator ->() const
+ {
+ return &*p;
+ }
+
+ T& operator *() const
+ {
+ return *p;
+ }
+
+ operator std::shared_ptr<T> ()
+ {
+ return p;
+ }
+
+ template<typename T2>
+ operator ref<T2> ()
+ {
+ return ref<T2>((std::shared_ptr<T2>) p);
+ }
+
+private:
+
+ template<typename T2, typename... Args>
+ friend ref<T2>
+ make_ref(Args&&... args);
+
+};
+
+template<typename T, typename... Args>
+inline ref<T>
+make_ref(Args&&... args)
+{
+ auto p = std::make_shared<T>(std::forward<Args>(args)...);
+ return ref<T>(p);
+}
+
+}
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index f136a1324..c9620e2bf 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -72,7 +72,17 @@ void FdSink::write(const unsigned char * data, size_t len)
warned = true;
}
}
- writeFull(fd, data, len);
+ try {
+ writeFull(fd, data, len);
+ } catch (SysError & e) {
+ _good = true;
+ }
+}
+
+
+bool FdSink::good()
+{
+ return _good;
}
@@ -119,12 +129,18 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
checkInterrupt();
n = ::read(fd, (char *) data, bufSize);
} while (n == -1 && errno == EINTR);
- if (n == -1) throw SysError("reading from file");
- if (n == 0) throw EndOfFile("unexpected end-of-file");
+ if (n == -1) { _good = false; throw SysError("reading from file"); }
+ if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
return n;
}
+bool FdSource::good()
+{
+ return _good;
+}
+
+
size_t StringSource::read(unsigned char * data, size_t len)
{
if (pos == s.size()) throw EndOfFile("end of string reached");
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 979ff849f..9e269f392 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -12,6 +12,7 @@ struct Sink
{
virtual ~Sink() { }
virtual void operator () (const unsigned char * data, size_t len) = 0;
+ virtual bool good() { return true; }
};
@@ -25,7 +26,7 @@ struct BufferedSink : Sink
: bufSize(bufSize), bufPos(0), buffer(0) { }
~BufferedSink();
- void operator () (const unsigned char * data, size_t len);
+ void operator () (const unsigned char * data, size_t len) override;
void flush();
@@ -47,6 +48,8 @@ struct Source
return the number of bytes stored. If blocks until at least
one byte is available. */
virtual size_t read(unsigned char * data, size_t len) = 0;
+
+ virtual bool good() { return true; }
};
@@ -60,7 +63,7 @@ struct BufferedSource : Source
: bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { }
~BufferedSource();
- size_t read(unsigned char * data, size_t len);
+ size_t read(unsigned char * data, size_t len) override;
/* Underlying read call, to be overridden. */
virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
@@ -80,7 +83,12 @@ struct FdSink : BufferedSink
FdSink(int fd) : fd(fd), warn(false), written(0) { }
~FdSink();
- void write(const unsigned char * data, size_t len);
+ void write(const unsigned char * data, size_t len) override;
+
+ bool good() override;
+
+private:
+ bool _good = true;
};
@@ -90,7 +98,10 @@ struct FdSource : BufferedSource
int fd;
FdSource() : fd(-1) { }
FdSource(int fd) : fd(fd) { }
- size_t readUnbuffered(unsigned char * data, size_t len);
+ size_t readUnbuffered(unsigned char * data, size_t len) override;
+ bool good() override;
+private:
+ bool _good = true;
};
@@ -98,7 +109,7 @@ struct FdSource : BufferedSource
struct StringSink : Sink
{
string s;
- void operator () (const unsigned char * data, size_t len);
+ void operator () (const unsigned char * data, size_t len) override;
};
@@ -108,7 +119,7 @@ struct StringSource : Source
const string & s;
size_t pos;
StringSource(const string & _s) : s(_s), pos(0) { }
- size_t read(unsigned char * data, size_t len);
+ size_t read(unsigned char * data, size_t len) override;
};
diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh
new file mode 100644
index 000000000..c99c098ac
--- /dev/null
+++ b/src/libutil/sync.hh
@@ -0,0 +1,78 @@
+#pragma once
+
+#include <mutex>
+#include <condition_variable>
+#include <cassert>
+
+namespace nix {
+
+/* This template class ensures synchronized access to a value of type
+ T. It is used as follows:
+
+ struct Data { int x; ... };
+
+ Sync<Data> data;
+
+ {
+ auto data_(data.lock());
+ data_->x = 123;
+ }
+
+ Here, "data" is automatically unlocked when "data_" goes out of
+ scope.
+*/
+
+template<class T>
+class Sync
+{
+private:
+ std::mutex mutex;
+ T data;
+
+public:
+
+ Sync() { }
+ Sync(const T & data) : data(data) { }
+
+ class Lock
+ {
+ private:
+ Sync * s;
+ std::unique_lock<std::mutex> lk;
+ friend Sync;
+ Lock(Sync * s) : s(s), lk(s->mutex) { }
+ public:
+ Lock(Lock && l) : s(l.s) { abort(); }
+ Lock(const Lock & l) = delete;
+ ~Lock() { }
+ T * operator -> () { return &s->data; }
+ T & operator * () { return s->data; }
+
+ void wait(std::condition_variable & cv)
+ {
+ assert(s);
+ cv.wait(lk);
+ }
+
+ template<class Rep, class Period, class Predicate>
+ bool wait_for(std::condition_variable & cv,
+ const std::chrono::duration<Rep, Period> & duration,
+ Predicate pred)
+ {
+ assert(s);
+ return cv.wait_for(lk, duration, pred);
+ }
+
+ template<class Clock, class Duration>
+ std::cv_status wait_until(std::condition_variable & cv,
+ const std::chrono::time_point<Clock, Duration> & duration)
+ {
+ assert(s);
+ return cv.wait_until(lk, duration);
+ }
+ };
+
+ Lock lock() { return Lock(this); }
+};
+
+}
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 23eb52512..33aaf5fc9 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -2,6 +2,8 @@
#include "config.h"
+#include "ref.hh"
+
#include <string>
#include <list>
#include <set>
@@ -97,70 +99,4 @@ typedef enum {
} Verbosity;
-/* A simple non-nullable reference-counted pointer. Actually a wrapper
- around std::shared_ptr that prevents non-null constructions. */
-template<typename T>
-class ref
-{
-private:
-
- std::shared_ptr<T> p;
-
-public:
-
- ref<T>(const ref<T> & r)
- : p(r.p)
- { }
-
- explicit ref<T>(const std::shared_ptr<T> & p)
- : p(p)
- {
- if (!p)
- throw std::invalid_argument("null pointer cast to ref");
- }
-
- T* operator ->() const
- {
- return &*p;
- }
-
- T& operator *() const
- {
- return *p;
- }
-
- operator std::shared_ptr<T> ()
- {
- return p;
- }
-
-private:
-
- template<typename T2, typename... Args>
- friend ref<T2>
- make_ref(Args&&... args);
-
- template<typename T2, typename T3, typename... Args>
- friend ref<T2>
- make_ref(Args&&... args);
-
-};
-
-template<typename T, typename... Args>
-inline ref<T>
-make_ref(Args&&... args)
-{
- auto p = std::make_shared<T>(std::forward<Args>(args)...);
- return ref<T>(p);
-}
-
-template<typename T, typename T2, typename... Args>
-inline ref<T>
-make_ref(Args&&... args)
-{
- auto p = std::make_shared<T2>(std::forward<Args>(args)...);
- return ref<T>(p);
-}
-
-
}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index def0525ab..3becbbabc 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -320,9 +320,11 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
{
checkInterrupt();
- printMsg(lvlVomit, format("%1%") % path);
-
- struct stat st = lstat(path);
+ struct stat st;
+ if (lstat(path.c_str(), &st) == -1) {
+ if (errno == ENOENT) return;
+ throw SysError(format("getting status of ‘%1%’") % path);
+ }
if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
bytesFreed += st.st_blocks * 512;
@@ -338,8 +340,10 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
_deletePath(path + "/" + i.name, bytesFreed);
}
- if (remove(path.c_str()) == -1)
+ if (remove(path.c_str()) == -1) {
+ if (errno == ENOENT) return;
throw SysError(format("cannot unlink ‘%1%’") % path);
+ }
}
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b714cdc64..3606f6ec9 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -92,8 +92,8 @@ string readLine(int fd);
void writeLine(int fd, string s);
/* Delete a path; i.e., in the case of a directory, it is deleted
- recursively. Don't use this at home, kids. The second variant
- returns the number of bytes and blocks freed. */
+ recursively. It's not an error if the path does not exist. The
+ second variant returns the number of bytes and blocks freed. */
void deletePath(const Path & path);
void deletePath(const Path & path, unsigned long long & bytesFreed);
@@ -366,6 +366,14 @@ template<class N> bool string2Int(const string & s, N & n)
return str && str.get() == EOF;
}
+/* Parse a string into a float. */
+template<class N> bool string2Float(const string & s, N & n)
+{
+ std::istringstream str(s);
+ str >> n;
+ return str && str.get() == EOF;
+}
+
/* Return true iff `s' ends in `suffix'. */
bool hasSuffix(const string & s, const string & suffix);
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 13:36:14 +0000