aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libstore/build.cc2
-rw-r--r--src/libstore/daemon.cc18
-rw-r--r--src/libstore/local-store.cc125
-rw-r--r--src/libstore/local-store.hh6
-rw-r--r--src/libstore/store-api.hh2
-rw-r--r--src/libutil/serialise.cc33
-rw-r--r--src/libutil/serialise.hh11
7 files changed, 93 insertions, 104 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ac2e67574..62294a08c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2774,7 +2774,7 @@ struct RestrictedStore : public LocalFSStore
goal.addDependency(info.path);
}
- StorePath addToStoreFromDump(const string & dump, const string & name,
+ StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index db7139374..69d7ef511 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -375,21 +375,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopAddToStore: {
- std::string s, baseName;
+ HashType hashAlgo;
+ std::string baseName;
FileIngestionMethod method;
{
- bool fixed; uint8_t recursive;
- from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+ bool fixed;
+ uint8_t recursive;
+ std::string hashAlgoRaw;
+ from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
- s = "sha256";
+ hashAlgoRaw = "sha256";
method = FileIngestionMethod::Recursive;
}
+ hashAlgo = parseHashType(hashAlgoRaw);
}
- HashType hashAlgo = parseHashType(s);
StringSink savedNAR;
TeeSource savedNARSource(from, savedNAR);
@@ -407,8 +410,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
+ StringSource dumpSource {
+ method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s
+ };
auto path = store->addToStoreFromDump(
- method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
+ dumpSource,
baseName,
method,
hashAlgo);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 26b226fe8..b9fae6089 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1033,62 +1033,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
}
-StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
+StorePath LocalStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{
- Hash h = hashString(hashAlgo, dump);
-
- auto dstPath = makeFixedOutputPath(method, h, name);
-
- addTempRoot(dstPath);
-
- if (repair || !isValidPath(dstPath)) {
-
- /* The first check above is an optimisation to prevent
- unnecessary lock acquisition. */
-
- auto realPath = Store::toRealPath(dstPath);
-
- PathLocks outputLock({realPath});
-
- if (repair || !isValidPath(dstPath)) {
-
- deletePath(realPath);
-
- autoGC();
-
- if (method == FileIngestionMethod::Recursive) {
- StringSource source(dump);
- restorePath(realPath, source);
- } else
- writeFile(realPath, dump);
-
- canonicalisePathMetaData(realPath, -1);
-
- /* Register the SHA-256 hash of the NAR serialisation of
- the path in the database. We may just have computed it
- above (if called with recursive == true and hashAlgo ==
- sha256); otherwise, compute it here. */
- HashResult hash;
- if (method == FileIngestionMethod::Recursive) {
- hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
- hash.second = dump.size();
- } else
- hash = hashPath(htSHA256, realPath);
-
- optimisePath(realPath); // FIXME: combine with hashPath()
-
- ValidPathInfo info(dstPath);
- info.narHash = hash.first;
- info.narSize = hash.second;
- info.ca = FixedOutputHash { .method = method, .hash = h };
- registerValidPath(info);
+ return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink, size_t & wanted) {
+ while (1) {
+ constexpr size_t bufSize = 1024;
+ uint8_t buf[bufSize];
+ auto n = dump.read(buf, std::min(wanted, bufSize));
+ sink(buf, n);
+ // when control is yielded back to us wanted will be updated.
}
-
- outputLock.setDeletion(true);
- }
-
- return dstPath;
+ });
}
@@ -1097,15 +1053,21 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
{
Path srcPath(absPath(_srcPath));
- /* For computing the NAR hash. */
- auto sha256Sink = std::make_unique<HashSink>(htSHA256);
+ return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink, size_t & _) {
+ if (method == FileIngestionMethod::Recursive)
+ dumpPath(srcPath, sink, filter);
+ else
+ readFile(srcPath, sink);
+ });
+}
+
- /* For computing the store path. In recursive SHA-256 mode, this
- is the same as the NAR hash, so no need to do it again. */
- std::unique_ptr<HashSink> hashSink =
- method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
- ? nullptr
- : std::make_unique<HashSink>(hashAlgo);
+StorePath LocalStore::addToStoreCommon(
+ const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair,
+ std::function<void(Sink &, size_t &)> demux)
+{
+ /* For computing the store path. */
+ auto hashSink = std::make_unique<HashSink>(hashAlgo);
/* Read the source path into memory, but only if it's up to
narBufferSize bytes. If it's larger, write it to a temporary
@@ -1114,32 +1076,26 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
temporary path. Otherwise, we move it to the destination store
path. */
bool inMemory = true;
- std::string nar;
-
- auto source = sinkToSource([&](Sink & sink) {
+ std::string dump;
+ auto source = sinkToSource([&](Sink & sink, size_t & wanted) {
LambdaSink sink2([&](const unsigned char * buf, size_t len) {
- (*sha256Sink)(buf, len);
- if (hashSink) (*hashSink)(buf, len);
+ (*hashSink)(buf, len);
if (inMemory) {
- if (nar.size() + len > settings.narBufferSize) {
+ if (dump.size() + len > settings.narBufferSize) {
inMemory = false;
sink << 1;
- sink((const unsigned char *) nar.data(), nar.size());
- nar.clear();
+ sink((const unsigned char *) dump.data(), dump.size());
+ dump.clear();
} else {
- nar.append((const char *) buf, len);
+ dump.append((const char *) buf, len);
}
}
if (!inMemory) sink(buf, len);
});
-
- if (method == FileIngestionMethod::Recursive)
- dumpPath(srcPath, sink2, filter);
- else
- readFile(srcPath, sink2);
+ demux(sink2, wanted);
});
std::unique_ptr<AutoDelete> delTempDir;
@@ -1165,9 +1121,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
/* The NAR fits in memory, so we didn't do restorePath(). */
}
- auto sha256 = sha256Sink->finish();
-
- Hash hash = hashSink ? hashSink->finish().first : sha256.first;
+ auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name);
@@ -1190,7 +1144,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
if (inMemory) {
/* Restore from the NAR in memory. */
- StringSource source(nar);
+ StringSource source(dump);
if (method == FileIngestionMethod::Recursive)
restorePath(realPath, source);
else
@@ -1201,13 +1155,22 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
throw Error("renaming '%s' to '%s'", tempPath, realPath);
}
+ /* For computing the nar hash. In recursive SHA-256 mode, this
+ is the same as the store hash, so no need to do it again. */
+ auto narHash = std::pair { hash, size };
+ if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
+ HashSink narSink { htSHA256 };
+ dumpPath(realPath, narSink);
+ narHash = narSink.finish();
+ }
+
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
optimisePath(realPath);
ValidPathInfo info(dstPath);
- info.narHash = sha256.first;
- info.narSize = sha256.second;
+ info.narHash = narHash.first;
+ info.narSize = narHash.second;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info);
}
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index c0e5d0286..ae23004c4 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -153,7 +153,7 @@ public:
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
- StorePath addToStoreFromDump(const string & dump, const string & name,
+ StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
StorePath addTextToStore(const string & name, const string & s,
@@ -290,6 +290,10 @@ private:
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info);
+ StorePath addToStoreCommon(
+ const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair,
+ std::function<void(Sink &, size_t &)> demux);
+
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index a4be0411e..d1cb2035f 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -460,7 +460,7 @@ public:
std::optional<Hash> expectedCAHash = {});
// FIXME: remove?
- virtual StorePath addToStoreFromDump(const string & dump, const string & name,
+ virtual StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{
throw Error("addToStoreFromDump() is not supported by this store");
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index c8b71188f..141e9e976 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -165,35 +165,43 @@ size_t StringSource::read(unsigned char * data, size_t len)
#endif
std::unique_ptr<Source> sinkToSource(
- std::function<void(Sink &)> fun,
+ std::function<void(Sink &, size_t &)> fun,
std::function<void()> eof)
{
struct SinkToSource : Source
{
- typedef boost::coroutines2::coroutine<std::string> coro_t;
+ typedef boost::coroutines2::coroutine<std::basic_string<uint8_t>> coro_t;
- std::function<void(Sink &)> fun;
+ std::function<void(Sink &, size_t &)> fun;
std::function<void()> eof;
std::optional<coro_t::pull_type> coro;
bool started = false;
- SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
+ /* It would be nicer to have the co-routines have both args and a
+ return value, but unfortunately that was removed from Boost's
+ implementation for some reason, so we use some extra state instead.
+ */
+ size_t wanted = 0;
+
+ SinkToSource(std::function<void(Sink &, size_t &)> fun, std::function<void()> eof)
: fun(fun), eof(eof)
{
}
- std::string cur;
+ std::basic_string<uint8_t> cur;
size_t pos = 0;
size_t read(unsigned char * data, size_t len) override
{
- if (!coro)
+ wanted = len < cur.size() ? 0 : len - cur.size();
+ if (!coro) {
coro = coro_t::pull_type([&](coro_t::push_type & yield) {
- LambdaSink sink([&](const unsigned char * data, size_t len) {
- if (len) yield(std::string((const char *) data, len));
+ LambdaSink sink([&](const uint8_t * data, size_t len) {
+ if (len) yield(std::basic_string<uint8_t> { data, len });
});
- fun(sink);
+ fun(sink, wanted);
});
+ }
if (!*coro) { eof(); abort(); }
@@ -203,11 +211,10 @@ std::unique_ptr<Source> sinkToSource(
pos = 0;
}
- auto n = std::min(cur.size() - pos, len);
- memcpy(data, (unsigned char *) cur.data() + pos, n);
- pos += n;
+ auto numCopied = cur.copy(data, len, pos);
+ pos += numCopied;
- return n;
+ return numCopied;
}
};
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 8386a4991..6cb9d1bf5 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -260,11 +260,20 @@ struct LambdaSource : Source
/* Convert a function that feeds data into a Sink into a Source. The
Source executes the function as a coroutine. */
std::unique_ptr<Source> sinkToSource(
- std::function<void(Sink &)> fun,
+ std::function<void(Sink &, size_t &)> fun,
std::function<void()> eof = []() {
throw EndOfFile("coroutine has finished");
});
+static inline std::unique_ptr<Source> sinkToSource(
+ std::function<void(Sink &)> fun,
+ std::function<void()> eof = []() {
+ throw EndOfFile("coroutine has finished");
+ })
+{
+ return sinkToSource([fun](Sink & s, size_t & _) { fun(s); }, eof);
+}
+
void writePadding(size_t len, Sink & sink);
void writeString(const unsigned char * buf, size_t len, Sink & sink);
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 13:00:25 +0000