aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libstore/content-address.cc12
-rw-r--r--src/libstore/content-address.hh2
-rw-r--r--src/libstore/globals.hh3
-rw-r--r--src/libstore/local-store.cc135
-rw-r--r--src/libstore/store-api.cc35
-rw-r--r--src/libstore/store-api.hh7
-rw-r--r--src/nix-prefetch-url/nix-prefetch-url.cc27
-rw-r--r--src/nix-store/nix-store.cc6
8 files changed, 192 insertions, 35 deletions
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 3d753836f..6cb69d0a9 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : "";
}
+Hash getContentAddressHash(const ContentAddress & ca)
+{
+ return std::visit(overloaded {
+ [](TextHash th) {
+ return th.hash;
+ },
+ [](FixedOutputHash fsh) {
+ return fsh.hash;
+ }
+ }, ca);
+}
+
}
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index ba4797f5b..22a039242 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
+Hash getContentAddressHash(const ContentAddress & ca);
+
}
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 58cf08763..4d5eec7bf 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -365,6 +365,9 @@ public:
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
+
+ Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
+ "Maximum size of NARs before spilling them to disk."};
};
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7ecaa8743..d1aca9790 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -969,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
PathLocks outputLock;
- Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
+ auto realPath = Store::toRealPath(info.path);
/* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a
@@ -1040,8 +1040,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
- Path realPath = realStoreDir + "/";
- realPath += dstPath.to_string();
+ auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
@@ -1091,16 +1090,125 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
{
Path srcPath(absPath(_srcPath));
- /* Read the whole path into memory. This is not a very scalable
- method for very large paths, but `copyPath' is mainly used for
- small files. */
- StringSink sink;
- if (method == FileIngestionMethod::Recursive)
- dumpPath(srcPath, sink, filter);
- else
- sink.s = make_ref<std::string>(readFile(srcPath));
+ /* For computing the NAR hash. */
+ auto sha256Sink = std::make_unique<HashSink>(htSHA256);
+
+ /* For computing the store path. In recursive SHA-256 mode, this
+ is the same as the NAR hash, so no need to do it again. */
+ std::unique_ptr<HashSink> hashSink =
+ method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
+ ? nullptr
+ : std::make_unique<HashSink>(hashAlgo);
+
+ /* Read the source path into memory, but only if it's up to
+ narBufferSize bytes. If it's larger, write it to a temporary
+ location in the Nix store. If the subsequently computed
+ destination store path is already valid, we just delete the
+ temporary path. Otherwise, we move it to the destination store
+ path. */
+ bool inMemory = true;
+ std::string nar;
+
+ auto source = sinkToSource([&](Sink & sink) {
+
+ LambdaSink sink2([&](const unsigned char * buf, size_t len) {
+ (*sha256Sink)(buf, len);
+ if (hashSink) (*hashSink)(buf, len);
+
+ if (inMemory) {
+ if (nar.size() + len > settings.narBufferSize) {
+ inMemory = false;
+ sink << 1;
+ sink((const unsigned char *) nar.data(), nar.size());
+ nar.clear();
+ } else {
+ nar.append((const char *) buf, len);
+ }
+ }
+
+ if (!inMemory) sink(buf, len);
+ });
+
+ if (method == FileIngestionMethod::Recursive)
+ dumpPath(srcPath, sink2, filter);
+ else
+ readFile(srcPath, sink2);
+ });
+
+ std::unique_ptr<AutoDelete> delTempDir;
+ Path tempPath;
+
+ try {
+ /* Wait for the source coroutine to give us some dummy
+ data. This is so that we don't create the temporary
+ directory if the NAR fits in memory. */
+ readInt(*source);
+
+ auto tempDir = createTempDir(realStoreDir, "add");
+ delTempDir = std::make_unique<AutoDelete>(tempDir);
+ tempPath = tempDir + "/x";
+
+ if (method == FileIngestionMethod::Recursive)
+ restorePath(tempPath, *source);
+ else
+ writeFile(tempPath, *source);
+
+ } catch (EndOfFile &) {
+ if (!inMemory) throw;
+ /* The NAR fits in memory, so we didn't do restorePath(). */
+ }
+
+ auto sha256 = sha256Sink->finish();
+
+ Hash hash = hashSink ? hashSink->finish().first : sha256.first;
+
+ auto dstPath = makeFixedOutputPath(method, hash, name);
+
+ addTempRoot(dstPath);
+
+ if (repair || !isValidPath(dstPath)) {
+
+ /* The first check above is an optimisation to prevent
+ unnecessary lock acquisition. */
+
+ auto realPath = Store::toRealPath(dstPath);
- return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
+ PathLocks outputLock({realPath});
+
+ if (repair || !isValidPath(dstPath)) {
+
+ deletePath(realPath);
+
+ autoGC();
+
+ if (inMemory) {
+ /* Restore from the NAR in memory. */
+ StringSource source(nar);
+ if (method == FileIngestionMethod::Recursive)
+ restorePath(realPath, source);
+ else
+ writeFile(realPath, source);
+ } else {
+ /* Move the temporary path we restored above. */
+ if (rename(tempPath.c_str(), realPath.c_str()))
+ throw Error("renaming '%s' to '%s'", tempPath, realPath);
+ }
+
+ canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
+
+ optimisePath(realPath);
+
+ ValidPathInfo info(dstPath);
+ info.narHash = sha256.first;
+ info.narSize = sha256.second;
+ info.ca = FixedOutputHash { .method = method, .hash = hash };
+ registerValidPath(info);
+ }
+
+ outputLock.setDeletion(true);
+ }
+
+ return dstPath;
}
@@ -1114,8 +1222,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(dstPath)) {
- Path realPath = realStoreDir + "/";
- realPath += dstPath.to_string();
+ auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c0a8bc9f6..8d46bb436 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -7,6 +7,7 @@
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
+#include "archive.hh"
#include <future>
@@ -221,6 +222,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
}
+ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
+ FileIngestionMethod method, HashType hashAlgo,
+ std::optional<Hash> expectedCAHash)
+{
+ /* FIXME: inefficient: we're reading/hashing 'tmpFile' three
+ times. */
+
+ auto [narHash, narSize] = hashPath(htSHA256, srcPath);
+
+ auto hash = method == FileIngestionMethod::Recursive
+ ? hashAlgo == htSHA256
+ ? narHash
+ : hashPath(hashAlgo, srcPath).first
+ : hashFile(hashAlgo, srcPath);
+
+ if (expectedCAHash && expectedCAHash != hash)
+ throw Error("hash mismatch for '%s'", srcPath);
+
+ ValidPathInfo info(makeFixedOutputPath(method, hash, name));
+ info.narHash = narHash;
+ info.narSize = narSize;
+ info.ca = FixedOutputHash { .method = method, .hash = hash };
+
+ if (!isValidPath(info.path)) {
+ auto source = sinkToSource([&](Sink & sink) {
+ dumpPath(srcPath, sink);
+ });
+ addToStore(info, *source);
+ }
+
+ return info;
+}
+
+
Store::Store(const Params & params)
: Config(params)
, state({(size_t) pathInfoCacheSize})
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b122e05d6..b1dd1f478 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -450,6 +450,13 @@ public:
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
+ /* Copy the contents of a path to the store and register the
+ validity the resulting path, using a constant amount of
+ memory. */
+ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
+ FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+ std::optional<Hash> expectedCAHash = {});
+
// FIXME: remove?
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 40b05a2f3..961e7fb6d 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -153,14 +153,15 @@ static int _main(int argc, char * * argv)
/* If an expected hash is given, the file may already exist in
the store. */
- Hash hash, expectedHash(ht);
+ std::optional<Hash> expectedHash;
+ Hash hash;
std::optional<StorePath> storePath;
if (args.size() == 2) {
expectedHash = Hash(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
- storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
+ storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
if (store->isValidPath(*storePath))
- hash = expectedHash;
+ hash = *expectedHash;
else
storePath.reset();
}
@@ -200,22 +201,12 @@ static int _main(int argc, char * * argv)
tmpFile = unpacked;
}
- /* FIXME: inefficient; addToStore() will also hash
- this. */
- hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
+ const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
- if (expectedHash != Hash(ht) && expectedHash != hash)
- throw Error("hash mismatch for '%1%'", uri);
-
- const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-
- /* Copy the file to the Nix store. FIXME: if RemoteStore
- implemented addToStoreFromDump() and downloadFile()
- supported a sink, we could stream the download directly
- into the Nix store. */
- storePath = store->addToStore(name, tmpFile, recursive, ht);
-
- assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
+ auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
+ storePath = info.path;
+ assert(info.ca);
+ hash = getContentAddressHash(*info.ca);
}
stopProgressBar();
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 2a2f16ac1..39a47e19a 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
store. */
static void opAddFixed(Strings opFlags, Strings opArgs)
{
- auto recursive = FileIngestionMethod::Flat;
+ auto method = FileIngestionMethod::Flat;
for (auto & i : opFlags)
- if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
+ if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i);
if (opArgs.empty())
@@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
opArgs.pop_front();
for (auto & i : opArgs)
- cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
+ std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 04:33:11 +0000