Merge remote-tracking branch 'upstream/master' into optional-derivation-output-storepath

10 files changed, 203 insertions, 35 deletions

diff --git a/.dir-locals.el b/.dir-locals.el
index 8b21d4e40..a2d1dc48d 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -1,6 +1,7 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
+  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7755466a0..47fa041e9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,3 +12,13 @@ jobs:
     - uses: actions/checkout@v2
     - uses: cachix/install-nix-action@v10
     - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
+  macos_perf_test:
+    runs-on: macos-latest
+    steps:
+    - name: Disable syspolicy assessments
+      run: |
+        spctl --status
+        sudo spctl --master-disable
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v10
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 3d753836f..6cb69d0a9 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
     return ca ? renderContentAddress(*ca) : "";
 }
 
+Hash getContentAddressHash(const ContentAddress & ca)
+{
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return th.hash;
+        },
+        [](FixedOutputHash fsh) {
+            return fsh.hash;
+        }
+    }, ca);
+}
+
 }
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index ba4797f5b..22a039242 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
 
+Hash getContentAddressHash(const ContentAddress & ca);
+
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 58cf08763..4d5eec7bf 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -365,6 +365,9 @@ public:
 
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
+
+    Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
+        "Maximum size of NARs before spilling them to disk."};
 };
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7ecaa8743..d1aca9790 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -969,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
         PathLocks outputLock;
 
-        Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
+        auto realPath = Store::toRealPath(info.path);
 
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
@@ -1040,8 +1040,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 
@@ -1091,16 +1090,125 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
 {
     Path srcPath(absPath(_srcPath));
 
-    /* Read the whole path into memory. This is not a very scalable
-       method for very large paths, but `copyPath' is mainly used for
-       small files. */
-    StringSink sink;
-    if (method == FileIngestionMethod::Recursive)
-        dumpPath(srcPath, sink, filter);
-    else
-        sink.s = make_ref<std::string>(readFile(srcPath));
+    /* For computing the NAR hash. */
+    auto sha256Sink = std::make_unique<HashSink>(htSHA256);
+
+    /* For computing the store path. In recursive SHA-256 mode, this
+       is the same as the NAR hash, so no need to do it again. */
+    std::unique_ptr<HashSink> hashSink =
+        method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
+        ? nullptr
+        : std::make_unique<HashSink>(hashAlgo);
+
+    /* Read the source path into memory, but only if it's up to
+       narBufferSize bytes. If it's larger, write it to a temporary
+       location in the Nix store. If the subsequently computed
+       destination store path is already valid, we just delete the
+       temporary path. Otherwise, we move it to the destination store
+       path. */
+    bool inMemory = true;
+    std::string nar;
+
+    auto source = sinkToSource([&](Sink & sink) {
+
+        LambdaSink sink2([&](const unsigned char * buf, size_t len) {
+            (*sha256Sink)(buf, len);
+            if (hashSink) (*hashSink)(buf, len);
+
+            if (inMemory) {
+                if (nar.size() + len > settings.narBufferSize) {
+                    inMemory = false;
+                    sink << 1;
+                    sink((const unsigned char *) nar.data(), nar.size());
+                    nar.clear();
+                } else {
+                    nar.append((const char *) buf, len);
+                }
+            }
+
+            if (!inMemory) sink(buf, len);
+        });
+
+        if (method == FileIngestionMethod::Recursive)
+            dumpPath(srcPath, sink2, filter);
+        else
+            readFile(srcPath, sink2);
+    });
+
+    std::unique_ptr<AutoDelete> delTempDir;
+    Path tempPath;
+
+    try {
+        /* Wait for the source coroutine to give us some dummy
+           data. This is so that we don't create the temporary
+           directory if the NAR fits in memory. */
+        readInt(*source);
+
+        auto tempDir = createTempDir(realStoreDir, "add");
+        delTempDir = std::make_unique<AutoDelete>(tempDir);
+        tempPath = tempDir + "/x";
+
+        if (method == FileIngestionMethod::Recursive)
+            restorePath(tempPath, *source);
+        else
+            writeFile(tempPath, *source);
+
+    } catch (EndOfFile &) {
+        if (!inMemory) throw;
+        /* The NAR fits in memory, so we didn't do restorePath(). */
+    }
+
+    auto sha256 = sha256Sink->finish();
+
+    Hash hash = hashSink ? hashSink->finish().first : sha256.first;
+
+    auto dstPath = makeFixedOutputPath(method, hash, name);
+
+    addTempRoot(dstPath);
+
+    if (repair || !isValidPath(dstPath)) {
+
+        /* The first check above is an optimisation to prevent
+           unnecessary lock acquisition. */
+
+        auto realPath = Store::toRealPath(dstPath);
 
-    return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
+        PathLocks outputLock({realPath});
+
+        if (repair || !isValidPath(dstPath)) {
+
+            deletePath(realPath);
+
+            autoGC();
+
+            if (inMemory) {
+                /* Restore from the NAR in memory. */
+                StringSource source(nar);
+                if (method == FileIngestionMethod::Recursive)
+                    restorePath(realPath, source);
+                else
+                    writeFile(realPath, source);
+            } else {
+                /* Move the temporary path we restored above. */
+                if (rename(tempPath.c_str(), realPath.c_str()))
+                    throw Error("renaming '%s' to '%s'", tempPath, realPath);
+            }
+
+            canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
+
+            optimisePath(realPath);
+
+            ValidPathInfo info(dstPath);
+            info.narHash = sha256.first;
+            info.narSize = sha256.second;
+            info.ca = FixedOutputHash { .method = method, .hash = hash };
+            registerValidPath(info);
+        }
+
+        outputLock.setDeletion(true);
+    }
+
+    return dstPath;
 }
 
 
@@ -1114,8 +1222,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c0a8bc9f6..8d46bb436 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -7,6 +7,7 @@
 #include "json.hh"
 #include "derivations.hh"
 #include "url.hh"
+#include "archive.hh"
 
 #include <future>
 
@@ -221,6 +222,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
 }
 
 
+ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
+    FileIngestionMethod method, HashType hashAlgo,
+    std::optional<Hash> expectedCAHash)
+{
+    /* FIXME: inefficient: we're reading/hashing 'tmpFile' three
+       times. */
+
+    auto [narHash, narSize] = hashPath(htSHA256, srcPath);
+
+    auto hash = method == FileIngestionMethod::Recursive
+        ? hashAlgo == htSHA256
+          ? narHash
+          : hashPath(hashAlgo, srcPath).first
+        : hashFile(hashAlgo, srcPath);
+
+    if (expectedCAHash && expectedCAHash != hash)
+        throw Error("hash mismatch for '%s'", srcPath);
+
+    ValidPathInfo info(makeFixedOutputPath(method, hash, name));
+    info.narHash = narHash;
+    info.narSize = narSize;
+    info.ca = FixedOutputHash { .method = method, .hash = hash };
+
+    if (!isValidPath(info.path)) {
+        auto source = sinkToSource([&](Sink & sink) {
+            dumpPath(srcPath, sink);
+        });
+        addToStore(info, *source);
+    }
+
+    return info;
+}
+
+
 Store::Store(const Params & params)
     : Config(params)
     , state({(size_t) pathInfoCacheSize})
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b122e05d6..b1dd1f478 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -450,6 +450,13 @@ public:
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
+    /* Copy the contents of a path to the store and register the
+       validity the resulting path, using a constant amount of
+       memory. */
+    ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+        std::optional<Hash> expectedCAHash = {});
+
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(const string & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 40b05a2f3..961e7fb6d 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -153,14 +153,15 @@ static int _main(int argc, char * * argv)
 
         /* If an expected hash is given, the file may already exist in
            the store. */
-        Hash hash, expectedHash(ht);
+        std::optional<Hash> expectedHash;
+        Hash hash;
         std::optional<StorePath> storePath;
         if (args.size() == 2) {
             expectedHash = Hash(args[1], ht);
             const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
+            storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
             if (store->isValidPath(*storePath))
-                hash = expectedHash;
+                hash = *expectedHash;
             else
                 storePath.reset();
         }
@@ -200,22 +201,12 @@ static int _main(int argc, char * * argv)
                     tmpFile = unpacked;
             }
 
-            /* FIXME: inefficient; addToStore() will also hash
-               this. */
-            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
+            const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
 
-            if (expectedHash != Hash(ht) && expectedHash != hash)
-                throw Error("hash mismatch for '%1%'", uri);
-
-            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-
-            /* Copy the file to the Nix store. FIXME: if RemoteStore
-               implemented addToStoreFromDump() and downloadFile()
-               supported a sink, we could stream the download directly
-               into the Nix store. */
-            storePath = store->addToStore(name, tmpFile, recursive, ht);
-
-            assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
+            auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
+            storePath = info.path;
+            assert(info.ca);
+            hash = getContentAddressHash(*info.ca);
         }
 
         stopProgressBar();
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 2a2f16ac1..39a47e19a 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
 {
-    auto recursive = FileIngestionMethod::Flat;
+    auto method = FileIngestionMethod::Flat;
 
     for (auto & i : opFlags)
-        if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
+        if (i == "--recursive") method = FileIngestionMethod::Recursive;
         else throw UsageError("unknown flag '%1%'", i);
 
     if (opArgs.empty())
@@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     opArgs.pop_front();
 
     for (auto & i : opArgs)
-        cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
+        std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
 }