From 91539d305ff035d53a6de75f8af1ebbd7df4e622 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 22 Apr 2016 18:19:48 +0200
Subject: nix copy: Parallelise

---
 src/nix/copy.cc | 71 ++++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 58 insertions(+), 13 deletions(-)

(limited to 'src/nix/copy.cc')

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 87e6f604a..16b16910c 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -3,6 +3,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "sync.hh"
+#include "thread-pool.hh"
 
 #include <atomic>
 
@@ -57,29 +58,73 @@ struct CmdCopy : StorePathsCommand
 
         progressBar.updateStatus(showProgress());
 
-        storePaths.reverse(); // FIXME: assumes reverse topo sort
+        struct Graph
+        {
+            std::set<Path> left;
+            std::map<Path, std::set<Path>> refs, rrefs;
+        };
 
-        for (auto & storePath : storePaths) {
+        Sync<Graph> graph_;
+        {
+            auto graph(graph_.lock());
+            graph->left = PathSet(storePaths.begin(), storePaths.end());
+        }
+
+        ThreadPool pool;
+
+        std::function<void(const Path &)> doPath;
+
+        doPath = [&](const Path & storePath) {
             checkInterrupt();
 
-            if (dstStore->isValidPath(storePath)) {
-                total--;
-                progressBar.updateStatus(showProgress());
-                continue;
-            }
+            if (!dstStore->isValidPath(storePath)) {
+                auto activity(progressBar.startActivity(format("copying ‘%s’...") % storePath));
 
-            auto activity(progressBar.startActivity(format("copying ‘%s’...") % storePath));
+                StringSink sink;
+                srcStore->exportPaths({storePath}, false, sink);
 
-            StringSink sink;
-            srcStore->exportPaths({storePath}, false, sink);
+                StringSource source(*sink.s);
+                dstStore->importPaths(false, source, 0);
 
-            StringSource source(*sink.s);
-            dstStore->importPaths(false, source, 0);
+                done++;
+            } else
+                total--;
 
-            done++;
             progressBar.updateStatus(showProgress());
+
+            /* Enqueue all paths that were waiting for this one. */
+            {
+                auto graph(graph_.lock());
+                graph->left.erase(storePath);
+                for (auto & rref : graph->rrefs[storePath]) {
+                    auto & refs(graph->refs[rref]);
+                    auto i = refs.find(storePath);
+                    assert(i != refs.end());
+                    refs.erase(i);
+                    if (refs.empty())
+                        pool.enqueue(std::bind(doPath, rref));
+                }
+            }
+        };
+
+        /* Build the dependency graph; enqueue all paths with no
+           dependencies. */
+        for (auto & storePath : storePaths) {
+            auto info = srcStore->queryPathInfo(storePath);
+            {
+                auto graph(graph_.lock());
+                for (auto & ref : info->references)
+                    if (ref != storePath && graph->left.count(ref)) {
+                        graph->refs[storePath].insert(ref);
+                        graph->rrefs[ref].insert(storePath);
+                    }
+                if (graph->refs[storePath].empty())
+                    pool.enqueue(std::bind(doPath, storePath));
+            }
         }
 
+        pool.process();
+
         progressBar.done();
     }
 };
-- 
cgit v1.2.3