* Add an API function exportPaths() that provides the functionality of

  ‘nix-store --export’.
* Add a Perl module that provides the functionality of
  ‘nix-copy-closure --to’.  This is used by build-remote.pl so it no
  longer needs to start a separate nix-copy-closure process.  Also, it
  uses the Perl API to do the export, so it doesn't need to start a
  separate nix-store process either.  As a result, nix-copy-closure
  and build-remote.pl should no longer fail on very large closures due
  to an "Argument list too long" error.  (Note that having very many
  dependencies in a single derivation can still fail because the
  environment can become too large.  Can't be helped though.)

10 files changed, 90 insertions, 38 deletions

diff --git a/perl/Makefile.am b/perl/Makefile.am
index eded469f9..93f5415c8 100644
--- a/perl/Makefile.am
+++ b/perl/Makefile.am
@@ -2,7 +2,7 @@ perlversion := $(shell perl -e 'use Config; print $$Config{version};')
 perlarchname := $(shell perl -e 'use Config; print $$Config{archname};')
 perllibdir = $(libdir)/perl5/site_perl/$(perlversion)/$(perlarchname)
 
-PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/Config.pm.in
+PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in
 
 all: $(PERL_MODULES:.in=)
 	ln -sfn $(abs_builddir)/.libs/libNixStore.so lib/Store.so
diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm
new file mode 100644
index 000000000..045f6bfaf
--- /dev/null
+++ b/perl/lib/Nix/CopyClosure.pm
@@ -0,0 +1,46 @@
+package Nix::CopyClosure;
+
+use strict;
+use Nix::Config;
+use Nix::Store;
+
+
+sub copyTo {
+    my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign) = @_;
+
+    $compressor = "$compressor |" if $compressor ne "";
+    $decompressor = "$decompressor |" if $decompressor ne "";
+    
+    # Get the closure of this path.
+    my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
+        map { followLinksToStorePath $_ } @{$storePaths})));
+
+    # Ask the remote host which paths are invalid.  Because of limits
+    # to the command line length, do this in chunks.  Eventually,
+    # we'll want to use ‘--from-stdin’, but we can't rely on the
+    # target having this option yet.
+    my @missing = ();
+    while (scalar(@closure) > 0) {
+        my @ps = splice(@closure, 0, 1500);
+        open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|");
+        while (<READ>) {
+            chomp;
+            push @missing, $_;
+        }
+        close READ or die;
+    }
+
+    # Export the store paths and import them on the remote machine.
+    if (scalar @missing > 0) {
+        print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n";
+        #print STDERR "  $_\n" foreach @missing;
+        unless ($dryRun) {
+            open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import'" or die;
+            exportPaths(fileno(SSH), $sign, @missing);
+            close SSH or die "copying store paths to remote machine `$sshHost' failed: $?";
+        }
+    }
+}
+
+
+1;
diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm
index bef6e7460..d96f8e9ab 100644
--- a/perl/lib/Nix/Store.pm
+++ b/perl/lib/Nix/Store.pm
@@ -12,7 +12,7 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] );
 
 our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
 
-our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath);
+our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath exportPaths);
 
 our $VERSION = '0.15';
 
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 9e51ea337..b50451f45 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -146,3 +146,16 @@ SV * followLinksToStorePath(char * path)
         }
     OUTPUT:
         RETVAL
+
+
+void exportPaths(int fd, int sign, ...)
+    PPCODE:
+        try {
+            doInit();
+            Paths paths;
+            for (int n = 2; n < items; ++n) paths.push_back(SvPV_nolen(ST(n)));
+            FdSink sink(fd);
+            exportPaths(*store, paths, sign, sink);
+        } catch (Error & e) {
+            croak(e.what());
+        }
diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in
index 110c95f22..72d09565b 100755
--- a/scripts/build-remote.pl.in
+++ b/scripts/build-remote.pl.in
@@ -5,6 +5,7 @@ use English '-no_match_vars';
 use IO::Handle;
 use Nix::Config;
 use Nix::SSH qw/sshOpts openSSHConnection/;
+use Nix::CopyClosure;
 no warnings('once');
 
 
@@ -225,8 +226,7 @@ sub removeRoots {
 
 
 # Copy the derivation and its dependencies to the build machine.
-system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0
-    or die "cannot copy inputs to $hostName: $?";
+Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "");
 
 
 # Perform the build.
diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in
index 172acd9e7..2eac56e3f 100755
--- a/scripts/nix-copy-closure.in
+++ b/scripts/nix-copy-closure.in
@@ -3,6 +3,7 @@
 use Nix::SSH;
 use Nix::Config;
 use Nix::Store;
+use Nix::CopyClosure;
 
 
 if (scalar @ARGV < 1) {
@@ -39,8 +40,8 @@ while (@ARGV) {
         $sign = 1;
     }
     elsif ($arg eq "--gzip") {
-        $compressor = "| gzip";
-        $decompressor = "gunzip |";
+        $compressor = "gzip";
+        $decompressor = "gunzip";
     }
     elsif ($arg eq "--from") {
         $toMode = 0;
@@ -67,30 +68,7 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n";
 
 
 if ($toMode) { # Copy TO the remote machine.
-
-    # Get the closure of this path.
-    my @allStorePaths = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, map { followLinksToStorePath $_ } @storePaths)));
-
-    # Ask the remote host which paths are invalid.
-    open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
-    my @missing = ();
-    while (<READ>) {
-        chomp;
-        push @missing, $_;
-    }
-    close READ or die;
-
-    # Export the store paths and import them on the remote machine.
-    if (scalar @missing > 0) {
-        print STDERR "copying these missing paths:\n";
-        print STDERR "  $_\n" foreach @missing;
-        unless ($dryRun) {
-            my $extraOpts = $sign ? "--sign" : "";
-            system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
-                or die "copying store paths to remote machine `$sshHost' failed: $?";
-        }
-    }
-
+    Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign);
 }
 
 else { # Copy FROM the remote machine.
@@ -112,8 +90,10 @@ else { # Copy FROM the remote machine.
 
     # Export the store paths on the remote machine and import them on locally.
     if (scalar @missing > 0) {
-        print STDERR "copying these missing paths:\n";
-        print STDERR "  $_\n" foreach @missing;
+        print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
+        #print STDERR "  $_\n" foreach @missing;
+        $compressor = "| $compressor" if $compressor ne "";
+        $decompressor = "$decompressor |" if $decompressor ne "";
         unless ($dryRun) {
             my $extraOpts = $sign ? "--sign" : "";
             system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $Nix::Config::binDir/nix-store --import") == 0
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 06cadcb0f..3c1f2ecac 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1255,7 +1255,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
     bool haveSignature = readInt(hashAndReadSource) == 1;
 
     if (requireSignature && !haveSignature)
-        throw Error("imported archive lacks a signature");
+        throw Error(format("imported archive of `%1%' lacks a signature") % dstPath);
     
     if (haveSignature) {
         string signature = readString(hashAndReadSource);
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index d67ff2c77..36ade2170 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -298,6 +298,17 @@ string showPaths(const PathSet & paths)
 }
 
 
+void exportPaths(StoreAPI & store, const Paths & paths,
+    bool sign, Sink & sink)
+{
+    foreach (Paths::const_iterator, i, paths) {
+        writeInt(1, sink);
+        store.exportPath(*i, sign, sink);
+    }
+    writeInt(0, sink);
+}
+
+
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b3e67436c..8bfb09880 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -342,6 +342,12 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
     bool hashGiven = false);
 
 
+/* Export multiple paths in the format expected by ‘nix-store
+   --import’. */
+void exportPaths(StoreAPI & store, const Paths & paths,
+    bool sign, Sink & sink);
+
+
 }
 
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 371ca54af..84d3da032 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -594,11 +594,7 @@ static void opExport(Strings opFlags, Strings opArgs)
         else throw UsageError(format("unknown flag `%1%'") % *i);
 
     FdSink sink(STDOUT_FILENO);
-    for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) {
-        writeInt(1, sink);
-        store->exportPath(*i, sign, sink);
-    }
-    writeInt(0, sink);
+    exportPaths(*store, opArgs, sign, sink);
 }