build-remote.pl: don't wait forever for the upload lock

In the build hook, don't wait forever to get the upload lock.  This
ensures progress if another process gets stuck while holding the
upload lock.

1 files changed, 17 insertions, 3 deletions

diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in
index 8bf77df04..fca00241b 100755
--- a/scripts/build-remote.pl.in
+++ b/scripts/build-remote.pl.in
@@ -232,10 +232,24 @@ sub removeRoots {
 # the same missing path simultaneously, causing the effective network
 # bandwidth and target disk speed to be divided by N.
 my $uploadLock = "$currentLoad/$hostName.upload-lock";
-sysopen MAINLOCK, "$uploadLock", O_RDWR|O_CREAT, 0600 or die;
-flock(MAINLOCK, LOCK_EX) or die;
+sysopen UPLOADLOCK, "$uploadLock", O_RDWR|O_CREAT, 0600 or die;
+eval {
+    local $SIG{ALRM} = sub { die "alarm\n" };
+    # Don't wait forever, so that a process that gets stuck while
+    # holding the lock doesn't block everybody else indefinitely.
+    # It's safe to continue after a timeout, just (potentially)
+    # inefficient.
+    alarm 15 * 60;
+    flock(UPLOADLOCK, LOCK_EX);
+    alarm 0;
+};
+if ($@) {
+    die unless $@ eq "alarm\n";
+    print STDERR "somebody is hogging $uploadLock, continuing...\n";
+    unlink $uploadLock;
+}
 Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "");
-close MAINLOCK;
+close UPLOADLOCK;
 
 
 # Perform the build.