* While waiting for a lock, print a sign of life every 5 minutes.

  This prevents remote builders from being killed by the
  `max-silent-time' inactivity monitor while they are waiting for a
  long garbage collection to finish.  This happens fairly often in the
  Hydra build farm.

2 files changed, 14 insertions, 3 deletions

diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index d8290815c..fe872ceed 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -37,7 +37,8 @@ void deleteLockFile(const Path & path, int fd)
 }
 
 
-bool lockFile(int fd, LockType lockType, bool wait)
+bool lockFile(int fd, LockType lockType, bool wait,
+    unsigned int progressInterval)
 {
     struct flock lock;
     if (lockType == ltRead) lock.l_type = F_RDLCK;
@@ -49,11 +50,20 @@ bool lockFile(int fd, LockType lockType, bool wait)
     lock.l_len = 0; /* entire file */
 
     if (wait) {
-        while (fcntl(fd, F_SETLKW, &lock) != 0) {
+        /* Wait until we acquire the lock.  If `progressInterval' is
+           non-zero, when print a message every `progressInterval'
+           seconds.  This is mostly to make sure that remote builders
+           aren't killed due to the `max-silent-time' inactivity
+           monitor while waiting for the garbage collector lock. */
+        while (1) {
+            if (progressInterval) alarm(progressInterval);
+            if (fcntl(fd, F_SETLKW, &lock) == 0) break;
             checkInterrupt();
             if (errno != EINTR)
                 throw SysError(format("acquiring/releasing lock"));
+            if (progressInterval) printMsg(lvlError, "still waiting for lock...");
         }
+        alarm(0);
     } else {
         while (fcntl(fd, F_SETLK, &lock) != 0) {
             checkInterrupt();
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 57ca1584a..8c6ac6a03 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -17,7 +17,8 @@ void deleteLockFile(const Path & path, int fd);
 
 enum LockType { ltRead, ltWrite, ltNone };
 
-bool lockFile(int fd, LockType lockType, bool wait);
+bool lockFile(int fd, LockType lockType, bool wait,
+    unsigned int progressInterval = 300);
 
 
 class PathLocks