s3: Avoid a thundering herd in g_lock_unlock

Only notify the first 5 pending lock waiters. This avoids a thundering herd
problem that is really nasty in a cluster. It also makes acquiring a lock a bit
more FIFO, lock waiters are added to the end of the array.

1 files changed, 16 insertions, 1 deletions

diff --git a/source3/lib/g_lock.c b/source3/lib/g_lock.c
index e2620255d9..512c0680d9 100644
--- a/source3/lib/g_lock.c
+++ b/source3/lib/g_lock.c
@@ -530,13 +530,23 @@ static NTSTATUS g_lock_force_unlock(struct g_lock_ctx *ctx, const char *name,
 	}
 
 	if ((lock_type & G_LOCK_PENDING) == 0) {
+		int num_wakeups = 0;
+
 		/*
-		 * We've been the lock holder. Tell all others to retry.
+		 * We've been the lock holder. Others to retry. Don't
+		 * tell all others to avoid a thundering herd. In case
+		 * this leads to a complete stall because we miss some
+		 * processes, the loop in g_lock_lock tries at least
+		 * once a minute.
 		 */
+
 		for (i=0; i<num_locks; i++) {
 			if ((locks[i].lock_type & G_LOCK_PENDING) == 0) {
 				continue;
 			}
+			if (!process_exists(locks[i].pid)) {
+				continue;
+			}
 
 			/*
 			 * Ping all waiters to retry
@@ -549,6 +559,11 @@ static NTSTATUS g_lock_force_unlock(struct g_lock_ctx *ctx, const char *name,
 					  procid_str(talloc_tos(),
 						     &locks[i].pid),
 					  nt_errstr(status)));
+			} else {
+				num_wakeups += 1;
+			}
+			if (num_wakeups > 5) {
+				break;
 			}
 		}
 	}