Fixed up mutex protection around winbindd logon code. Sync with APP-HEAD.

Jeremy.
(This used to be commit 3d04872499332ef2d8e7479b924afc8fc1ac29d7)

4 files changed, 77 insertions, 38 deletions

diff --git a/source3/include/local.h b/source3/include/local.h
index 20fa5ff52a..bf828b5894 100644
--- a/source3/include/local.h
+++ b/source3/include/local.h
@@ -216,4 +216,12 @@
 /* Max number of open RPC pipes. */
 #define MAX_OPEN_PIPES 2048
 
+/* Tuning for server auth mutex. */
+#define CLI_AUTH_TIMEOUT 5000 /* In milli-seconds. */
+#define NUM_CLI_AUTH_CONNECT_RETRIES 3
+/* Number in seconds to wait for the mutex. This must be less than 30 seconds. */
+#define SERVER_MUTEX_WAIT_TIME ( ((NUM_CLI_AUTH_CONNECT_RETRIES) * ((CLI_AUTH_TIMEOUT)/1000)) + 5)
+/* Number in seconds for winbindd to wait for the mutex. Make this 2 * smbd wait time. */
+#define WINBIND_SERVER_MUTEX_WAIT_TIME (( ((NUM_CLI_AUTH_CONNECT_RETRIES) * ((CLI_AUTH_TIMEOUT)/1000)) + 5)*2)
+
 #endif
diff --git a/source3/lib/server_mutex.c b/source3/lib/server_mutex.c
index 3e5512c734..878e5497d8 100644
--- a/source3/lib/server_mutex.c
+++ b/source3/lib/server_mutex.c
@@ -30,6 +30,8 @@
    like the single-connection that NT makes. */
 
 static char *mutex_server_name;
+/* FIXME. ref_count should be allocated per name... JRA. */
+size_t ref_count;
 
 BOOL grab_server_mutex(const char *name)
 {
@@ -38,7 +40,7 @@ BOOL grab_server_mutex(const char *name)
 		DEBUG(0,("grab_server_mutex: malloc failed for %s\n", name));
 		return False;
 	}
-	if (!secrets_named_mutex(mutex_server_name, 10)) {
+	if (!secrets_named_mutex(mutex_server_name, 10, &ref_count)) {
 		DEBUG(10,("grab_server_mutex: failed for %s\n", name));
 		SAFE_FREE(mutex_server_name);
 		return False;
@@ -50,7 +52,7 @@ BOOL grab_server_mutex(const char *name)
 void release_server_mutex(void)
 {
 	if (mutex_server_name) {
-		secrets_named_mutex_release(mutex_server_name);
+		secrets_named_mutex_release(mutex_server_name, &ref_count);
 		SAFE_FREE(mutex_server_name);
 	}
 }
diff --git a/source3/nsswitch/winbindd_cm.c b/source3/nsswitch/winbindd_cm.c
index 1105c77572..54096c0c1d 100644
--- a/source3/nsswitch/winbindd_cm.c
+++ b/source3/nsswitch/winbindd_cm.c
@@ -72,6 +72,7 @@ struct winbindd_cm_conn {
 	fstring domain;
 	fstring controller;
 	fstring pipe_name;
+	size_t mutex_ref_count;
 	struct cli_state *cli;
 	POLICY_HND pol;
 };
@@ -301,7 +302,7 @@ static void add_failed_connection_entry(struct winbindd_cm_conn *new_conn,
 /* Open a connction to the remote server, cache failures for 30 seconds */
 
 static NTSTATUS cm_open_connection(const char *domain, const int pipe_index,
-			       struct winbindd_cm_conn *new_conn)
+			       struct winbindd_cm_conn *new_conn, BOOL keep_mutex)
 {
 	struct failed_connection_cache *fcc;
 	NTSTATUS result;
@@ -309,6 +310,7 @@ static NTSTATUS cm_open_connection(const char *domain, const int pipe_index,
 	struct in_addr dc_ip;
 	int i;
 	BOOL retry = True;
+	BOOL got_mutex = False;
 
 	ZERO_STRUCT(dc_ip);
 
@@ -365,12 +367,23 @@ static NTSTATUS cm_open_connection(const char *domain, const int pipe_index,
 
 	for (i = 0; retry && (i < 3); i++) {
 		
+		if (!secrets_named_mutex(new_conn->controller, WINBIND_SERVER_MUTEX_WAIT_TIME, &new_conn->mutex_ref_count)) {
+			DEBUG(0,("cm_open_connection: mutex grab failed for %s\n", new_conn->controller));
+			result = NT_STATUS_POSSIBLE_DEADLOCK;
+			continue;
+		}
+
+		got_mutex = True;
+
 		result = cli_full_connection(&new_conn->cli, global_myname(), new_conn->controller, 
 			&dc_ip, 0, "IPC$", "IPC", ipc_username, ipc_domain, 
 			ipc_password, 0, &retry);
 
 		if (NT_STATUS_IS_OK(result))
 			break;
+
+		secrets_named_mutex_release(new_conn->controller, &new_conn->mutex_ref_count);
+		got_mutex = False;
 	}
 
 	SAFE_FREE(ipc_username);
@@ -378,6 +391,8 @@ static NTSTATUS cm_open_connection(const char *domain, const int pipe_index,
 	SAFE_FREE(ipc_password);
 
 	if (!NT_STATUS_IS_OK(result)) {
+		if (got_mutex)
+			secrets_named_mutex_release(new_conn->controller, &new_conn->mutex_ref_count);
 		add_failed_connection_entry(new_conn, result);
 		return result;
 	}
@@ -392,12 +407,16 @@ static NTSTATUS cm_open_connection(const char *domain, const int pipe_index,
 		 * if the PDC is an NT4 box.   but since there is only one 2k 
 		 * specific UUID right now, i'm not going to bother.  --jerry
 		 */
+		if (got_mutex)
+			secrets_named_mutex_release(new_conn->controller, &new_conn->mutex_ref_count);
 		if ( !is_win2k_pipe(pipe_index) )
 			add_failed_connection_entry(new_conn, result);
 		cli_shutdown(new_conn->cli);
 		return result;
 	}
 
+	if ((got_mutex) && !keep_mutex)
+		secrets_named_mutex_release(new_conn->controller, &new_conn->mutex_ref_count);
 	return NT_STATUS_OK;
 }
 
@@ -435,7 +454,8 @@ static BOOL connection_ok(struct winbindd_cm_conn *conn)
 
 /* Get a connection to the remote DC and open the pipe.  If there is already a connection, use that */
 
-static NTSTATUS get_connection_from_cache(const char *domain, const char *pipe_name, struct winbindd_cm_conn **conn_out) 
+static NTSTATUS get_connection_from_cache(const char *domain, const char *pipe_name,
+		struct winbindd_cm_conn **conn_out, BOOL keep_mutex) 
 {
 	struct winbindd_cm_conn *conn, conn_temp;
 	NTSTATUS result;
@@ -452,6 +472,12 @@ static NTSTATUS get_connection_from_cache(const char *domain, const char *pipe_n
 				SAFE_FREE(conn);
 				conn = &conn_temp;  /* Just to keep the loop moving */
 			} else {
+				if (keep_mutex) {
+					if (!secrets_named_mutex(conn->controller,
+							WINBIND_SERVER_MUTEX_WAIT_TIME, &conn->mutex_ref_count))
+						DEBUG(0,("get_connection_from_cache: mutex grab failed for %s\n",
+							conn->controller));
+				}
 				break;
 			}
 		}
@@ -463,7 +489,7 @@ static NTSTATUS get_connection_from_cache(const char *domain, const char *pipe_n
 		
 		ZERO_STRUCTP(conn);
 		
-		if (!NT_STATUS_IS_OK(result = cm_open_connection(domain, get_pipe_index(pipe_name), conn))) {
+		if (!NT_STATUS_IS_OK(result = cm_open_connection(domain, get_pipe_index(pipe_name), conn, keep_mutex))) {
 			DEBUG(3, ("Could not open a connection to %s for %s (%s)\n", 
 				  domain, pipe_name, nt_errstr(result)));
 		        SAFE_FREE(conn);
@@ -491,7 +517,7 @@ BOOL cm_check_for_native_mode_win2k( const char *domain )
 	ZERO_STRUCT( ctr );
 	
 	
-	if ( !NT_STATUS_IS_OK(result = cm_open_connection(domain, PI_LSARPC_DS, &conn)) ) {
+	if ( !NT_STATUS_IS_OK(result = cm_open_connection(domain, PI_LSARPC_DS, &conn, False)) ) {
 		DEBUG(5, ("cm_check_for_native_mode_win2k: Could not open a connection to %s for PIPE_LSARPC (%s)\n", 
 			  domain, nt_errstr(result)));
 		return False;
@@ -529,7 +555,7 @@ CLI_POLICY_HND *cm_get_lsa_handle(const char *domain)
 
 	/* Look for existing connections */
 
-	if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_LSARPC, &conn)))
+	if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_LSARPC, &conn, False)))
 		return NULL;
 
 	/* This *shitty* code needs scrapping ! JRA */
@@ -545,7 +571,7 @@ CLI_POLICY_HND *cm_get_lsa_handle(const char *domain)
 	if (!NT_STATUS_IS_OK(result)) {
 		/* Hit the cache code again.  This cleans out the old connection and gets a new one */
 		if (conn->cli->fd == -1) { /* Try again, if the remote host disapeared */
-			if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_LSARPC, &conn)))
+			if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_LSARPC, &conn, False)))
 				return NULL;
 
 			result = cli_lsa_open_policy(conn->cli, conn->cli->mem_ctx, False, 
@@ -577,7 +603,7 @@ CLI_POLICY_HND *cm_get_sam_handle(char *domain)
 
 	/* Look for existing connections */
 
-	if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_SAMR, &conn)))
+	if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_SAMR, &conn, False)))
 		return NULL;
 	
 	/* This *shitty* code needs scrapping ! JRA */
@@ -592,7 +618,7 @@ CLI_POLICY_HND *cm_get_sam_handle(char *domain)
 	if (!NT_STATUS_IS_OK(result)) {
 		/* Hit the cache code again.  This cleans out the old connection and gets a new one */
 		if (conn->cli->fd == -1) { /* Try again, if the remote host disapeared */
-			if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_SAMR, &conn)))
+			if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_SAMR, &conn, False)))
 				return NULL;
 
 			result = cli_samr_connect(conn->cli, conn->cli->mem_ctx,
@@ -849,28 +875,19 @@ NTSTATUS cm_get_netlogon_cli(const char *domain, const unsigned char *trust_pass
 	NTSTATUS result = NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND;
 	struct winbindd_cm_conn *conn;
 	uint32 neg_flags = 0x000001ff;
-	fstring srv_name;
-	struct in_addr dc_ip;
 
 	if (!cli)
 		return NT_STATUS_INVALID_PARAMETER;
 
-	if (!cm_get_dc_name(domain, srv_name, &dc_ip))
-		return NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND;
+	/* Open an initial conection - keep the mutex. */
 
-	if (!secrets_named_mutex(srv_name, 10)) {
-		DEBUG(0,("cm_get_netlogon_cli: mutex grab failed for %s\n", srv_name));
-		return NT_STATUS_POSSIBLE_DEADLOCK;
-	}
-
-	/* Open an initial conection */
-
-	if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_NETLOGON, &conn)))
+	if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_NETLOGON, &conn, True)))
 		return result;
 	
 	result = cli_nt_setup_creds(conn->cli, get_sec_chan(), trust_passwd, &neg_flags, 2);
 
-	secrets_named_mutex_release(srv_name);
+	if (conn->mutex_ref_count)
+		secrets_named_mutex_release(conn->controller, &conn->mutex_ref_count);
 
 	if (!NT_STATUS_IS_OK(result)) {
 		DEBUG(0, ("error connecting to domain password server: %s\n",
@@ -879,18 +896,14 @@ NTSTATUS cm_get_netlogon_cli(const char *domain, const unsigned char *trust_pass
 		/* Hit the cache code again.  This cleans out the old connection and gets a new one */
 		if (conn->cli->fd == -1) {
 
-			if (!secrets_named_mutex(srv_name, 10)) {
-				DEBUG(0,("cm_get_netlogon_cli: mutex grab failed for %s\n", srv_name));
-				return NT_STATUS_POSSIBLE_DEADLOCK;
-			}
-
-			if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_NETLOGON, &conn)))
+			if (!NT_STATUS_IS_OK(result = get_connection_from_cache(domain, PIPE_NETLOGON, &conn, True)))
 				return result;
 			
 			/* Try again */
 			result = cli_nt_setup_creds( conn->cli, get_sec_chan(),trust_passwd, &neg_flags, 2);
 
-			secrets_named_mutex_release(srv_name);
+			if (conn->mutex_ref_count)
+				secrets_named_mutex_release(conn->controller, &conn->mutex_ref_count);
 		}
 		
 		if (!NT_STATUS_IS_OK(result)) {
diff --git a/source3/passdb/secrets.c b/source3/passdb/secrets.c
index 5ba9f1dc1e..a52e264270 100644
--- a/source3/passdb/secrets.c
+++ b/source3/passdb/secrets.c
@@ -588,17 +588,24 @@ NTSTATUS secrets_get_trusted_domains(TALLOC_CTX* ctx, int* enum_ctx, int max_num
  between smbd instances.
 *******************************************************************************/
 
-BOOL secrets_named_mutex(const char *name, unsigned int timeout)
+BOOL secrets_named_mutex(const char *name, unsigned int timeout, size_t *p_ref_count)
 {
-	int ret;
+	size_t ref_count = *p_ref_count;
+	int ret = 0;
 
 	if (!message_init())
 		return False;
 
-	ret = tdb_lock_bystring(tdb, name, timeout);
-	if (ret == 0)
-		DEBUG(10,("secrets_named_mutex: got mutex for %s\n", name ));
+	if (ref_count == 0) {
+		ret = tdb_lock_bystring(tdb, name, timeout);
+		if (ret == 0)
+			DEBUG(10,("secrets_named_mutex: got mutex for %s\n", name ));
+	}
 
+	if (ret == 0) {
+		*p_ref_count = ++ref_count;
+		DEBUG(10,("secrets_named_mutex: ref_count for mutex %s = %u\n", name, (unsigned int)ref_count ));
+	}
 	return (ret == 0);
 }
 
@@ -606,10 +613,19 @@ BOOL secrets_named_mutex(const char *name, unsigned int timeout)
  Unlock a named mutex.
 *******************************************************************************/
 
-void secrets_named_mutex_release(const char *name)
+void secrets_named_mutex_release(const char *name, size_t *p_ref_count)
 {
-	tdb_unlock_bystring(tdb, name);
-	DEBUG(10,("secrets_named_mutex: released mutex for %s\n", name ));
+	size_t ref_count = *p_ref_count;
+
+	SMB_ASSERT(ref_count != 0);
+
+	if (ref_count == 1) {
+		tdb_unlock_bystring(tdb, name);
+		DEBUG(10,("secrets_named_mutex: released mutex for %s\n", name ));
+	}
+
+	*p_ref_count = --ref_count;
+	DEBUG(10,("secrets_named_mutex_release: ref_count for mutex %s = %u\n", name, (unsigned int)ref_count ));
 }
 
 /*********************************************************