r18191: Fix the online/offline state handling of winbindd.

Instead of trying to do this in the winbindd_cache
entries, add a timed even handler to probe every
5 mins when disconnected.
Fix events to run all pending events, rather than
only one.
Jeremy.
(This used to be commit 7bfbe1b4fb9a91c6678035f220bbf0b4f5afdcac)

6 files changed, 159 insertions, 41 deletions

diff --git a/source3/lib/events.c b/source3/lib/events.c
index 6db9930c30..7da44c3466 100644
--- a/source3/lib/events.c
+++ b/source3/lib/events.c
@@ -79,27 +79,24 @@ struct timed_event *add_timed_event(TALLOC_CTX *mem_ctx,
 
 void run_events(void)
 {
-	struct timeval now;
-
-	if (timed_events == NULL) {
-		/* No syscall if there are no events */
-		DEBUG(11, ("run_events: No events\n"));
-		return;
-	}
+	/* Run all events that are pending, not just one (as we
+	   did previously. */
 
-	GetTimeOfDay(&now);
+	while (timed_events) {
+		struct timeval now;
+		GetTimeOfDay(&now);
 
-	if (timeval_compare(&now, &timed_events->when) < 0) {
-		/* Nothing to do yet */
-		DEBUG(11, ("run_events: Nothing to do\n"));
-		return;
-	}
+		if (timeval_compare(&now, &timed_events->when) < 0) {
+			/* Nothing to do yet */
+			DEBUG(11, ("run_events: Nothing to do\n"));
+			return;
+		}
 
-	DEBUG(10, ("Running event \"%s\" %lx\n", timed_events->event_name,
-		(unsigned long)timed_events));
+		DEBUG(10, ("Running event \"%s\" %lx\n", timed_events->event_name,
+			(unsigned long)timed_events));
 
-	timed_events->handler(timed_events, &now, timed_events->private_data);
-	return;
+		timed_events->handler(timed_events, &now, timed_events->private_data);
+	}
 }
 
 struct timeval *get_timed_events_timeout(struct timeval *to_ret)
diff --git a/source3/nsswitch/winbindd.h b/source3/nsswitch/winbindd.h
index a16613258b..73aaff9a11 100644
--- a/source3/nsswitch/winbindd.h
+++ b/source3/nsswitch/winbindd.h
@@ -199,6 +199,10 @@ struct winbindd_domain {
 
 	struct winbindd_child child;
 
+	/* Callback we use to try put us back online. */
+
+	struct timed_event *check_online_event;
+
 	/* Linked list info */
 
 	struct winbindd_domain *prev, *next;
diff --git a/source3/nsswitch/winbindd_cache.c b/source3/nsswitch/winbindd_cache.c
index 27d37267b5..cb4f405ee0 100644
--- a/source3/nsswitch/winbindd_cache.c
+++ b/source3/nsswitch/winbindd_cache.c
@@ -473,12 +473,10 @@ static BOOL centry_expired(struct winbindd_domain *domain, const char *keystr, s
 		return False;
 	}
 
-	/* when the domain is offline and we havent checked in the last 30
-	 * seconds if it has become online again, return the cached entry.
+	/* when the domain is offline return the cached entry.
 	 * This deals with transient offline states... */
 
-	if (!domain->online && 
-	    !NT_STATUS_IS_OK(check_negative_conn_cache(domain->name, domain->dcname))) {
+	if (!domain->online) {
 		DEBUG(10,("centry_expired: Key %s for domain %s valid as domain is offline.\n",
 			keystr, domain->name ));
 		return False;
@@ -2552,7 +2550,7 @@ void set_global_winbindd_state_online(void)
 	tdb_delete_bystring(wcache->tdb, "WINBINDD_OFFLINE");
 }
 
-BOOL get_global_winbindd_state_online(void)
+BOOL get_global_winbindd_state_offline(void)
 {
 	return global_winbindd_offline_state;
 }
diff --git a/source3/nsswitch/winbindd_cm.c b/source3/nsswitch/winbindd_cm.c
index 76c0c48232..4b9072621a 100644
--- a/source3/nsswitch/winbindd_cm.c
+++ b/source3/nsswitch/winbindd_cm.c
@@ -64,6 +64,113 @@
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
 
+static NTSTATUS init_dc_connection(struct winbindd_domain *domain);
+
+/****************************************************************
+ Handler triggered if we're offline to try and detect a DC.
+****************************************************************/
+
+static void check_domain_online_handler(struct timed_event *te,
+					const struct timeval *now,
+					void *private_data)
+{
+        struct winbindd_domain *domain =
+                (struct winbindd_domain *)private_data;
+
+	DEBUG(10,("check_domain_online_handler: called for domain %s\n",
+		domain->name ));
+
+	if (domain->check_online_event) {
+		TALLOC_FREE(domain->check_online_event);
+	}
+
+	/* We've been told to stay offline, so stay
+	   that way. */
+
+	if (get_global_winbindd_state_offline()) {
+		DEBUG(10,("check_domain_online_handler: domain %s remaining globally offline\n",
+			domain->name ));
+		return;
+	}
+
+	/* This call takes care of setting the online
+	   flag to true if we connected, or re-adding
+	   the offline handler if false. */
+	init_dc_connection(domain);
+}
+
+/****************************************************************
+ Set domain offline and also add handler to put us back online
+ if we detect a DC.
+****************************************************************/
+
+void set_domain_offline(struct winbindd_domain *domain)
+{
+	DEBUG(10,("set_domain_offline: called for domain %s\n",
+		domain->name ));
+
+	if (domain->check_online_event) {
+		TALLOC_FREE(domain->check_online_event);
+	}
+
+	domain->online = False;
+
+	/* We only add the timeout handler that checks and
+	   allows us to go back online when we've not
+	   been told to remain offline. */
+
+	if (get_global_winbindd_state_offline()) {
+		DEBUG(10,("set_domain_offline: domain %s remaining globally offline\n",
+			domain->name ));
+		return;
+	}
+
+	domain->check_online_event = add_timed_event( NULL,
+						timeval_current_ofs(lp_winbind_cache_time(), 0),
+						"check_domain_online_handler",
+						check_domain_online_handler,
+						domain);
+
+	/* The above *has* to succeed for winbindd to work. */
+	if (!domain->check_online_event) {
+		smb_panic("set_domain_offline: failed to add online handler.\n");
+	}
+
+	DEBUG(10,("set_domain_offline: added event handler for domain %s\n",
+		domain->name ));
+}
+
+/****************************************************************
+ Set domain online - if allowed.
+****************************************************************/
+
+void set_domain_online(struct winbindd_domain *domain)
+{
+	DEBUG(10,("set_domain_offline: called for domain %s\n",
+		domain->name ));
+
+	if (get_global_winbindd_state_offline()) {
+		DEBUG(10,("set_domain_online: domain %s remaining globally offline\n",
+			domain->name ));
+		return;
+	}
+
+	domain->online = True;
+}
+
+/****************************************************************
+ Add -ve connection cache entries for domain and realm.
+****************************************************************/
+
+void winbind_add_failed_connection_entry(const struct winbindd_domain *domain,
+					const char *server,
+					NTSTATUS result)
+{
+	add_failed_connection_entry(domain->name, server, result);
+	if (*domain->alt_name) {
+		add_failed_connection_entry(domain->alt_name, server, result);
+	}
+}
 
 /* Choose between anonymous or authenticated connections.  We need to use
    an authenticated connection if DCs have the RestrictAnonymous registry
@@ -403,7 +510,7 @@ static NTSTATUS cm_prepare_connection(const struct winbindd_domain *domain,
 	SAFE_FREE(ipc_password);
 
 	if (!NT_STATUS_IS_OK(result)) {
-		add_failed_connection_entry(domain->name, controller, result);
+		winbind_add_failed_connection_entry(domain, controller, result);
 		if ((*cli) != NULL) {
 			cli_shutdown(*cli);
 			*cli = NULL;
@@ -761,7 +868,7 @@ static BOOL find_new_dc(TALLOC_CTX *mem_ctx,
 	if ( !open_any_socket_out(addrs, num_addrs, 5000, &fd_index, fd) ) 
 	{
 		for (i=0; i<num_dcs; i++) {
-			add_failed_connection_entry(domain->name,
+			winbind_add_failed_connection_entry(domain,
 				dcs[i].name, NT_STATUS_UNSUCCESSFUL);
 		}
 		return False;
@@ -782,7 +889,7 @@ static BOOL find_new_dc(TALLOC_CTX *mem_ctx,
 	}
 
 	/* We can not continue without the DC's name */
-	add_failed_connection_entry(domain->name, dcs[fd_index].name,
+	winbind_add_failed_connection_entry(domain, dcs[fd_index].name,
 				    NT_STATUS_UNSUCCESSFUL);
 	goto again;
 }
@@ -797,6 +904,7 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 
 	if ((mem_ctx = talloc_init("cm_open_connection")) == NULL) {
 		SAFE_FREE(saf_servername);
+		set_domain_offline(domain);
 		return NT_STATUS_NO_MEMORY;
 	}
 
@@ -819,8 +927,8 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 					  &domain->sid, ip, saf_name )) {
 				fstrcpy( domain->dcname, saf_name );
 			} else {
-				add_failed_connection_entry(
-					domain->name, saf_servername,
+				winbind_add_failed_connection_entry(
+					domain, saf_servername,
 					NT_STATUS_UNSUCCESSFUL);
 			}
 		} else {
@@ -850,7 +958,6 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 
 			/* 5 second timeout. */
 			if (!open_any_socket_out(addrs, num_addrs, 5000, &dummy, &fd)) {
-				domain->online = False;
 				fd = -1;
 			}
 		}
@@ -863,7 +970,6 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 			   to true, if a "WINBINDD_OFFLINE" entry
 			   is found in the winbindd cache. */
 			set_global_winbindd_state_offline();
-			domain->online = False;
 			break;
 		}
 
@@ -881,7 +987,10 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 			/* We're changing state from offline to online. */
 			set_global_winbindd_state_online();
 		}
-		domain->online = True;
+		set_domain_online(domain);
+	} else {
+		/* Ensure we setup the retry handler. */
+		set_domain_offline(domain);
 	}
 
 	talloc_destroy(mem_ctx);
diff --git a/source3/nsswitch/winbindd_dual.c b/source3/nsswitch/winbindd_dual.c
index 55f897603c..f19672be16 100644
--- a/source3/nsswitch/winbindd_dual.c
+++ b/source3/nsswitch/winbindd_dual.c
@@ -576,7 +576,17 @@ static void child_msg_offline(int msg_type, struct process_id src, void *buf, si
 
 	for (domain = domain_list(); domain; domain = domain->next) {
 		DEBUG(5,("child_msg_offline: marking %s offline.\n", domain->name));
-		domain->online = False;
+		set_domain_offline(domain);
+	}
+}
+
+/* Ensure any negative cache entries with the netbios or realm names are removed. */
+
+static void winbindd_flush_negative_conn_cache(struct winbindd_domain *domain)
+{
+	check_negative_conn_cache_timeout(domain->name, domain->dcname, 0);
+	if (*domain->alt_name) {
+		check_negative_conn_cache_timeout(domain->alt_name, domain->dcname, 0);
 	}
 }
 
@@ -599,12 +609,12 @@ static void child_msg_online(int msg_type, struct process_id src, void *buf, siz
 	winbindd_flush_nscd_cache();
 
 	/* Mark everything online - delete any negative cache entries
-	   to force an immediate reconnect. */
+	   to force a reconnect on the next query from the parent to this child. */
 
 	for (domain = domain_list(); domain; domain = domain->next) {
 		DEBUG(5,("child_msg_online: marking %s online.\n", domain->name));
-		domain->online = True;
-		check_negative_conn_cache_timeout(domain->name, domain->dcname, 0);
+		set_domain_online(domain);
+		winbindd_flush_negative_conn_cache(domain);
 	}
 }
 
@@ -614,7 +624,7 @@ static const char *collect_onlinestatus(TALLOC_CTX *mem_ctx)
 	char *buf = NULL;
 
 	if ((buf = talloc_asprintf(mem_ctx, "global:%s ", 
-				   get_global_winbindd_state_online() ? 
+				   get_global_winbindd_state_offline() ? 
 				   "Offline":"Online")) == NULL) {
 		return NULL;
 	}
diff --git a/source3/nsswitch/winbindd_rpc.c b/source3/nsswitch/winbindd_rpc.c
index 122e2c98e0..7f3ebdeee4 100644
--- a/source3/nsswitch/winbindd_rpc.c
+++ b/source3/nsswitch/winbindd_rpc.c
@@ -764,19 +764,19 @@ static int get_ldap_seq(const char *server, int port, uint32 *seq)
 
 /**********************************************************************
  Get the sequence number for a Windows AD native mode domain using
- LDAP queries
+ LDAP queries. 
 **********************************************************************/
 
-static int get_ldap_sequence_number( const char* domain, uint32 *seq)
+static int get_ldap_sequence_number(struct winbindd_domain *domain, uint32 *seq)
 {
 	int ret = -1;
 	int i, port = LDAP_PORT;
 	struct ip_service *ip_list = NULL;
 	int count;
 	
-	if ( !NT_STATUS_IS_OK(get_sorted_dc_list(domain, &ip_list, &count,
+	if ( !NT_STATUS_IS_OK(get_sorted_dc_list(domain->name, &ip_list, &count,
 						 False)) ) {
-		DEBUG(3, ("Could not look up dc's for domain %s\n", domain));
+		DEBUG(3, ("Could not look up dc's for domain %s\n", domain->name));
 		return False;
 	}
 
@@ -799,7 +799,7 @@ static int get_ldap_sequence_number( const char* domain, uint32 *seq)
 			goto done;
 
 		/* add to failed connection cache */
-		add_failed_connection_entry( domain, ipstr,
+		winbind_add_failed_connection_entry( domain, ipstr,
 					     NT_STATUS_UNSUCCESSFUL );
 	}
 
@@ -807,7 +807,7 @@ done:
 	if ( ret == 0 ) {
 		DEBUG(3, ("get_ldap_sequence_number: Retrieved sequence "
 			  "number for Domain (%s) from DC (%s:%d)\n", 
-			domain, inet_ntoa(ip_list[i].ip), port));
+			domain->name, inet_ntoa(ip_list[i].ip), port));
 	}
 
 	SAFE_FREE(ip_list);