s3:winbind: Fork multiple children per domain

This makes us scale better with many simultaneous winbind requests,
some of which might be slow.

This implementation breaks offline logons, as the cached credentials are
maintained in a child (this needs fixing). So, if the offline logons are
active, only allow one DC connection.

Probably the offline logon and the scalable file server cases are
separate enough so that this patch is useful even with the restriction.

22 files changed, 92 insertions, 28 deletions

diff --git a/source3/winbindd/wb_dsgetdcname.c b/source3/winbindd/wb_dsgetdcname.c
index fdd9a6d7df..207d1b61ea 100644
--- a/source3/winbindd/wb_dsgetdcname.c
+++ b/source3/winbindd/wb_dsgetdcname.c
@@ -64,7 +64,7 @@ struct tevent_req *wb_dsgetdcname_send(TALLOC_CTX *mem_ctx,
 		child = locator_child();
 	} else {
 		struct winbindd_domain *domain = find_our_domain();
-		child = &domain->child;
+		child = choose_domain_child(domain);
 	}
 
 	if (domain_guid != NULL) {
diff --git a/source3/winbindd/wb_group_members.c b/source3/winbindd/wb_group_members.c
index 024b3173b9..c9603e73f8 100644
--- a/source3/winbindd/wb_group_members.c
+++ b/source3/winbindd/wb_group_members.c
@@ -73,7 +73,7 @@ static struct tevent_req *wb_lookupgroupmem_send(TALLOC_CTX *mem_ctx,
 	}
 
 	subreq = dcerpc_wbint_LookupGroupMembers_send(
-		state, ev, domain->child.binding_handle, &state->sid, type,
+		state, ev, dom_child_handle(domain), &state->sid, type,
 		&state->members);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
diff --git a/source3/winbindd/wb_lookupname.c b/source3/winbindd/wb_lookupname.c
index 4429149f7a..a9b4dfa586 100644
--- a/source3/winbindd/wb_lookupname.c
+++ b/source3/winbindd/wb_lookupname.c
@@ -70,7 +70,8 @@ struct tevent_req *wb_lookupname_send(TALLOC_CTX *mem_ctx,
 	}
 
 	subreq = dcerpc_wbint_LookupName_send(
-		state, ev, domain->child.binding_handle, state->dom_name, state->name,
+		state, ev, dom_child_handle(domain),
+		state->dom_name, state->name,
 		flags, &state->type, &state->sid);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
@@ -111,7 +112,8 @@ static void wb_lookupname_done(struct tevent_req *subreq)
 	}
 
 	subreq = dcerpc_wbint_LookupName_send(
-		state, state->ev, root_domain->child.binding_handle, state->dom_name,
+		state, state->ev, dom_child_handle(root_domain),
+		state->dom_name,
 		state->name, state->flags, &state->type, &state->sid);
 	if (tevent_req_nomem(subreq, req)) {
 		return;
diff --git a/source3/winbindd/wb_lookupsid.c b/source3/winbindd/wb_lookupsid.c
index 52e8bcbdea..2e1b21a8d8 100644
--- a/source3/winbindd/wb_lookupsid.c
+++ b/source3/winbindd/wb_lookupsid.c
@@ -56,7 +56,7 @@ struct tevent_req *wb_lookupsid_send(TALLOC_CTX *mem_ctx,
 	}
 
 	subreq = dcerpc_wbint_LookupSid_send(
-		state, ev, state->lookup_domain->child.binding_handle,
+		state, ev, dom_child_handle(state->lookup_domain),
 		&state->sid, &state->type, &state->domname, &state->name);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
@@ -96,7 +96,7 @@ static void wb_lookupsid_done(struct tevent_req *subreq)
 	state->lookup_domain = forest_root;
 
 	subreq = dcerpc_wbint_LookupSid_send(
-		state, state->ev, state->lookup_domain->child.binding_handle,
+		state, state->ev, dom_child_handle(state->lookup_domain),
 		&state->sid, &state->type, &state->domname, &state->name);
 	if (tevent_req_nomem(subreq, req)) {
 		return;
diff --git a/source3/winbindd/wb_lookupuseraliases.c b/source3/winbindd/wb_lookupuseraliases.c
index 1e9b4c318e..f6fe855a85 100644
--- a/source3/winbindd/wb_lookupuseraliases.c
+++ b/source3/winbindd/wb_lookupuseraliases.c
@@ -47,7 +47,7 @@ struct tevent_req *wb_lookupuseraliases_send(TALLOC_CTX *mem_ctx,
 	state->sids.sids = CONST_DISCARD(struct dom_sid *, sids);
 
 	subreq = dcerpc_wbint_LookupUserAliases_send(
-		state, ev, domain->child.binding_handle, &state->sids, &state->rids);
+		state, ev, dom_child_handle(domain), &state->sids, &state->rids);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/winbindd/wb_lookupusergroups.c b/source3/winbindd/wb_lookupusergroups.c
index 27a6a5b00e..aeffc178d9 100644
--- a/source3/winbindd/wb_lookupusergroups.c
+++ b/source3/winbindd/wb_lookupusergroups.c
@@ -46,7 +46,7 @@ struct tevent_req *wb_lookupusergroups_send(TALLOC_CTX *mem_ctx,
 	sid_copy(&state->sid, sid);
 
 	subreq = dcerpc_wbint_LookupUserGroups_send(
-		state, ev, domain->child.binding_handle, &state->sid, &state->sids);
+		state, ev, dom_child_handle(domain), &state->sid, &state->sids);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/winbindd/wb_next_grent.c b/source3/winbindd/wb_next_grent.c
index 7eec3e0b94..54c4c1c440 100644
--- a/source3/winbindd/wb_next_grent.c
+++ b/source3/winbindd/wb_next_grent.c
@@ -70,7 +70,7 @@ struct tevent_req *wb_next_grent_send(TALLOC_CTX *mem_ctx,
 			return tevent_req_post(req, ev);
 		}
 		subreq = dcerpc_wbint_QueryGroupList_send(
-			state, state->ev, state->gstate->domain->child.binding_handle,
+			state, state->ev, dom_child_handle(state->gstate->domain),
 			&state->next_groups);
 		if (tevent_req_nomem(subreq, req)) {
 			return tevent_req_post(req, ev);
@@ -134,7 +134,7 @@ static void wb_next_grent_fetch_done(struct tevent_req *subreq)
 			return;
 		}
 		subreq = dcerpc_wbint_QueryGroupList_send(
-			state, state->ev, state->gstate->domain->child.binding_handle,
+			state, state->ev, dom_child_handle(state->gstate->domain),
 			&state->next_groups);
 		if (tevent_req_nomem(subreq, req)) {
 			return;
diff --git a/source3/winbindd/wb_query_user_list.c b/source3/winbindd/wb_query_user_list.c
index c4eb200d88..abbf7668e9 100644
--- a/source3/winbindd/wb_query_user_list.c
+++ b/source3/winbindd/wb_query_user_list.c
@@ -41,7 +41,7 @@ struct tevent_req *wb_query_user_list_send(TALLOC_CTX *mem_ctx,
 	}
 
 	subreq = dcerpc_wbint_QueryUserList_send(state, ev,
-						 domain->child.binding_handle,
+						 dom_child_handle(domain),
 						 &state->users);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
diff --git a/source3/winbindd/wb_queryuser.c b/source3/winbindd/wb_queryuser.c
index 870bfd3366..33416b9017 100644
--- a/source3/winbindd/wb_queryuser.c
+++ b/source3/winbindd/wb_queryuser.c
@@ -54,7 +54,7 @@ struct tevent_req *wb_queryuser_send(TALLOC_CTX *mem_ctx,
 		return tevent_req_post(req, ev);
 	}
 
-	subreq = dcerpc_wbint_QueryUser_send(state, ev, domain->child.binding_handle,
+	subreq = dcerpc_wbint_QueryUser_send(state, ev, dom_child_handle(domain),
 					     &state->sid, state->info);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
diff --git a/source3/winbindd/wb_seqnum.c b/source3/winbindd/wb_seqnum.c
index 0a314e7abe..4a92e036dc 100644
--- a/source3/winbindd/wb_seqnum.c
+++ b/source3/winbindd/wb_seqnum.c
@@ -39,7 +39,7 @@ struct tevent_req *wb_seqnum_send(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 	subreq = dcerpc_wbint_QuerySequenceNumber_send(
-		state, ev, domain->child.binding_handle, &state->seqnum);
+		state, ev, dom_child_handle(domain), &state->seqnum);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/winbindd/winbindd.h b/source3/winbindd/winbindd.h
index d276b86947..3217acc8ea 100644
--- a/source3/winbindd/winbindd.h
+++ b/source3/winbindd/winbindd.h
@@ -205,7 +205,7 @@ struct winbindd_domain {
 
 	/* The child pid we're talking to */
 
-	struct winbindd_child child;
+	struct winbindd_child *children;
 
 	/* Callback we use to try put us back online. */
 
diff --git a/source3/winbindd/winbindd_change_machine_acct.c b/source3/winbindd/winbindd_change_machine_acct.c
index ad050ad742..a5514e22eb 100644
--- a/source3/winbindd/winbindd_change_machine_acct.c
+++ b/source3/winbindd/winbindd_change_machine_acct.c
@@ -58,7 +58,7 @@ struct tevent_req *winbindd_change_machine_acct_send(TALLOC_CTX *mem_ctx,
 	}
 
 	subreq = dcerpc_wbint_ChangeMachineAccount_send(state, ev,
-							domain->child.binding_handle);
+							dom_child_handle(domain));
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/winbindd/winbindd_check_machine_acct.c b/source3/winbindd/winbindd_check_machine_acct.c
index 6d969c9353..279370146a 100644
--- a/source3/winbindd/winbindd_check_machine_acct.c
+++ b/source3/winbindd/winbindd_check_machine_acct.c
@@ -62,7 +62,7 @@ struct tevent_req *winbindd_check_machine_acct_send(TALLOC_CTX *mem_ctx,
 	}
 
 	subreq = dcerpc_wbint_CheckMachineAccount_send(state, ev,
-						       domain->child.binding_handle);
+						       dom_child_handle(domain));
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/winbindd/winbindd_domain.c b/source3/winbindd/winbindd_domain.c
index cd3370852b..e998275c8e 100644
--- a/source3/winbindd/winbindd_domain.c
+++ b/source3/winbindd/winbindd_domain.c
@@ -69,6 +69,12 @@ static const struct winbindd_child_dispatch_table domain_dispatch_table[] = {
 
 void setup_domain_child(struct winbindd_domain *domain)
 {
-	setup_child(domain, &domain->child, domain_dispatch_table,
-		    "log.wb", domain->name);
+	int i;
+
+        for (i=0; i<lp_winbind_max_domain_connections(); i++) {
+                setup_child(domain, &domain->children[i],
+			    domain_dispatch_table,
+                            "log.wb", domain->name);
+		domain->children[i].domain = domain;
+	}
 }
diff --git a/source3/winbindd/winbindd_dual.c b/source3/winbindd/winbindd_dual.c
index 344b7e4bd1..a32459d595 100644
--- a/source3/winbindd/winbindd_dual.c
+++ b/source3/winbindd/winbindd_dual.c
@@ -193,9 +193,47 @@ int wb_child_request_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
 	return 0;
 }
 
+static bool winbindd_child_busy(struct winbindd_child *child)
+{
+	return tevent_queue_length(child->queue) > 0;
+}
+
+static struct winbindd_child *find_idle_child(struct winbindd_domain *domain)
+{
+	int i;
+
+	for (i=0; i<lp_winbind_max_domain_connections(); i++) {
+		if (!winbindd_child_busy(&domain->children[i])) {
+			return &domain->children[i];
+		}
+	}
+
+	return NULL;
+}
+
+struct winbindd_child *choose_domain_child(struct winbindd_domain *domain)
+{
+	struct winbindd_child *result;
+
+	result = find_idle_child(domain);
+	if (result != NULL) {
+		return result;
+	}
+	return &domain->children[rand() % lp_winbind_max_domain_connections()];
+}
+
+struct dcerpc_binding_handle *dom_child_handle(struct winbindd_domain *domain)
+{
+	struct winbindd_child *child;
+
+	child = choose_domain_child(domain);
+	return child->binding_handle;
+}
+
 struct wb_domain_request_state {
 	struct tevent_context *ev;
 	struct winbindd_domain *domain;
+	struct winbindd_child *child;
 	struct winbindd_request *request;
 	struct winbindd_request *init_req;
 	struct winbindd_response *response;
@@ -219,8 +257,10 @@ struct tevent_req *wb_domain_request_send(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
+	state->child = choose_domain_child(domain);
+
 	if (domain->initialized) {
-		subreq = wb_child_request_send(state, ev, &domain->child,
+		subreq = wb_child_request_send(state, ev, state->child,
 					       request);
 		if (tevent_req_nomem(subreq, req)) {
 			return tevent_req_post(req, ev);
@@ -245,7 +285,7 @@ struct tevent_req *wb_domain_request_send(TALLOC_CTX *mem_ctx,
 		state->init_req->data.init_conn.is_primary = domain->primary;
 		fstrcpy(state->init_req->data.init_conn.dcname, "");
 
-		subreq = wb_child_request_send(state, ev, &domain->child,
+		subreq = wb_child_request_send(state, ev, state->child,
 					       state->init_req);
 		if (tevent_req_nomem(subreq, req)) {
 			return tevent_req_post(req, ev);
@@ -266,7 +306,7 @@ struct tevent_req *wb_domain_request_send(TALLOC_CTX *mem_ctx,
 	state->init_req->cmd = WINBINDD_GETDCNAME;
 	fstrcpy(state->init_req->domain_name, domain->name);
 
-	subreq = wb_child_request_send(state, ev, &domain->child, request);
+	subreq = wb_child_request_send(state, ev, state->child, request);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
@@ -297,7 +337,7 @@ static void wb_domain_request_gotdc(struct tevent_req *subreq)
 
 	TALLOC_FREE(response);
 
-	subreq = wb_child_request_send(state, state->ev, &state->domain->child,
+	subreq = wb_child_request_send(state, state->ev, state->child,
 				       state->init_req);
 	if (tevent_req_nomem(subreq, req)) {
 		return;
@@ -337,7 +377,7 @@ static void wb_domain_request_initialized(struct tevent_req *subreq)
 
 	TALLOC_FREE(response);
 
-	subreq = wb_child_request_send(state, state->ev, &state->domain->child,
+	subreq = wb_child_request_send(state, state->ev, state->child,
 				       state->request);
 	if (tevent_req_nomem(subreq, req)) {
 		return;
diff --git a/source3/winbindd/winbindd_list_groups.c b/source3/winbindd/winbindd_list_groups.c
index 4236794485..2e2c70a33d 100644
--- a/source3/winbindd/winbindd_list_groups.c
+++ b/source3/winbindd/winbindd_list_groups.c
@@ -91,7 +91,7 @@ struct tevent_req *winbindd_list_groups_send(TALLOC_CTX *mem_ctx,
 		struct winbindd_list_groups_domstate *d = &state->domains[i];
 
 		d->subreq = dcerpc_wbint_QueryGroupList_send(
-			state->domains, ev, d->domain->child.binding_handle,
+			state->domains, ev, dom_child_handle(d->domain),
 			&d->groups);
 		if (tevent_req_nomem(d->subreq, req)) {
 			TALLOC_FREE(state->domains);
diff --git a/source3/winbindd/winbindd_list_users.c b/source3/winbindd/winbindd_list_users.c
index 18b8fdd589..54e0106e3d 100644
--- a/source3/winbindd/winbindd_list_users.c
+++ b/source3/winbindd/winbindd_list_users.c
@@ -91,7 +91,7 @@ struct tevent_req *winbindd_list_users_send(TALLOC_CTX *mem_ctx,
 		struct winbindd_list_users_domstate *d = &state->domains[i];
 
 		d->subreq = dcerpc_wbint_QueryUserList_send(
-			state->domains, ev, d->domain->child.binding_handle,
+			state->domains, ev, dom_child_handle(d->domain),
 			&d->users);
 		if (tevent_req_nomem(d->subreq, req)) {
 			TALLOC_FREE(state->domains);
diff --git a/source3/winbindd/winbindd_lookuprids.c b/source3/winbindd/winbindd_lookuprids.c
index b921818bc6..738adbaefc 100644
--- a/source3/winbindd/winbindd_lookuprids.c
+++ b/source3/winbindd/winbindd_lookuprids.c
@@ -84,7 +84,7 @@ struct tevent_req *winbindd_lookuprids_send(TALLOC_CTX *mem_ctx,
 	}
 
 	subreq = dcerpc_wbint_LookupRids_send(
-		state, ev, domain->child.binding_handle, &state->rids,
+		state, ev, dom_child_handle(domain), &state->rids,
 		&state->domain_name, &state->names);
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
diff --git a/source3/winbindd/winbindd_ndr.c b/source3/winbindd/winbindd_ndr.c
index 396aa06985..94aa7eb965 100644
--- a/source3/winbindd/winbindd_ndr.c
+++ b/source3/winbindd/winbindd_ndr.c
@@ -120,6 +120,7 @@ void ndr_print_winbindd_domain(struct ndr_print *ndr,
 			       const char *name,
 			       const struct winbindd_domain *r)
 {
+	int i;
 	if (!r) {
 		return;
 	}
@@ -150,7 +151,9 @@ void ndr_print_winbindd_domain(struct ndr_print *ndr,
 	ndr_print_uint32(ndr, "sequence_number", r->sequence_number);
 	ndr_print_NTSTATUS(ndr, "last_status", r->last_status);
 	ndr_print_winbindd_cm_conn(ndr, "conn", &r->conn);
-	ndr_print_winbindd_child(ndr, "child", &r->child);
+	for (i=0; i<lp_winbind_max_domain_connections(); i++) {
+		ndr_print_winbindd_child(ndr, "children", &r->children[i]);
+	}
 	ndr_print_uint32(ndr, "check_online_timeout", r->check_online_timeout);
 	ndr_print_ptr(ndr, "check_online_event", r->check_online_event);
 	ndr->depth--;
diff --git a/source3/winbindd/winbindd_ping_dc.c b/source3/winbindd/winbindd_ping_dc.c
index 36c4def0ab..2304828030 100644
--- a/source3/winbindd/winbindd_ping_dc.c
+++ b/source3/winbindd/winbindd_ping_dc.c
@@ -61,7 +61,7 @@ struct tevent_req *winbindd_ping_dc_send(TALLOC_CTX *mem_ctx,
 		return tevent_req_post(req, ev);
 	}
 
-	subreq = dcerpc_wbint_PingDc_send(state, ev, domain->child.binding_handle);
+	subreq = dcerpc_wbint_PingDc_send(state, ev, dom_child_handle(domain));
 	if (tevent_req_nomem(subreq, req)) {
 		return tevent_req_post(req, ev);
 	}
diff --git a/source3/winbindd/winbindd_proto.h b/source3/winbindd/winbindd_proto.h
index 5cb6c4c155..c822baa462 100644
--- a/source3/winbindd/winbindd_proto.h
+++ b/source3/winbindd/winbindd_proto.h
@@ -227,6 +227,9 @@ void setup_domain_child(struct winbindd_domain *domain);
 
 /* The following definitions come from winbindd/winbindd_dual.c  */
 
+struct dcerpc_binding_handle *dom_child_handle(struct winbindd_domain *domain);
+struct winbindd_child *choose_domain_child(struct winbindd_domain *domain);
+
 struct tevent_req *wb_child_request_send(TALLOC_CTX *mem_ctx,
 					 struct tevent_context *ev,
 					 struct winbindd_child *child,
diff --git a/source3/winbindd/winbindd_util.c b/source3/winbindd/winbindd_util.c
index e27c35fe60..58cec444aa 100644
--- a/source3/winbindd/winbindd_util.c
+++ b/source3/winbindd/winbindd_util.c
@@ -162,6 +162,16 @@ static struct winbindd_domain *add_trusted_domain(const char *domain_name, const
 
 	ZERO_STRUCTP(domain);
 
+	domain->children = SMB_MALLOC_ARRAY(
+		struct winbindd_child, lp_winbind_max_domain_connections());
+	if (domain->children == NULL) {
+		SAFE_FREE(domain);
+		return NULL;
+	}
+	memset(domain->children, 0,
+	       sizeof(struct winbindd_child)
+	       * lp_winbind_max_domain_connections());
+
 	fstrcpy(domain->name, domain_name);
 	if (alternative_name) {
 		fstrcpy(domain->alt_name, alternative_name);