Fix for bug 707, getent group for huge ads groups (>1500 members)

This introduces range retrieval of ADS attributes.

VL rewrote most of Günther's patch, partly to remove code duplication and
partly to get the retrieval of members in one rush, not interrupted by the
lookups for the DN.

I rewrote that patch, to ensure that we can keep an eye on the USN
(sequence number) of the entry - this allows us to ensure the read was
atomic.

In particular, the range retrieval is now generic, for strings.  It
could easily be made generic for any attribute type, if need be.

Andrew Bartlett
(This used to be commit 131bb928f19c7b1f582c4ad9ac42e5f3d9dfb622)

2 files changed, 202 insertions, 30 deletions

diff --git a/source3/libads/ldap.c b/source3/libads/ldap.c
index ce0341b72c..f9c7820275 100644
--- a/source3/libads/ldap.c
+++ b/source3/libads/ldap.c
@@ -1579,27 +1579,26 @@ char *ads_pull_string(ADS_STRUCT *ads,
  * @return Result strings in talloc context
  **/
 char **ads_pull_strings(ADS_STRUCT *ads, 
-		       TALLOC_CTX *mem_ctx, void *msg, const char *field)
+			TALLOC_CTX *mem_ctx, void *msg, const char *field,
+			size_t *num_values)
 {
 	char **values;
 	char **ret = NULL;
-	int i, n;
+	int i;
 
 	values = ldap_get_values(ads->ld, msg, field);
 	if (!values)
 		return NULL;
 
-	for (i=0;values[i];i++)
-		/* noop */ ;
-	n = i;
+	*num_values = ldap_count_values(values);
 
-	ret = talloc(mem_ctx, sizeof(char *) * (n+1));
+	ret = talloc(mem_ctx, sizeof(char *) * (*num_values+1));
 	if (!ret) {
 		ldap_value_free(values);
 		return NULL;
 	}
 
-	for (i=0;i<n;i++) {
+	for (i=0;i<*num_values;i++) {
 		if (pull_utf8_talloc(mem_ctx, &ret[i], values[i]) == -1) {
 			ldap_value_free(values);
 			return NULL;
@@ -1611,6 +1610,127 @@ char **ads_pull_strings(ADS_STRUCT *ads,
 	return ret;
 }
 
+/**
+ * pull an array of strings from a ADS result 
+ *  (handle large multivalue attributes with range retrieval)
+ * @param ads connection to ads server
+ * @param mem_ctx TALLOC_CTX to use for allocating result string
+ * @param msg Results of search
+ * @param field Attribute to retrieve
+ * @param current_strings strings returned by a previous call to this function
+ * @param next_attribute The next query should ask for this attribute
+ * @param num_values How many values did we get this time?
+ * @param more_values Are there more values to get?
+ * @return Result strings in talloc context
+ **/
+char **ads_pull_strings_range(ADS_STRUCT *ads, 
+			      TALLOC_CTX *mem_ctx,
+			      void *msg, const char *field,
+			      char **current_strings,
+			      const char **next_attribute,
+			      size_t *num_strings,
+			      BOOL *more_strings)
+{
+	char *attr;
+	char *expected_range_attrib, *range_attr;
+	BerElement *ptr = NULL;
+	char **strings;
+	char **new_strings;
+	size_t num_new_strings;
+	unsigned long int range_start;
+	unsigned long int range_end;
+	
+	/* we might have been given the whole lot anyway */
+	if ((strings = ads_pull_strings(ads, mem_ctx, msg, field, num_strings))) {
+		*more_strings = False;
+		return strings;
+	}
+
+	expected_range_attrib = talloc_asprintf(mem_ctx, "%s;Range=", field);
+
+	/* look for Range result */
+	for (attr = ldap_first_attribute(ads->ld, (LDAPMessage *)msg, &ptr); 
+	     attr; 
+	     attr = ldap_next_attribute(ads->ld, (LDAPMessage *)msg, ptr)) {
+		/* we ignore the fact that this is utf8, as all attributes are ascii... */
+		if (strnequal(attr, expected_range_attrib, strlen(expected_range_attrib))) {
+			range_attr = attr;
+			break;
+		}
+		ldap_memfree(attr);
+	}
+	if (!attr) {
+		ber_free(ptr, 0);
+		/* nothing here - this feild is just empty */
+		*more_strings = False;
+		return NULL;
+	}
+	
+	if (sscanf(&range_attr[strlen(expected_range_attrib)], "%lu-%lu", 
+		   &range_start, &range_end) == 2) {
+		*more_strings = True;
+	} else {
+		if (sscanf(&range_attr[strlen(expected_range_attrib)], "%lu-*", 
+			   &range_start) == 1) {
+			*more_strings = False;
+		} else {
+			DEBUG(1, ("ads_pull_strings_range:  Cannot parse Range attriubte (%s)\n", range_attr));
+			ldap_memfree(range_attr);
+			*more_strings = False;
+			return NULL;
+		}
+	}
+
+	if ((*num_strings) != range_start) {
+		DEBUG(1, ("ads_pull_strings_range: Range attribute (%s) doesn't start at %u, but at %lu - aborting range retreival\n",
+			  range_attr, *num_strings + 1, range_start));
+		ldap_memfree(range_attr);
+		*more_strings = False;
+		return NULL;
+	}
+
+	new_strings = ads_pull_strings(ads, mem_ctx, msg, range_attr, &num_new_strings);
+	
+	if (*more_strings && ((*num_strings + num_new_strings) != (range_end + 1))) {
+		DEBUG(1, ("ads_pull_strings_range: Range attribute (%s) tells us we have %lu strings in this bunch, but we only got %lu - aborting range retreival\n",
+			  range_attr, (unsigned long int)range_end - range_start + 1, (unsigned long int)num_new_strings));
+		ldap_memfree(range_attr);
+		*more_strings = False;
+		return NULL;
+	}
+
+	strings = talloc_realloc(mem_ctx, current_strings,
+				 sizeof(*current_strings) *
+				 (*num_strings + num_new_strings));
+	
+	if (strings == NULL) {
+		ldap_memfree(range_attr);
+		*more_strings = False;
+		return NULL;
+	}
+	
+	memcpy(&strings[*num_strings], new_strings,
+	       sizeof(*new_strings) * num_new_strings);
+
+	(*num_strings) += num_new_strings;
+
+	if (*more_strings) {
+		*next_attribute = talloc_asprintf(mem_ctx,
+						  "member;range=%d-*", 
+						  *num_strings);
+		
+		if (!*next_attribute) {
+			DEBUG(1, ("talloc_asprintf for next attribute failed!\n"));
+			ldap_memfree(range_attr);
+			*more_strings = False;
+			return NULL;
+		}
+	}
+
+	ldap_memfree(range_attr);
+
+	return strings;
+}
 
 /**
  * pull a single uint32 from a ADS result
@@ -1960,6 +2080,7 @@ ADS_STATUS ads_workgroup_name(ADS_STRUCT *ads, TALLOC_CTX *mem_ctx, const char *
 	int i;
 	void *res;
 	const char *attrs[] = {"servicePrincipalName", NULL};
+	int num_principals;
 
 	(*workgroup) = NULL;
 
@@ -1972,7 +2093,8 @@ ADS_STATUS ads_workgroup_name(ADS_STRUCT *ads, TALLOC_CTX *mem_ctx, const char *
 		return rc;
 	}
 
-	principles = ads_pull_strings(ads, mem_ctx, res, "servicePrincipalName");
+	principles = ads_pull_strings(ads, mem_ctx, res,
+				      "servicePrincipalName", &num_principals);
 
 	ads_msgfree(ads, res);
 
diff --git a/source3/nsswitch/winbindd_ads.c b/source3/nsswitch/winbindd_ads.c
index 82d0afd03b..e66706c444 100644
--- a/source3/nsswitch/winbindd_ads.c
+++ b/source3/nsswitch/winbindd_ads.c
@@ -688,10 +688,14 @@ static NTSTATUS lookup_groupmem(struct winbindd_domain *domain,
 	char *ldap_exp;
 	NTSTATUS status = NT_STATUS_UNSUCCESSFUL;
 	char *sidstr;
-	const char *attrs[] = {"member", NULL};
 	char **members;
 	int i, num_members;
 	fstring sid_string;
+	BOOL more_values;
+	const char **attrs;
+	uint32 first_usn;
+	uint32 current_usn;
+	int num_retries = 0;
 
 	DEBUG(10,("ads: lookup_groupmem %s sid=%s\n", domain->name, sid_string_static(group_sid)));
 
@@ -707,34 +711,80 @@ static NTSTATUS lookup_groupmem(struct winbindd_domain *domain,
 	sidstr = sid_binstring(group_sid);
 
 	/* search for all members of the group */
-	asprintf(&ldap_exp, "(objectSid=%s)",sidstr);
-	rc = ads_search_retry(ads, &res, ldap_exp, attrs);
-	free(ldap_exp);
-	free(sidstr);
-
-	if (!ADS_ERR_OK(rc) || !res) {
-		DEBUG(1,("query_user_list ads_search: %s\n", ads_errstr(rc)));
+	if (!(ldap_exp = talloc_asprintf(mem_ctx, "(objectSid=%s)",sidstr))) {
+		SAFE_FREE(sidstr);
+		DEBUG(1, ("ads: lookup_groupmem: tallloc_asprintf for ldap_exp failed!\n"));
+		status = NT_STATUS_NO_MEMORY;
 		goto done;
 	}
+	SAFE_FREE(sidstr);
 
-	count = ads_count_replies(ads, res);
-	if (count == 0) {
-		status = NT_STATUS_OK;
-		goto done;
-	}
+	members = NULL;
+	num_members = 0;
 
-	members = ads_pull_strings(ads, mem_ctx, res, "member");
-	if (!members) {
-		/* no members? ok ... */
-		status = NT_STATUS_OK;
-		goto done;
-	}
+	attrs = talloc(mem_ctx, 3 * sizeof(*attrs));
+	attrs[1] = talloc_strdup(mem_ctx, "usnChanged");
+	attrs[2] = NULL;
+		
+	do {
+		if (num_members == 0) 
+			attrs[0] = talloc_strdup(mem_ctx, "member");
 
+		DEBUG(10, ("Searching for attrs[0] = %s, attrs[1] = %s\n", attrs[0], attrs[1]));
+
+		rc = ads_search_retry(ads, &res, ldap_exp, attrs);
+
+		if (!ADS_ERR_OK(rc) || !res) {
+			DEBUG(1,("ads: lookup_groupmem ads_search: %s\n",
+				 ads_errstr(rc)));
+			status = ads_ntstatus(rc);
+			goto done;
+		}
+
+		count = ads_count_replies(ads, res);
+		if (count == 0)
+			break;
+
+		if (num_members == 0) {
+			if (!ads_pull_uint32(ads, res, "usnChanged", &first_usn)) {
+				DEBUG(1, ("ads: lookup_groupmem could not pull usnChanged!\n"));
+				goto done;
+			}
+		}
+
+		if (!ads_pull_uint32(ads, res, "usnChanged", &current_usn)) {
+			DEBUG(1, ("ads: lookup_groupmem could not pull usnChanged!\n"));
+			goto done;
+		}
+
+		if (first_usn != current_usn) {
+			DEBUG(5, ("ads: lookup_groupmem USN on this record changed - restarting search\n"));
+			if (num_retries < 5) {
+				num_retries++;
+				num_members = 0;
+				continue;
+			} else {
+				DEBUG(5, ("ads: lookup_groupmem USN on this record changed - restarted search too many times, aborting!\n"));
+				status = NT_STATUS_UNSUCCESSFUL;
+				goto done;
+			}
+		}
+
+		members = ads_pull_strings_range(ads, mem_ctx, res,
+						 "member",
+						 members,
+						 &attrs[0],
+						 &num_members,
+						 &more_values);
+
+		if ((members == NULL) || (num_members == 0))
+			break;
+
+	} while (more_values);
+		
 	/* now we need to turn a list of members into rids, names and name types 
 	   the problem is that the members are in the form of distinguised names
 	*/
-	for (i=0;members[i];i++) /* noop */ ;
-	num_members = i;
 
 	(*sid_mem) = talloc_zero(mem_ctx, sizeof(**sid_mem) * num_members);
 	(*name_types) = talloc_zero(mem_ctx, sizeof(**name_types) * num_members);
@@ -761,13 +811,13 @@ static NTSTATUS lookup_groupmem(struct winbindd_domain *domain,
 	status = NT_STATUS_OK;
 	DEBUG(3,("ads lookup_groupmem for sid=%s\n", sid_to_string(sid_string, group_sid)));
 done:
+
 	if (res) 
 		ads_msgfree(ads, res);
 
 	return status;
 }
 
-
 /* find the sequence number for a domain */
 static NTSTATUS sequence_number(struct winbindd_domain *domain, uint32 *seq)
 {