changed the auth path to use extended DN ops to avoid non-indexed searches

Logs showed that every SAM authentication was causing a non-indexed
ldb search for member=XXX. This was previously indexed in Samba4, but
since we switched to using the indexes from the full AD schema it now
isn't.

The fix is to use the extended DN operations to allow us to ask the
server for the memberOf attribute instead, with with the SIDs attached
to the result. This also means one less search on every
authentication.

The patch is made more complex by the fact that some common routines
use the result of these user searches, so we had to update all
searches that uses user_attrs and those common routines to make sure
they all returned a ldb_message with a memberOf filled in and the SIDs
attached.

3 files changed, 91 insertions, 92 deletions

diff --git a/source4/auth/ntlm/auth_sam.c b/source4/auth/ntlm/auth_sam.c
index 75ed3243d4..0bb79e234c 100644
--- a/source4/auth/ntlm/auth_sam.c
+++ b/source4/auth/ntlm/auth_sam.c
@@ -43,32 +43,23 @@ extern const char *domain_ref_attrs[];
 static NTSTATUS authsam_search_account(TALLOC_CTX *mem_ctx, struct ldb_context *sam_ctx,
 				       const char *account_name,
 				       struct ldb_dn *domain_dn,
-				       struct ldb_message ***ret_msgs)
+				       struct ldb_message **ret_msg)
 {
-	struct ldb_message **msgs;
-
 	int ret;
 
 	/* pull the user attributes */
-	ret = gendb_search(sam_ctx, mem_ctx, domain_dn, &msgs, user_attrs,
-			   "(&(sAMAccountName=%s)(objectclass=user))", 
-			   ldb_binary_encode_string(mem_ctx, account_name));
-	if (ret == -1) {
-		return NT_STATUS_INTERNAL_DB_CORRUPTION;
-	}
-
-	if (ret == 0) {
+	ret = gendb_search_single_extended_dn(sam_ctx, mem_ctx, domain_dn, LDB_SCOPE_SUBTREE,
+					      ret_msg, user_attrs,
+					      "(&(sAMAccountName=%s)(objectclass=user))", 
+					      ldb_binary_encode_string(mem_ctx, account_name));
+	if (ret == LDB_ERR_NO_SUCH_OBJECT) {
 		DEBUG(3,("sam_search_user: Couldn't find user [%s] in samdb, under %s\n", 
 			 account_name, ldb_dn_get_linearized(domain_dn)));
-		return NT_STATUS_NO_SUCH_USER;
+		return NT_STATUS_NO_SUCH_USER;		
 	}
-
-	if (ret > 1) {
-		DEBUG(0,("Found %d records matching user [%s]\n", ret, account_name));
+	if (ret != LDB_SUCCESS) {
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
-
-	*ret_msgs = msgs;
 	
 	return NT_STATUS_OK;
 }
@@ -147,14 +138,14 @@ static NTSTATUS authsam_password_ok(struct auth_context *auth_context,
 static NTSTATUS authsam_authenticate(struct auth_context *auth_context, 
 				     TALLOC_CTX *mem_ctx, struct ldb_context *sam_ctx, 
 				     struct ldb_dn *domain_dn,
-				     struct ldb_message **msgs,
+				     struct ldb_message *msg,
 				     const struct auth_usersupplied_info *user_info, 
 				     DATA_BLOB *user_sess_key, DATA_BLOB *lm_sess_key) 
 {
 	struct samr_Password *lm_pwd, *nt_pwd;
 	NTSTATUS nt_status;
 
-	uint16_t acct_flags = samdb_result_acct_flags(sam_ctx, mem_ctx, msgs[0], domain_dn);
+	uint16_t acct_flags = samdb_result_acct_flags(sam_ctx, mem_ctx, msg, domain_dn);
 	
 	/* Quit if the account was locked out. */
 	if (acct_flags & ACB_AUTOLOCK) {
@@ -170,7 +161,7 @@ static NTSTATUS authsam_authenticate(struct auth_context *auth_context,
 		}
 	}
 
-	nt_status = samdb_result_passwords(mem_ctx, auth_context->lp_ctx, msgs[0], &lm_pwd, &nt_pwd);
+	nt_status = samdb_result_passwords(mem_ctx, auth_context->lp_ctx, msg, &lm_pwd, &nt_pwd);
 	NT_STATUS_NOT_OK_RETURN(nt_status);
 
 	nt_status = authsam_password_ok(auth_context, mem_ctx, 
@@ -181,7 +172,7 @@ static NTSTATUS authsam_authenticate(struct auth_context *auth_context,
 	nt_status = authsam_account_ok(mem_ctx, sam_ctx, 
 				       user_info->logon_parameters,
 				       domain_dn,
-				       msgs[0],
+				       msg,
 				       user_info->workstation_name,
 				       user_info->mapped.account_name,
 				       false);
@@ -198,7 +189,7 @@ static NTSTATUS authsam_check_password_internals(struct auth_method_context *ctx
 {
 	NTSTATUS nt_status;
 	const char *account_name = user_info->mapped.account_name;
-	struct ldb_message **msgs;
+	struct ldb_message *msg;
 	struct ldb_context *sam_ctx;
 	struct ldb_dn *domain_dn;
 	DATA_BLOB user_sess_key, lm_sess_key;
@@ -226,13 +217,13 @@ static NTSTATUS authsam_check_password_internals(struct auth_method_context *ctx
 		return NT_STATUS_NO_SUCH_DOMAIN;
 	}
 
-	nt_status = authsam_search_account(tmp_ctx, sam_ctx, account_name, domain_dn, &msgs);
+	nt_status = authsam_search_account(tmp_ctx, sam_ctx, account_name, domain_dn, &msg);
 	if (!NT_STATUS_IS_OK(nt_status)) {
 		talloc_free(tmp_ctx);
 		return nt_status;
 	}
 
-	nt_status = authsam_authenticate(ctx->auth_ctx, tmp_ctx, sam_ctx, domain_dn, msgs, user_info,
+	nt_status = authsam_authenticate(ctx->auth_ctx, tmp_ctx, sam_ctx, domain_dn, msg, user_info,
 					 &user_sess_key, &lm_sess_key);
 	if (!NT_STATUS_IS_OK(nt_status)) {
 		talloc_free(tmp_ctx);
@@ -242,7 +233,7 @@ static NTSTATUS authsam_check_password_internals(struct auth_method_context *ctx
 	nt_status = authsam_make_server_info(tmp_ctx, sam_ctx, lp_netbios_name(ctx->auth_ctx->lp_ctx), 
  					     lp_sam_name(ctx->auth_ctx->lp_ctx),
 					     domain_dn,
-					     msgs[0],
+					     msg,
 					     user_sess_key, lm_sess_key,
 					     server_info);
 	if (!NT_STATUS_IS_OK(nt_status)) {
@@ -322,7 +313,7 @@ NTSTATUS authsam_get_server_info_principal(TALLOC_CTX *mem_ctx,
 	DATA_BLOB user_sess_key = data_blob(NULL, 0);
 	DATA_BLOB lm_sess_key = data_blob(NULL, 0);
 
-	struct ldb_message **msgs;
+	struct ldb_message *msg;
 	struct ldb_context *sam_ctx;
 	struct ldb_dn *domain_dn;
 	
@@ -339,7 +330,7 @@ NTSTATUS authsam_get_server_info_principal(TALLOC_CTX *mem_ctx,
 	}
 
 	nt_status = sam_get_results_principal(sam_ctx, tmp_ctx, principal, 
-					      &domain_dn, &msgs);
+					      &domain_dn, &msg);
 	if (!NT_STATUS_IS_OK(nt_status)) {
 		return nt_status;
 	}
@@ -348,7 +339,7 @@ NTSTATUS authsam_get_server_info_principal(TALLOC_CTX *mem_ctx,
 					     lp_netbios_name(auth_context->lp_ctx),
  					     lp_workgroup(auth_context->lp_ctx),
 					     domain_dn, 
-					     msgs[0],
+					     msg,
 					     user_sess_key, lm_sess_key,
 					     server_info);
 	if (NT_STATUS_IS_OK(nt_status)) {
diff --git a/source4/auth/sam.c b/source4/auth/sam.c
index c70c02cb3c..68eaacf255 100644
--- a/source4/auth/sam.c
+++ b/source4/auth/sam.c
@@ -28,6 +28,7 @@
 #include "libcli/security/security.h"
 #include "libcli/ldap/ldap.h"
 #include "librpc/gen_ndr/ndr_netlogon.h"
+#include "librpc/gen_ndr/ndr_security.h"
 #include "param/param.h"
 #include "auth/auth_sam.h"
 
@@ -66,6 +67,7 @@ const char *user_attrs[] = {
 	"badPwdCount",
 	"logonCount",
 	"primaryGroupID",
+	"memberOf",
 	NULL,
 };
 
@@ -261,9 +263,7 @@ _PUBLIC_ NTSTATUS authsam_make_server_info(TALLOC_CTX *mem_ctx, struct ldb_conte
 					   struct auth_serversupplied_info **_server_info)
 {
 	struct auth_serversupplied_info *server_info;
-	struct ldb_message **group_msgs;
-	int group_ret;
-	const char *group_attrs[3] = { "sAMAccountType", "objectSid", NULL }; 
+	int group_ret = 0;
 	/* find list of sids */
 	struct dom_sid **groupSIDs = NULL;
 	struct dom_sid *account_sid;
@@ -271,39 +271,48 @@ _PUBLIC_ NTSTATUS authsam_make_server_info(TALLOC_CTX *mem_ctx, struct ldb_conte
 	const char *str;
 	int i;
 	uint_t rid;
-	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
-
-	group_ret = gendb_search(sam_ctx,
-				 tmp_ctx, NULL, &group_msgs, group_attrs,
-				 "(&(member=%s)(sAMAccountType=*))", 
-				 ldb_dn_get_linearized(msg->dn));
-	if (group_ret == -1) {
-		talloc_free(tmp_ctx);
-		return NT_STATUS_INTERNAL_DB_CORRUPTION;
-	}
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);	
+	struct ldb_message_element *el;
 
-	server_info = talloc(mem_ctx, struct auth_serversupplied_info);
-	NT_STATUS_HAVE_NO_MEMORY(server_info);
+	server_info = talloc(tmp_ctx, struct auth_serversupplied_info);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info, tmp_ctx);
 	
-	if (group_ret > 0) {
+	el = ldb_msg_find_element(msg, "memberOf");
+	if (el != NULL) {
+		group_ret = el->num_values;
 		groupSIDs = talloc_array(server_info, struct dom_sid *, group_ret);
-		NT_STATUS_HAVE_NO_MEMORY(groupSIDs);
+		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(groupSIDs, tmp_ctx);
 	}
 
-	/* Need to unroll some nested groups, but not aliases */
+	/* TODO Note: this is incomplete. We need to unroll some
+	 * nested groups, but not aliases */
 	for (i = 0; i < group_ret; i++) {
-		groupSIDs[i] = samdb_result_dom_sid(groupSIDs, 
-						    group_msgs[i], "objectSid");
-		NT_STATUS_HAVE_NO_MEMORY(groupSIDs[i]);
+		struct ldb_dn *dn;
+		const struct ldb_val *v;
+		enum ndr_err_code ndr_err;
+
+		dn = ldb_dn_from_ldb_val(tmp_ctx, sam_ctx, &el->values[i]);
+		if (dn == NULL) {
+			talloc_free(tmp_ctx);
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+		v = ldb_dn_get_extended_component(dn, "SID");
+		groupSIDs[i] = talloc(groupSIDs, struct dom_sid);
+		NT_STATUS_HAVE_NO_MEMORY_AND_FREE(groupSIDs[i], tmp_ctx);
+
+		ndr_err = ndr_pull_struct_blob(v, groupSIDs[i], NULL, groupSIDs[i], 
+					       (ndr_pull_flags_fn_t)ndr_pull_dom_sid);
+		if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+			talloc_free(tmp_ctx);
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
 	}
 
-	talloc_free(tmp_ctx);
-
 	account_sid = samdb_result_dom_sid(server_info, msg, "objectSid");
-	NT_STATUS_HAVE_NO_MEMORY(account_sid);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(account_sid, tmp_ctx);
 
 	primary_group_sid = dom_sid_dup(server_info, account_sid);
-	NT_STATUS_HAVE_NO_MEMORY(primary_group_sid);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(primary_group_sid, tmp_ctx);
 
 	rid = samdb_result_uint(msg, "primaryGroupID", ~0);
 	if (rid == ~0) {
@@ -325,30 +334,30 @@ _PUBLIC_ NTSTATUS authsam_make_server_info(TALLOC_CTX *mem_ctx, struct ldb_conte
 	server_info->account_name = talloc_steal(server_info, samdb_result_string(msg, "sAMAccountName", NULL));
 
 	server_info->domain_name = talloc_strdup(server_info, domain_name);
-	NT_STATUS_HAVE_NO_MEMORY(server_info->domain_name);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info->domain_name, tmp_ctx);
 
 	str = samdb_result_string(msg, "displayName", "");
 	server_info->full_name = talloc_strdup(server_info, str);
-	NT_STATUS_HAVE_NO_MEMORY(server_info->full_name);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info->full_name, tmp_ctx);
 
 	str = samdb_result_string(msg, "scriptPath", "");
 	server_info->logon_script = talloc_strdup(server_info, str);
-	NT_STATUS_HAVE_NO_MEMORY(server_info->logon_script);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info->logon_script, tmp_ctx);
 
 	str = samdb_result_string(msg, "profilePath", "");
 	server_info->profile_path = talloc_strdup(server_info, str);
-	NT_STATUS_HAVE_NO_MEMORY(server_info->profile_path);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info->profile_path, tmp_ctx);
 
 	str = samdb_result_string(msg, "homeDirectory", "");
 	server_info->home_directory = talloc_strdup(server_info, str);
-	NT_STATUS_HAVE_NO_MEMORY(server_info->home_directory);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info->home_directory, tmp_ctx);
 
 	str = samdb_result_string(msg, "homeDrive", "");
 	server_info->home_drive = talloc_strdup(server_info, str);
-	NT_STATUS_HAVE_NO_MEMORY(server_info->home_drive);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info->home_drive, tmp_ctx);
 
 	server_info->logon_server = talloc_strdup(server_info, netbios_name);
-	NT_STATUS_HAVE_NO_MEMORY(server_info->logon_server);
+	NT_STATUS_HAVE_NO_MEMORY_AND_FREE(server_info->logon_server, tmp_ctx);
 
 	server_info->last_logon = samdb_result_nttime(msg, "lastLogon", 0);
 	server_info->last_logoff = samdb_result_nttime(msg, "lastLogoff", 0);
@@ -373,7 +382,7 @@ _PUBLIC_ NTSTATUS authsam_make_server_info(TALLOC_CTX *mem_ctx, struct ldb_conte
 
 	server_info->authenticated = true;
 
-	*_server_info = server_info;
+	*_server_info = talloc_steal(mem_ctx, server_info);
 
 	return NT_STATUS_OK;
 }
@@ -381,7 +390,7 @@ _PUBLIC_ NTSTATUS authsam_make_server_info(TALLOC_CTX *mem_ctx, struct ldb_conte
 NTSTATUS sam_get_results_principal(struct ldb_context *sam_ctx,
 				   TALLOC_CTX *mem_ctx, const char *principal,
 				   struct ldb_dn **domain_dn,
-				   struct ldb_message ***msgs)
+				   struct ldb_message **msg)
 {			   
 	struct ldb_dn *user_dn;
 	NTSTATUS nt_status;
@@ -399,12 +408,13 @@ NTSTATUS sam_get_results_principal(struct ldb_context *sam_ctx,
 	}
 	
 	/* pull the user attributes */
-	ret = gendb_search_dn(sam_ctx, tmp_ctx, user_dn, msgs, user_attrs);
-	if (ret != 1) {
+	ret = gendb_search_single_extended_dn(sam_ctx, tmp_ctx, user_dn, LDB_SCOPE_BASE,
+					      msg, user_attrs, "(objectClass=*)");
+	if (ret != LDB_SUCCESS) {
 		talloc_free(tmp_ctx);
 		return NT_STATUS_INTERNAL_DB_CORRUPTION;
 	}
-	talloc_steal(mem_ctx, *msgs);
+	talloc_steal(mem_ctx, *msg);
 	talloc_steal(mem_ctx, *domain_dn);
 	talloc_free(tmp_ctx);
 	
diff --git a/source4/kdc/hdb-samba4.c b/source4/kdc/hdb-samba4.c
index 585285795f..28a82bcf61 100644
--- a/source4/kdc/hdb-samba4.c
+++ b/source4/kdc/hdb-samba4.c
@@ -921,18 +921,15 @@ static krb5_error_code LDB_lookup_principal(krb5_context context, struct ldb_con
 					    krb5_const_principal principal,
 					    enum hdb_ldb_ent_type ent_type,
 					    struct ldb_dn *realm_dn,
-					    struct ldb_message ***pmsg)
+					    struct ldb_message **pmsg)
 {
 	krb5_error_code ret;
 	int lret;
 	char *filter = NULL;
 	const char * const *princ_attrs = user_attrs;
-
 	char *short_princ;
 	char *short_princ_talloc;
 
-	struct ldb_result *res = NULL;
-
 	ret = krb5_unparse_name_flags(context, principal,  KRB5_PRINCIPAL_UNPARSE_NO_REALM, &short_princ);
 
 	if (ret != 0) {
@@ -969,19 +966,18 @@ static krb5_error_code LDB_lookup_principal(krb5_context context, struct ldb_con
 		return ENOMEM;
 	}
 
-	lret = ldb_search(ldb_ctx, mem_ctx, &res, realm_dn,
-			  LDB_SCOPE_SUBTREE, princ_attrs, "%s", filter);
-	if (lret != LDB_SUCCESS) {
-		DEBUG(3, ("Failed to search for %s: %s\n", filter, ldb_errstring(ldb_ctx)));
+	lret = gendb_search_single_extended_dn(ldb_ctx, mem_ctx, 
+					       realm_dn, LDB_SCOPE_SUBTREE,
+					       pmsg, princ_attrs, "%s", filter);
+	if (lret == LDB_ERR_NO_SUCH_OBJECT) {
+		DEBUG(3, ("Failed find a entry for %s\n", filter));
 		return HDB_ERR_NOENTRY;
-	} else if (res->count == 0 || res->count > 1) {
-		DEBUG(3, ("Failed find a single entry for %s: got %d\n", filter, res->count));
-		talloc_free(res);
+	}
+	if (lret != LDB_SUCCESS) {
+		DEBUG(3, ("Failed single search for for %s - %s\n", 
+			  filter, ldb_errstring(ldb_ctx)));
 		return HDB_ERR_NOENTRY;
 	}
-	talloc_steal(mem_ctx, res->msgs);
-	*pmsg = res->msgs;
-	talloc_free(res);
 	return 0;
 }
 
@@ -989,7 +985,7 @@ static krb5_error_code LDB_lookup_trust(krb5_context context, struct ldb_context
 					TALLOC_CTX *mem_ctx,
 					const char *realm,
 					struct ldb_dn *realm_dn,
-					struct ldb_message ***pmsg)
+					struct ldb_message **pmsg)
 {
 	int lret;
 	char *filter = NULL;
@@ -1015,7 +1011,7 @@ static krb5_error_code LDB_lookup_trust(krb5_context context, struct ldb_context
 		return HDB_ERR_NOENTRY;
 	}
 	talloc_steal(mem_ctx, res->msgs);
-	*pmsg = res->msgs;
+	*pmsg = res->msgs[0];
 	talloc_free(res);
 	return 0;
 }
@@ -1060,7 +1056,7 @@ static krb5_error_code LDB_fetch_client(krb5_context context, HDB *db,
 	char *principal_string;
 	struct ldb_dn *realm_dn;
 	krb5_error_code ret;
-	struct ldb_message **msg = NULL;
+	struct ldb_message *msg = NULL;
 
 	ret = krb5_unparse_name(context, principal, &principal_string);
 	
@@ -1082,7 +1078,7 @@ static krb5_error_code LDB_fetch_client(krb5_context context, HDB *db,
 	
 	ret = LDB_message2entry(context, db, mem_ctx, 
 				principal, HDB_SAMBA4_ENT_TYPE_CLIENT,
-				realm_dn, msg[0], entry_ex);
+				realm_dn, msg, entry_ex);
 	return ret;
 }
 
@@ -1093,7 +1089,7 @@ static krb5_error_code LDB_fetch_krbtgt(krb5_context context, HDB *db,
 					hdb_entry_ex *entry_ex)
 {
 	krb5_error_code ret;
-	struct ldb_message **msg = NULL;
+	struct ldb_message *msg = NULL;
 	struct ldb_dn *realm_dn = ldb_get_default_basedn(db->hdb_db);
 	const char *realm;
 	struct loadparm_context *lp_ctx = talloc_get_type(ldb_get_opaque(db->hdb_db, "loadparm"), struct loadparm_context);
@@ -1146,7 +1142,7 @@ static krb5_error_code LDB_fetch_krbtgt(krb5_context context, HDB *db,
 		
 		ret = LDB_message2entry(context, db, mem_ctx, 
 					principal, HDB_SAMBA4_ENT_TYPE_KRBTGT, 
-					realm_dn, msg[0], entry_ex);
+					realm_dn, msg, entry_ex);
 		if (ret != 0) {
 			krb5_warnx(context, "LDB_fetch: self krbtgt message2entry failed");	
 		}
@@ -1183,7 +1179,7 @@ static krb5_error_code LDB_fetch_krbtgt(krb5_context context, HDB *db,
 		
 		ret = LDB_trust_message2entry(context, db, lp_ctx, mem_ctx, 
 					      principal, direction, 
-					      realm_dn, msg[0], entry_ex);
+					      realm_dn, msg, entry_ex);
 		if (ret != 0) {
 			krb5_warnx(context, "LDB_fetch: trust_message2entry failed");	
 		}
@@ -1204,7 +1200,7 @@ static krb5_error_code LDB_fetch_server(krb5_context context, HDB *db,
 {
 	krb5_error_code ret;
 	const char *realm;
-	struct ldb_message **msg = NULL;
+	struct ldb_message *msg = NULL;
 	struct ldb_dn *realm_dn;
 	if (principal->name.name_string.len >= 2) {
 		/* 'normal server' case */
@@ -1232,10 +1228,12 @@ static krb5_error_code LDB_fetch_server(krb5_context context, HDB *db,
 			return HDB_ERR_NOENTRY;
 		}
 		
-		ldb_ret = gendb_search_dn((struct ldb_context *)db->hdb_db,
-					  mem_ctx, user_dn, &msg, user_attrs);
-		
-		if (ldb_ret != 1) {
+		ldb_ret = gendb_search_single_extended_dn((struct ldb_context *)db->hdb_db,
+							  mem_ctx, 
+							  user_dn, LDB_SCOPE_BASE,
+							  &msg, user_attrs,
+							  "(objectClass=*)");
+		if (ldb_ret != LDB_SUCCESS) {
 			return HDB_ERR_NOENTRY;
 		}
 		
@@ -1257,7 +1255,7 @@ static krb5_error_code LDB_fetch_server(krb5_context context, HDB *db,
 
 	ret = LDB_message2entry(context, db, mem_ctx, 
 				principal, HDB_SAMBA4_ENT_TYPE_SERVER,
-				realm_dn, msg[0], entry_ex);
+				realm_dn, msg, entry_ex);
 	if (ret != 0) {
 		krb5_warnx(context, "LDB_fetch: message2entry failed");	
 	}