s4-ldbwrap: added re-use of ldb contexts in ldb_wrap_connect()

This allows us to reuse a ldb context if it is open twice, instead
of going through the expensive process of a full ldb open. We can
reuse it if all of the parameters are the same.

The change relies on callers using talloc_unlink() or free of a parent
to close a ldb context.

20 files changed, 119 insertions, 50 deletions

diff --git a/source4/auth/gensec/schannel.c b/source4/auth/gensec/schannel.c
index 2fe97fede4..58cbb6a593 100644
--- a/source4/auth/gensec/schannel.c
+++ b/source4/auth/gensec/schannel.c
@@ -146,7 +146,7 @@ static NTSTATUS schannel_update(struct gensec_security *gensec_security, TALLOC_
 		/* pull the session key for this client */
 		status = schannel_fetch_session_key_ldb(schannel_ldb,
 							out_mem_ctx, workstation, &creds);
-		talloc_free(schannel_ldb);
+		talloc_unlink(out_mem_ctx, schannel_ldb);
 		if (!NT_STATUS_IS_OK(status)) {
 			DEBUG(3, ("Could not find session key for attempted schannel connection from %s: %s\n",
 				  workstation, nt_errstr(status)));
diff --git a/source4/dsdb/samdb/samdb_privilege.c b/source4/dsdb/samdb/samdb_privilege.c
index e7742c681c..dcb96d8c0e 100644
--- a/source4/dsdb/samdb/samdb_privilege.c
+++ b/source4/dsdb/samdb/samdb_privilege.c
@@ -40,7 +40,7 @@ struct ldb_context *privilege_connect(TALLOC_CTX *mem_ctx,
 	if (!path) return NULL;
 
 	pdb = ldb_wrap_connect(mem_ctx, ev_ctx, lp_ctx, path, 
-			       NULL, NULL, 0, NULL);
+			       NULL, NULL, 0);
 	talloc_free(path);
 
 	return pdb;
diff --git a/source4/ldap_server/ldap_server.c b/source4/ldap_server/ldap_server.c
index 0255c72086..c45a4168f9 100644
--- a/source4/ldap_server/ldap_server.c
+++ b/source4/ldap_server/ldap_server.c
@@ -548,7 +548,7 @@ static NTSTATUS add_socket(struct tevent_context *event_context,
 
 	/* And once we are bound, free the tempoary ldb, it will
 	 * connect again on each incoming LDAP connection */
-	talloc_free(ldb);
+	talloc_unlink(ldap_service, ldb);
 
 	return status;
 }
diff --git a/source4/lib/ldb_wrap.c b/source4/lib/ldb_wrap.c
index 74502afde2..0427a9c378 100644
--- a/source4/lib/ldb_wrap.c
+++ b/source4/lib/ldb_wrap.c
@@ -3,7 +3,7 @@
 
    LDB wrap functions
 
-   Copyright (C) Andrew Tridgell 2004
+   Copyright (C) Andrew Tridgell 2004-2009
    
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -34,6 +34,7 @@
 #include "ldb_wrap.h"
 #include "dsdb/samdb/samdb.h"
 #include "param/param.h"
+#include "dlinklist.h"
 
 /*
   this is used to catch debug messages from ldb
@@ -67,26 +68,53 @@ static void ldb_wrap_debug(void *context, enum ldb_debug_level level,
 	free(s);
 }
 
-/* check for memory leaks on the ldb context */
-static int ldb_wrap_destructor(struct ldb_context *ldb)
+
+/*
+  connecting to a ldb can be a relatively expensive operation because
+  of the schema and partition loads. We keep a list of open ldb
+  contexts here, and try to re-use when possible. 
+
+  This means callers of ldb_wrap_connect() must use talloc_unlink() or
+  the free of a parent to destroy the context
+ */
+static struct ldb_wrap {
+	struct ldb_wrap *next, *prev;
+	struct ldb_wrap_context {
+		/* the context is what we use to tell if two ldb
+		 * connections are exactly equivalent 
+		 */		 
+		const char *url;
+		struct tevent_context *ev;
+		struct loadparm_context *lp_ctx;
+		struct auth_session_info *session_info;
+		struct cli_credentials *credentials;
+		unsigned int flags;
+	} context;
+	struct ldb_context *ldb;
+} *ldb_wrap_list;
+
+/*
+  see if two database opens are equivalent
+ */
+static bool ldb_wrap_same_context(const struct ldb_wrap_context *c1,
+				  const struct ldb_wrap_context *c2)
 {
-	size_t *startup_blocks = (size_t *)ldb_get_opaque(ldb, "startup_blocks");
-
-	if (startup_blocks &&
-	    talloc_total_blocks(ldb) > *startup_blocks + 400) {
-		DEBUG(0,("WARNING: probable memory leak in ldb %s - %lu blocks (startup %lu) %lu bytes\n",
-			 (char *)ldb_get_opaque(ldb, "wrap_url"), 
-			 (unsigned long)talloc_total_blocks(ldb), 
-			 (unsigned long)*startup_blocks,
-			 (unsigned long)talloc_total_size(ldb)));
-#if 0
-		talloc_report_full(ldb, stdout);
-		call_backtrace();
-		smb_panic("probable memory leak in ldb");
-#endif
-	}
+	return (c1->ev == c2->ev &&
+		c1->lp_ctx == c2->lp_ctx &&
+		c1->session_info == c2->session_info &&
+		c1->credentials == c2->credentials &&
+		c1->flags == c2->flags &&
+		(c1->url == c2->url || strcmp(c1->url, c2->url) == 0));
+}
+
+/* 
+   free a ldb_wrap structure
+ */
+static int ldb_wrap_destructor(struct ldb_wrap *w)
+{
+	DLIST_REMOVE(ldb_wrap_list, w);
 	return 0;
-}				 
+}
 
 /*
   wrapped connection to a ldb database
@@ -100,13 +128,27 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
 				     const char *url,
 				     struct auth_session_info *session_info,
 				     struct cli_credentials *credentials,
-				     unsigned int flags,
-				     const char *options[])
+				     unsigned int flags)
 {
 	struct ldb_context *ldb;
 	int ret;
 	char *real_url = NULL;
-	size_t *startup_blocks;
+	struct ldb_wrap *w;
+	struct ldb_wrap_context c;
+
+	c.url          = url;
+	c.ev           = ev;
+	c.lp_ctx       = lp_ctx;
+	c.session_info = session_info;
+	c.credentials  = credentials;
+	c.flags        = flags;
+
+	/* see if we can re-use an existing ldb */
+	for (w=ldb_wrap_list; w; w=w->next) {
+		if (ldb_wrap_same_context(&c, &w->context)) {
+			return talloc_reference(mem_ctx, w->ldb);
+		}
+	}
 
 	/* we want to use the existing event context if possible. This
 	   relies on the fact that in smbd, everything is a child of
@@ -178,7 +220,7 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
 	   ldb_wrap_connect() */
 	ldb_set_create_perms(ldb, 0600);
 	
-	ret = ldb_connect(ldb, real_url, flags, options);
+	ret = ldb_connect(ldb, real_url, flags, NULL);
 	if (ret != LDB_SUCCESS) {
 		talloc_free(ldb);
 		return NULL;
@@ -186,11 +228,38 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
 
 	/* setup for leak detection */
 	ldb_set_opaque(ldb, "wrap_url", real_url);
-	startup_blocks = talloc(ldb, size_t);
-	*startup_blocks = talloc_total_blocks(ldb);
-	ldb_set_opaque(ldb, "startup_blocks", startup_blocks);
 	
-	talloc_set_destructor(ldb, ldb_wrap_destructor);
+	/* add to the list of open ldb contexts */
+	w = talloc(ldb, struct ldb_wrap);
+	if (w == NULL) {
+		talloc_free(ldb);
+		return NULL;
+	}
+
+	w->context = c;
+	w->context.url = talloc_strdup(w, url);
+	if (w->context.url == NULL) {
+		talloc_free(ldb);
+		return NULL;
+	}
+
+	w->ldb = ldb;
+
+	DLIST_ADD(ldb_wrap_list, w);
+
+	DEBUG(3,("ldb_wrap open of %s\n", url));
+
+	talloc_set_destructor(w, ldb_wrap_destructor);
 
 	return ldb;
 }
+
+/*
+  when we fork() we need to make sure that any open ldb contexts have
+  any open transactions cancelled
+ */
+void ldb_wrap_fork_hook(void)
+{
+	
+}
+
diff --git a/source4/lib/ldb_wrap.h b/source4/lib/ldb_wrap.h
index f44ff8c599..650f97d17d 100644
--- a/source4/lib/ldb_wrap.h
+++ b/source4/lib/ldb_wrap.h
@@ -37,7 +37,7 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
 				     const char *url,
 				     struct auth_session_info *session_info,
 				     struct cli_credentials *credentials,
-				     unsigned int flags,
-				     const char *options[]);
+				     unsigned int flags);
 
+void ldb_wrap_fork_hook(void);
 #endif /* _LDB_WRAP_H_ */
diff --git a/source4/lib/registry/ldb.c b/source4/lib/registry/ldb.c
index c558805e04..033fdcb780 100644
--- a/source4/lib/registry/ldb.c
+++ b/source4/lib/registry/ldb.c
@@ -441,7 +441,7 @@ WERROR reg_open_ldb_file(TALLOC_CTX *parent_ctx, const char *location,
 		return WERR_INVALID_PARAM;
 
 	wrap = ldb_wrap_connect(parent_ctx, ev_ctx, lp_ctx,
-				location, session_info, credentials, 0, NULL);
+				location, session_info, credentials, 0);
 
 	if (wrap == NULL) {
 		DEBUG(1, (__FILE__": unable to connect\n"));
diff --git a/source4/libgpo/ads_convenience.c b/source4/libgpo/ads_convenience.c
index 6e2dbca538..09ceecc752 100644
--- a/source4/libgpo/ads_convenience.c
+++ b/source4/libgpo/ads_convenience.c
@@ -56,7 +56,7 @@ static ADS_STATUS ads_connect(ADS_STRUCT *ads)
 
 	url = talloc_asprintf(ads, "ldap://%s", io->out.dcs[0].name);
 	ads->ldbctx = ldb_wrap_connect(ads, ads->netctx->event_ctx, ads->netctx->lp_ctx,
-	                 url, NULL, ads->netctx->cred, 0, NULL);
+	                 url, NULL, ads->netctx->cred, 0);
 	if (ads->ldbctx == NULL) {
 		return ADS_ERROR_NT(NT_STATUS_UNSUCCESSFUL);
 	}
diff --git a/source4/libnet/libnet_become_dc.c b/source4/libnet/libnet_become_dc.c
index 3fb308dcd7..6668be83ac 100644
--- a/source4/libnet/libnet_become_dc.c
+++ b/source4/libnet/libnet_become_dc.c
@@ -814,7 +814,7 @@ static NTSTATUS becomeDC_ldap_connect(struct libnet_BecomeDC_state *s,
 	ldap->ldb = ldb_wrap_connect(s, s->libnet->event_ctx, s->libnet->lp_ctx, url,
 				     NULL,
 				     s->libnet->cred,
-				     0, NULL);
+				     0);
 	talloc_free(url);
 	if (ldap->ldb == NULL) {
 		return NT_STATUS_UNEXPECTED_NETWORK_ERROR;
diff --git a/source4/libnet/libnet_join.c b/source4/libnet/libnet_join.c
index fc7de10506..e60d45e316 100644
--- a/source4/libnet/libnet_join.c
+++ b/source4/libnet/libnet_join.c
@@ -247,7 +247,7 @@ static NTSTATUS libnet_JoinADSDomain(struct libnet_context *ctx, struct libnet_J
 
 	remote_ldb = ldb_wrap_connect(tmp_ctx, ctx->event_ctx, ctx->lp_ctx, 
 				      remote_ldb_url, 
-				      NULL, ctx->cred, 0, NULL);
+				      NULL, ctx->cred, 0);
 	if (!remote_ldb) {
 		r->out.error_string = NULL;
 		talloc_free(tmp_ctx);
diff --git a/source4/libnet/libnet_samsync_ldb.c b/source4/libnet/libnet_samsync_ldb.c
index 164e806906..4cd23cd8ed 100644
--- a/source4/libnet/libnet_samsync_ldb.c
+++ b/source4/libnet/libnet_samsync_ldb.c
@@ -1191,7 +1191,7 @@ static NTSTATUS libnet_samsync_ldb_init(TALLOC_CTX *mem_ctx,
 						     state->samsync_state->machine_net_ctx->lp_ctx, 
 						     ldap_url, 
 						     NULL, state->samsync_state->machine_net_ctx->cred,
-						     0, NULL);
+						     0);
 		if (!state->remote_ldb) {
 			*error_string = talloc_asprintf(mem_ctx, "Failed to connect to remote LDAP server at %s (used to extract additional data in SamSync replication)", ldap_url);
 			return NT_STATUS_NO_LOGON_SERVERS;
diff --git a/source4/libnet/libnet_unbecome_dc.c b/source4/libnet/libnet_unbecome_dc.c
index 3bd7a4e287..6e1ecf072d 100644
--- a/source4/libnet/libnet_unbecome_dc.c
+++ b/source4/libnet/libnet_unbecome_dc.c
@@ -316,7 +316,7 @@ static NTSTATUS unbecomeDC_ldap_connect(struct libnet_UnbecomeDC_state *s)
 	s->ldap.ldb = ldb_wrap_connect(s, s->libnet->event_ctx, s->libnet->lp_ctx, url,
 				       NULL,
 				       s->libnet->cred,
-				       0, NULL);
+				       0);
 	talloc_free(url);
 	if (s->ldap.ldb == NULL) {
 		return NT_STATUS_UNEXPECTED_NETWORK_ERROR;
diff --git a/source4/nbt_server/wins/winsdb.c b/source4/nbt_server/wins/winsdb.c
index a502050299..56ba146be1 100644
--- a/source4/nbt_server/wins/winsdb.c
+++ b/source4/nbt_server/wins/winsdb.c
@@ -980,7 +980,7 @@ static bool winsdb_check_or_add_module_list(struct tevent_context *ev_ctx,
 	}
 
 	h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lock_path(h, lp_ctx, lp_wins_url(lp_ctx)),
-				  NULL, NULL, flags, NULL);
+				  NULL, NULL, flags);
 	if (!h->ldb) goto failed;
 
 	talloc_free(tmp_ctx);
@@ -1016,7 +1016,7 @@ struct winsdb_handle *winsdb_connect(TALLOC_CTX *mem_ctx,
 	}
 
 	h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lock_path(h, lp_ctx, lp_wins_url(lp_ctx)),
-				  NULL, NULL, flags, NULL);
+				  NULL, NULL, flags);
 	if (!h->ldb) goto failed;	
 
 	h->caller = caller;
diff --git a/source4/rpc_server/netlogon/dcerpc_netlogon.c b/source4/rpc_server/netlogon/dcerpc_netlogon.c
index c3cdfb1581..d59cb6fb82 100644
--- a/source4/rpc_server/netlogon/dcerpc_netlogon.c
+++ b/source4/rpc_server/netlogon/dcerpc_netlogon.c
@@ -256,7 +256,7 @@ static NTSTATUS dcesrv_netr_ServerAuthenticate3(struct dcesrv_call_state *dce_ca
 	}
 
 	nt_status = schannel_store_session_key_ldb(schannel_ldb, mem_ctx, creds);
-	talloc_free(schannel_ldb);
+	talloc_unlink(mem_ctx, schannel_ldb);
 
 	return nt_status;
 }
@@ -343,7 +343,7 @@ static NTSTATUS dcesrv_netr_creds_server_step_check(struct dcesrv_call_state *dc
 							 schannel_in_use,
 							 received_authenticator,
 							 return_authenticator, creds_out);
-	talloc_free(ldb);
+	talloc_unlink(mem_ctx, ldb);
 	return nt_status;
 }
 
diff --git a/source4/rpc_server/samr/dcesrv_samr.c b/source4/rpc_server/samr/dcesrv_samr.c
index f4ada2c7c7..08f1471665 100644
--- a/source4/rpc_server/samr/dcesrv_samr.c
+++ b/source4/rpc_server/samr/dcesrv_samr.c
@@ -4205,7 +4205,7 @@ static NTSTATUS dcesrv_samr_GetDomPwInfo(struct dcesrv_call_state *dce_call, TAL
 		"pwdProperties", 1);
 
 	talloc_free(msgs);
-	talloc_free(sam_ctx);
+	talloc_unlink(mem_ctx, sam_ctx);
 
 	return NT_STATUS_OK;
 }
diff --git a/source4/torture/ldap/ldap_sort.c b/source4/torture/ldap/ldap_sort.c
index 4d732cf997..03fcfd8482 100644
--- a/source4/torture/ldap/ldap_sort.c
+++ b/source4/torture/ldap/ldap_sort.c
@@ -62,7 +62,7 @@ bool torture_ldap_sort(struct torture_context *torture)
 	ldb = ldb_wrap_connect(torture, torture->ev, torture->lp_ctx, url,
 						 NULL,
 						 cmdline_credentials,
-						 0, NULL);
+						 0);
 	torture_assert(torture, ldb, "Failed to make LDB connection to target");
 
 	ctx = talloc_zero(ldb, struct ldb_result);
diff --git a/source4/torture/ldap/schema.c b/source4/torture/ldap/schema.c
index e746a32f2b..7fe171691a 100644
--- a/source4/torture/ldap/schema.c
+++ b/source4/torture/ldap/schema.c
@@ -387,7 +387,7 @@ bool torture_ldap_schema(struct torture_context *torture)
 	ldb = ldb_wrap_connect(torture, torture->ev, torture->lp_ctx, url,
 			       NULL,
 			       cmdline_credentials,
-			       0, NULL);
+			       0);
 	if (!ldb) goto failed;
 
 	ret &= test_search_rootDSE(ldb, &rootDSE);
diff --git a/source4/torture/ldap/uptodatevector.c b/source4/torture/ldap/uptodatevector.c
index c53459a58e..a4ca7e5357 100644
--- a/source4/torture/ldap/uptodatevector.c
+++ b/source4/torture/ldap/uptodatevector.c
@@ -159,7 +159,7 @@ bool torture_ldap_uptodatevector(struct torture_context *torture)
 	ldb = ldb_wrap_connect(torture, torture->ev, torture->lp_ctx, url,
 			       NULL,
 			       cmdline_credentials,
-			       0, NULL);
+			       0);
 	if (!ldb) goto failed;
 
 	ret &= test_check_uptodatevector(torture, ldb, samdb_base_dn(ldb));
diff --git a/source4/torture/local/dbspeed.c b/source4/torture/local/dbspeed.c
index 9547e9ff74..236a52b517 100644
--- a/source4/torture/local/dbspeed.c
+++ b/source4/torture/local/dbspeed.c
@@ -177,7 +177,7 @@ static bool test_ldb_speed(struct torture_context *torture, const void *_data)
 	torture_comment(torture, "Testing ldb speed for sidmap\n");
 
 	ldb = ldb_wrap_connect(tmp_ctx, torture->ev, torture->lp_ctx, "tdb://test.ldb", 
-				NULL, NULL, LDB_FLG_NOSYNC, NULL);
+				NULL, NULL, LDB_FLG_NOSYNC);
 	if (!ldb) {
 		unlink("./test.ldb");
 		talloc_free(tmp_ctx);
diff --git a/source4/torture/rpc/netlogon.c b/source4/torture/rpc/netlogon.c
index 1ef58fd3f7..db949ada2a 100644
--- a/source4/torture/rpc/netlogon.c
+++ b/source4/torture/rpc/netlogon.c
@@ -2330,7 +2330,7 @@ static bool test_GetDomainInfo(struct torture_context *tctx,
 		sam_ctx = ldb_wrap_connect(tctx, tctx->ev, tctx->lp_ctx, url,
 					   NULL,
 					   cmdline_credentials,
-					   0, NULL);
+					   0);
 		
 		torture_assert(tctx, sam_ctx, "Connection to the SAMDB on DC failed!");
 	}
diff --git a/source4/winbind/idmap.c b/source4/winbind/idmap.c
index 2ef6c93970..153b5c363e 100644
--- a/source4/winbind/idmap.c
+++ b/source4/winbind/idmap.c
@@ -167,8 +167,8 @@ struct idmap_context *idmap_init(TALLOC_CTX *mem_ctx,
 
 	idmap_ctx->ldb_ctx = ldb_wrap_connect(mem_ctx, ev_ctx, lp_ctx,
 					      lp_idmap_url(lp_ctx),
-					      system_session(mem_ctx, lp_ctx),
-					      NULL, 0, NULL);
+					      system_session(lp_ctx),
+					      NULL, 0);
 	if (idmap_ctx->ldb_ctx == NULL) {
 		return NULL;
 	}