r19430: merge recent ldb changes from Samba4. This includes memory leak fixes

and significant speedups
(This used to be commit bb5c205fef90aa8b89ba400fb9f2f37a111676a8)

26 files changed, 170 insertions, 108 deletions

diff --git a/source3/lib/ldb/common/ldb.c b/source3/lib/ldb/common/ldb.c
index 28d1c7235a..7648abf795 100644
--- a/source3/lib/ldb/common/ldb.c
+++ b/source3/lib/ldb/common/ldb.c
@@ -208,7 +208,8 @@ int ldb_connect(struct ldb_context *ldb, const char *url, unsigned int flags, co
 	}
 
 	if (ldb_load_modules(ldb, options) != LDB_SUCCESS) {
-		ldb_debug(ldb, LDB_DEBUG_FATAL, "Unable to load modules for '%s'\n", url);
+		ldb_debug(ldb, LDB_DEBUG_FATAL, "Unable to load modules for %s: %s\n",
+			  url, ldb_errstring(ldb));
 		return LDB_ERR_OTHER;
 	}
 
@@ -536,8 +537,9 @@ static int ldb_search_callback(struct ldb_context *ldb, void *context, struct ld
 	if (!res || !ares) {
 		goto error;
 	}
-
-	if (ares->type == LDB_REPLY_ENTRY) {
+	
+	switch (ares->type) {
+	case LDB_REPLY_ENTRY:
 		res->msgs = talloc_realloc(res, res->msgs, struct ldb_message *, res->count + 2);
 		if (! res->msgs) {
 			goto error;
@@ -547,9 +549,8 @@ static int ldb_search_callback(struct ldb_context *ldb, void *context, struct ld
 
 		res->msgs[res->count] = talloc_move(res->msgs, &ares->message);
 		res->count++;
-	}
-
-	if (ares->type == LDB_REPLY_REFERRAL) {
+		break;
+	case LDB_REPLY_REFERRAL:
 		if (res->refs) {
 			for (n = 0; res->refs[n]; n++) /*noop*/ ;
 		} else {
@@ -563,8 +564,11 @@ static int ldb_search_callback(struct ldb_context *ldb, void *context, struct ld
 
 		res->refs[n] = talloc_move(res->refs, &ares->referral);
 		res->refs[n + 1] = NULL;
+	case LDB_REPLY_DONE:
+		/* Should do something here to detect if this never
+		 * happens */
+		break;		
 	}
-
 	talloc_steal(res, ares->controls);
 	talloc_free(ares);
 	return LDB_SUCCESS;
diff --git a/source3/lib/ldb/common/ldb_attributes.c b/source3/lib/ldb/common/ldb_attributes.c
index c8a7909b4c..2d9f0e6cf8 100644
--- a/source3/lib/ldb/common/ldb_attributes.c
+++ b/source3/lib/ldb/common/ldb_attributes.c
@@ -39,6 +39,7 @@ int ldb_set_attrib_handlers(struct ldb_context *ldb,
 			    const struct ldb_attrib_handler *handlers, 
 			    unsigned num_handlers)
 {
+	int i;
 	struct ldb_attrib_handler *h;
 	h = talloc_realloc(ldb, ldb->schema.attrib_handlers,
 			   struct ldb_attrib_handler,
@@ -50,6 +51,16 @@ int ldb_set_attrib_handlers(struct ldb_context *ldb,
 	ldb->schema.attrib_handlers = h;
 	memcpy(h + ldb->schema.num_attrib_handlers, 
 	       handlers, sizeof(*h) * num_handlers);
+	for (i=0;i<num_handlers;i++) {
+		if (h[ldb->schema.num_attrib_handlers+i].flags & LDB_ATTR_FLAG_ALLOCATED) {
+			h[ldb->schema.num_attrib_handlers+i].attr = talloc_strdup(ldb->schema.attrib_handlers,
+										  h[ldb->schema.num_attrib_handlers+i].attr);
+			if (h[ldb->schema.num_attrib_handlers+i].attr == NULL) {
+				ldb_oom(ldb);
+				return -1;
+			}
+		}
+	}
 	ldb->schema.num_attrib_handlers += num_handlers;
 	return 0;
 }
@@ -129,6 +140,9 @@ void ldb_remove_attrib_handler(struct ldb_context *ldb, const char *attrib)
 	if (h == &ldb_default_attrib_handler) {
 		return;
 	}
+	if (h->flags & LDB_ATTR_FLAG_ALLOCATED) {
+		talloc_free(h->attr);
+	}
 	i = h - ldb->schema.attrib_handlers;
 	if (i < ldb->schema.num_attrib_handlers - 1) {
 		memmove(&ldb->schema.attrib_handlers[i], 
diff --git a/source3/lib/ldb/common/ldb_dn.c b/source3/lib/ldb/common/ldb_dn.c
index d035b0d3c2..e937a2b7fc 100644
--- a/source3/lib/ldb/common/ldb_dn.c
+++ b/source3/lib/ldb/common/ldb_dn.c
@@ -337,6 +337,9 @@ failed:
 	return NULL;
 }
 
+/*
+  explode a DN string into a ldb_dn structure
+*/
 struct ldb_dn *ldb_dn_explode(void *mem_ctx, const char *dn)
 {
 	struct ldb_dn *edn; /* the exploded dn */
@@ -848,7 +851,7 @@ failed:
 
 struct ldb_dn *ldb_dn_string_compose(void *mem_ctx, const struct ldb_dn *base, const char *child_fmt, ...)
 {
-	struct ldb_dn *dn;
+	struct ldb_dn *dn, *dn1;
 	char *child_str;
 	va_list ap;
 	
@@ -860,9 +863,11 @@ struct ldb_dn *ldb_dn_string_compose(void *mem_ctx, const struct ldb_dn *base, c
 
 	if (child_str == NULL) return NULL;
 
-	dn = ldb_dn_compose(mem_ctx, ldb_dn_explode(mem_ctx, child_str), base);
+	dn1 = ldb_dn_explode(mem_ctx, child_str);
+	dn = ldb_dn_compose(mem_ctx, dn1, base);
 
 	talloc_free(child_str);
+	talloc_free(dn1);
 
 	return dn;
 }
diff --git a/source3/lib/ldb/common/ldb_ldif.c b/source3/lib/ldb/common/ldb_ldif.c
index 0c31f25cc7..4992eb01ad 100644
--- a/source3/lib/ldb/common/ldb_ldif.c
+++ b/source3/lib/ldb/common/ldb_ldif.c
@@ -232,6 +232,8 @@ static int fold_string(int (*fprintf_fn)(void *, const char *, ...), void *priva
 	return total;
 }
 
+#undef CHECK_RET
+
 /*
   encode as base64 to a file
 */
@@ -264,6 +266,9 @@ static const struct {
 	{NULL, 0}
 };
 
+/* this macro is used to handle the return checking on fprintf_fn() */
+#define CHECK_RET do { if (ret < 0) { talloc_free(mem_ctx); return ret; } total += ret; } while (0)
+
 /*
   write to ldif, using a caller supplied write method
 */
@@ -272,10 +277,13 @@ int ldb_ldif_write(struct ldb_context *ldb,
 		   void *private_data,
 		   const struct ldb_ldif *ldif)
 {
+	TALLOC_CTX *mem_ctx;
 	unsigned int i, j;
 	int total=0, ret;
 	const struct ldb_message *msg;
 
+	mem_ctx = talloc_named_const(NULL, 0, "ldb_ldif_write");
+
 	msg = ldif->msg;
 
 	ret = fprintf_fn(private_data, "dn: %s\n", ldb_dn_linearize(msg->dn, msg->dn));
@@ -290,6 +298,7 @@ int ldb_ldif_write(struct ldb_context *ldb,
 		if (!ldb_changetypes[i].name) {
 			ldb_debug(ldb, LDB_DEBUG_ERROR, "Error: Invalid ldif changetype %d\n",
 				  ldif->changetype);
+			talloc_free(mem_ctx);
 			return -1;
 		}
 		ret = fprintf_fn(private_data, "changetype: %s\n", ldb_changetypes[i].name);
@@ -320,7 +329,7 @@ int ldb_ldif_write(struct ldb_context *ldb,
 
 		for (j=0;j<msg->elements[i].num_values;j++) {
 			struct ldb_val v;
-			ret = h->ldif_write_fn(ldb, ldb, &msg->elements[i].values[j], &v);
+			ret = h->ldif_write_fn(ldb, mem_ctx, &msg->elements[i].values[j], &v);
 			CHECK_RET;
 			if (ldb_should_b64_encode(&v)) {
 				ret = fprintf_fn(private_data, "%s:: ", 
@@ -541,6 +550,7 @@ struct ldb_ldif *ldb_ldif_read(struct ldb_context *ldb,
 	if (!chunk) {
 		goto failed;
 	}
+	talloc_steal(ldif, chunk);
 
 	msg->private_data = chunk;
 	s = chunk;
diff --git a/source3/lib/ldb/common/ldb_modules.c b/source3/lib/ldb/common/ldb_modules.c
index d627f3b9fa..325cab6dfd 100644
--- a/source3/lib/ldb/common/ldb_modules.c
+++ b/source3/lib/ldb/common/ldb_modules.c
@@ -266,6 +266,7 @@ int ldb_load_modules_list(struct ldb_context *ldb, const char **module_list, str
 		if (current == NULL) {
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
+		talloc_set_name(current, "ldb_module: %s", module_list[i]);
 		
 		current->ldb = ldb;
 		current->ops = ops;
diff --git a/source3/lib/ldb/include/ldb.h b/source3/lib/ldb/include/ldb.h
index d2f3bd53d3..0af734eb13 100644
--- a/source3/lib/ldb/include/ldb.h
+++ b/source3/lib/ldb/include/ldb.h
@@ -357,6 +357,9 @@ struct ldb_attrib_handler {
 */
 #define LDB_ATTR_FLAG_HIDDEN       (1<<0) 
 
+/* the attribute handler name should be freed when released */
+#define LDB_ATTR_FLAG_ALLOCATED    (1<<1) 
+
 /**
    The attribute is constructed from other attributes
 */
diff --git a/source3/lib/ldb/ldb_ildap/ldb_ildap.c b/source3/lib/ldb/ldb_ildap/ldb_ildap.c
index 5b69ac06c9..3843d2b825 100644
--- a/source3/lib/ldb/ldb_ildap/ldb_ildap.c
+++ b/source3/lib/ldb/ldb_ildap/ldb_ildap.c
@@ -776,6 +776,7 @@ static int ildb_connect(struct ldb_context *ldb, const char *url,
 		talloc_free(ildb);
 		return -1;
 	}
+	talloc_set_name_const(*module, "ldb_ildap backend");
 	(*module)->ldb = ldb;
 	(*module)->prev = (*module)->next = NULL;
 	(*module)->private_data = ildb;
diff --git a/source3/lib/ldb/ldb_ldap/ldb_ldap.c b/source3/lib/ldb/ldb_ldap/ldb_ldap.c
index 10563816b9..410ad64b4a 100644
--- a/source3/lib/ldb/ldb_ldap/ldb_ldap.c
+++ b/source3/lib/ldb/ldb_ldap/ldb_ldap.c
@@ -497,9 +497,11 @@ static int lldb_parse_result(struct ldb_handle *handle, LDAPMessage *result)
 	char **referralsp = NULL;
 	LDAPControl **serverctrlsp = NULL;
 	int ret = LDB_SUCCESS;
-
+	
 	type = ldap_msgtype(result);
 
+	handle->status = 0;
+
 	switch (type) {
 
 	case LDAP_RES_SEARCH_ENTRY:
@@ -631,15 +633,19 @@ static int lldb_parse_result(struct ldb_handle *handle, LDAPMessage *result)
 	}
 
 	if (matcheddnp) ldap_memfree(matcheddnp);
-	if (errmsgp) {
+	if (errmsgp && *errmsgp) {
 		ldb_set_errstring(ac->module->ldb, errmsgp);
+	} else if (handle->status) {
+		ldb_set_errstring(ac->module->ldb, ldap_err2string(handle->status));
+	}
+	if (errmsgp) {
 		ldap_memfree(errmsgp);
 	}
 	if (referralsp) ldap_value_free(referralsp);
 	if (serverctrlsp) ldap_controls_free(serverctrlsp);
 
 	ldap_msgfree(result);
-	return ret;
+	return lldb_ldap_to_ldb(handle->status);
 
 error:
 	handle->state = LDB_ASYNC_DONE;
@@ -816,6 +822,7 @@ static int lldb_connect(struct ldb_context *ldb,
 		talloc_free(lldb);
 		return -1;
 	}
+	talloc_set_name_const(*module, "ldb_ldap backend");
 	(*module)->ldb = ldb;
 	(*module)->prev = (*module)->next = NULL;
 	(*module)->private_data = lldb;
diff --git a/source3/lib/ldb/ldb_sqlite3/ldb_sqlite3.c b/source3/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
index 91256222b1..8dd25db1d6 100644
--- a/source3/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
+++ b/source3/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
@@ -2101,6 +2101,7 @@ static int lsqlite3_connect(struct ldb_context *ldb,
 		ldb_oom(ldb);
 		goto failed;
 	}
+	talloc_set_name_const(*module, "ldb_sqlite3 backend");
 	(*module)->ldb = ldb;
 	(*module)->prev = (*module)->next = NULL;
 	(*module)->private_data = lsqlite3;
diff --git a/source3/lib/ldb/ldb_tdb/ldb_cache.c b/source3/lib/ldb/ldb_tdb/ldb_cache.c
index a6092c45f9..467f1ac34d 100644
--- a/source3/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/source3/lib/ldb/ldb_tdb/ldb_cache.c
@@ -71,13 +71,7 @@ static void ltdb_attributes_unload(struct ldb_module *module)
 
 	msg = ltdb->cache->attributes;
 	for (i=0;i<msg->num_elements;i++) {
-		const struct ldb_attrib_handler *h;
-		/* this is rather ugly - a consequence of const handling */
-		h = ldb_attrib_handler(module->ldb, msg->elements[i].name);
 		ldb_remove_attrib_handler(module->ldb, msg->elements[i].name);
-		if (strcmp(h->attr, msg->elements[i].name) == 0) {
-			talloc_steal(msg, h->attr);
-		}
 	}
 
 	talloc_free(ltdb->cache->attributes);
@@ -163,7 +157,8 @@ static int ltdb_attributes_load(struct ldb_module *module)
 			goto failed;
 		}
 		h2 = *h;
-		h2.attr = talloc_strdup(module, msg->elements[i].name);
+		h2.attr = msg->elements[i].name;
+		h2.flags |= LDB_ATTR_FLAG_ALLOCATED;
 		if (ldb_set_attrib_handlers(module->ldb, &h2, 1) != 0) {
 			goto failed;
 		}
@@ -319,6 +314,13 @@ int ltdb_cache_load(struct ldb_module *module)
 	struct ldb_dn *baseinfo_dn = NULL;
 	struct ldb_dn *indexlist_dn = NULL;
 	uint64_t seq;
+	struct ldb_message *baseinfo;
+
+	/* a very fast check to avoid extra database reads */
+	if (ltdb->cache != NULL && 
+	    tdb_get_seqnum(ltdb->tdb) == ltdb->tdb_seqnum) {
+		return 0;
+	}
 
 	if (ltdb->cache == NULL) {
 		ltdb->cache = talloc_zero(ltdb, struct ltdb_cache);
@@ -333,30 +335,31 @@ int ltdb_cache_load(struct ldb_module *module)
 		}
 	}
 
-	talloc_free(ltdb->cache->baseinfo);
-	ltdb->cache->baseinfo = talloc(ltdb->cache, struct ldb_message);
-	if (ltdb->cache->baseinfo == NULL) goto failed;
+	baseinfo = talloc(ltdb->cache, struct ldb_message);
+	if (baseinfo == NULL) goto failed;
 
 	baseinfo_dn = ldb_dn_explode(module->ldb, LTDB_BASEINFO);
 	if (baseinfo_dn == NULL) goto failed;
 
-	if (ltdb_search_dn1(module, baseinfo_dn, ltdb->cache->baseinfo) == -1) {
+	if (ltdb_search_dn1(module, baseinfo_dn, baseinfo) == -1) {
 		goto failed;
 	}
 	
 	/* possibly initialise the baseinfo */
-	if (!ltdb->cache->baseinfo->dn) {
+	if (!baseinfo->dn) {
 		if (ltdb_baseinfo_init(module) != 0) {
 			goto failed;
 		}
-		if (ltdb_search_dn1(module, baseinfo_dn, ltdb->cache->baseinfo) != 1) {
+		if (ltdb_search_dn1(module, baseinfo_dn, baseinfo) != 1) {
 			goto failed;
 		}
 	}
 
+	ltdb->tdb_seqnum = tdb_get_seqnum(ltdb->tdb);
+
 	/* if the current internal sequence number is the same as the one
 	   in the database then assume the rest of the cache is OK */
-	seq = ldb_msg_find_attr_as_uint64(ltdb->cache->baseinfo, LTDB_SEQUENCE_NUMBER, 0);
+	seq = ldb_msg_find_attr_as_uint64(baseinfo, LTDB_SEQUENCE_NUMBER, 0);
 	if (seq == ltdb->sequence_number) {
 		goto done;
 	}
@@ -395,11 +398,13 @@ int ltdb_cache_load(struct ldb_module *module)
 	}
 
 done:
+	talloc_free(baseinfo);
 	talloc_free(baseinfo_dn);
 	talloc_free(indexlist_dn);
 	return 0;
 
 failed:
+	talloc_free(baseinfo);
 	talloc_free(baseinfo_dn);
 	talloc_free(indexlist_dn);
 	return -1;
diff --git a/source3/lib/ldb/ldb_tdb/ldb_index.c b/source3/lib/ldb/ldb_tdb/ldb_index.c
index 8a9a82a98c..0c9d1f33a1 100644
--- a/source3/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source3/lib/ldb/ldb_tdb/ldb_index.c
@@ -1030,6 +1030,12 @@ int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg)
 	char *dn;
 	unsigned int i, j;
 
+	/* find the list of indexed fields */	
+	if (ltdb->cache->indexlist->num_elements == 0) {
+		/* no indexed fields */
+		return 0;
+	}
+
 	if (ldb_dn_is_special(msg->dn)) {
 		return 0;
 	}
@@ -1039,12 +1045,6 @@ int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg)
 		return -1;
 	}
 
-	/* find the list of indexed fields */	
-	if (ltdb->cache->indexlist->num_elements == 0) {
-		/* no indexed fields */
-		return 0;
-	}
-
 	for (i = 0; i < msg->num_elements; i++) {
 		ret = ldb_msg_find_idx(ltdb->cache->indexlist, msg->elements[i].name, 
 				       NULL, LTDB_IDXATTR);
diff --git a/source3/lib/ldb/ldb_tdb/ldb_search.c b/source3/lib/ldb/ldb_tdb/ldb_search.c
index 7cdb2b672f..4b76c437b6 100644
--- a/source3/lib/ldb/ldb_tdb/ldb_search.c
+++ b/source3/lib/ldb/ldb_tdb/ldb_search.c
@@ -248,25 +248,13 @@ int ltdb_search_dn1(struct ldb_module *module, const struct ldb_dn *dn, struct l
 	return 1;
 }
 
-/* the lock key for search locking. Note that this is not a DN, its
-   just an arbitrary key to give to tdb. Also note that as we and
-   using transactions for all write operations and transactions take
-   care of their own locks, we don't need to do any locking anywhere
-   other than in ldb_search() */
-#define LDBLOCK	"INT_LDBLOCK"
-
 /*
   lock the database for read - use by ltdb_search
 */
 static int ltdb_lock_read(struct ldb_module *module)
 {
 	struct ltdb_private *ltdb = module->private_data;
-	TDB_DATA key;
-
-	key.dptr = discard_const(LDBLOCK);
-	key.dsize = strlen(LDBLOCK);
-
-	return tdb_chainlock_read(ltdb->tdb, key);
+	return tdb_lockall_read(ltdb->tdb);
 }
 
 /*
@@ -275,12 +263,7 @@ static int ltdb_lock_read(struct ldb_module *module)
 static int ltdb_unlock_read(struct ldb_module *module)
 {
 	struct ltdb_private *ltdb = module->private_data;
-	TDB_DATA key;
-
-	key.dptr = discard_const(LDBLOCK);
-	key.dsize = strlen(LDBLOCK);
-
-	return tdb_chainunlock_read(ltdb->tdb, key);
+	return tdb_unlockall_read(ltdb->tdb);
 }
 
 /*
@@ -499,12 +482,11 @@ int ltdb_search(struct ldb_module *module, struct ldb_request *req)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	req->handle = init_ltdb_handle(ltdb, module, req->context, req->callback);
+	req->handle = init_ltdb_handle(ltdb, module, req);
 	if (req->handle == NULL) {
 		ltdb_unlock_read(module);
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
-
 	ltdb_ac = talloc_get_type(req->handle->private_data, struct ltdb_context);
 
 	ltdb_ac->tree = req->op.search.tree;
diff --git a/source3/lib/ldb/ldb_tdb/ldb_tdb.c b/source3/lib/ldb/ldb_tdb/ldb_tdb.c
index 608120e3a7..3f9db39097 100644
--- a/source3/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/source3/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -79,13 +79,12 @@ static int ltdb_err_map(enum TDB_ERROR tdb_code)
 
 
 struct ldb_handle *init_ltdb_handle(struct ltdb_private *ltdb, struct ldb_module *module,
-					  void *context,
-					  int (*callback)(struct ldb_context *, void *, struct ldb_reply *))
+				    struct ldb_request *req)
 {
 	struct ltdb_context *ac;
 	struct ldb_handle *h;
 
-	h = talloc_zero(ltdb, struct ldb_handle);
+	h = talloc_zero(req, struct ldb_handle);
 	if (h == NULL) {
 		ldb_set_errstring(module->ldb, "Out of Memory");
 		return NULL;
@@ -106,8 +105,8 @@ struct ldb_handle *init_ltdb_handle(struct ltdb_private *ltdb, struct ldb_module
 	h->status = LDB_SUCCESS;
 
 	ac->module = module;
-	ac->context = context;
-	ac->callback = callback;
+	ac->context = req->context;
+	ac->callback = req->callback;
 
 	return h;
 }
@@ -307,7 +306,7 @@ static int ltdb_add(struct ldb_module *module, struct ldb_request *req)
 		}
 	}
 	
-	req->handle = init_ltdb_handle(ltdb, module, req->context, req->callback);
+	req->handle = init_ltdb_handle(ltdb, module, req);
 	if (req->handle == NULL) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
@@ -387,6 +386,7 @@ static int ltdb_delete_internal(struct ldb_module *module, const struct ldb_dn *
 
 	ret = ltdb_modified(module, dn);
 	if (ret != LDB_SUCCESS) {
+		talloc_free(msg);
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -416,7 +416,7 @@ static int ltdb_delete(struct ldb_module *module, struct ldb_request *req)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	req->handle = init_ltdb_handle(ltdb, module, req->context, req->callback);
+	req->handle = init_ltdb_handle(ltdb, module, req);
 	if (req->handle == NULL) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
@@ -772,7 +772,7 @@ static int ltdb_modify(struct ldb_module *module, struct ldb_request *req)
 	
 	req->handle = NULL;
 
-	req->handle = init_ltdb_handle(ltdb, module, req->context, req->callback);
+	req->handle = init_ltdb_handle(ltdb, module, req);
 	if (req->handle == NULL) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
@@ -826,7 +826,7 @@ static int ltdb_rename(struct ldb_module *module, struct ldb_request *req)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	req->handle = init_ltdb_handle(ltdb, module, req->context, req->callback);
+	req->handle = init_ltdb_handle(ltdb, module, req);
 	if (req->handle == NULL) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
@@ -1021,7 +1021,7 @@ static int ltdb_connect(struct ldb_context *ldb, const char *url,
 		path = url;
 	}
 
-	tdb_flags = TDB_DEFAULT;
+	tdb_flags = TDB_DEFAULT | TDB_SEQNUM;
 
 	/* check for the 'nosync' option */
 	if (flags & LDB_FLG_NOSYNC) {
@@ -1058,11 +1058,18 @@ static int ltdb_connect(struct ldb_context *ldb, const char *url,
 		talloc_free(ltdb);
 		return -1;
 	}
+	talloc_set_name_const(*module, "ldb_tdb backend");
 	(*module)->ldb = ldb;
 	(*module)->prev = (*module)->next = NULL;
 	(*module)->private_data = ltdb;
 	(*module)->ops = &ltdb_ops;
 
+	if (ltdb_cache_load(*module) != 0) {
+		talloc_free(*module);
+		talloc_free(ltdb);
+		return -1;
+	}
+
 	return 0;
 }
 
diff --git a/source3/lib/ldb/ldb_tdb/ldb_tdb.h b/source3/lib/ldb/ldb_tdb/ldb_tdb.h
index 670d3b6801..42f3dc2421 100644
--- a/source3/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source3/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -21,8 +21,11 @@ struct ltdb_private {
 	   handling. It has plenty of digits of precision */
 	unsigned long long sequence_number;
 
+	/* the low level tdb seqnum - used to avoid loading BASEINFO when
+	   possible */
+	int tdb_seqnum;
+
 	struct ltdb_cache {
-		struct ldb_message *baseinfo;
 		struct ldb_message *indexlist;
 		struct ldb_message *attributes;
 		struct ldb_message *subclasses;
@@ -110,8 +113,7 @@ int ltdb_search(struct ldb_module *module, struct ldb_request *req);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_tdb.c  */
 struct ldb_handle *init_ltdb_handle(struct ltdb_private *ltdb, struct ldb_module *module,
-					  void *context,
-					  int (*callback)(struct ldb_context *, void *, struct ldb_reply *));
+				    struct ldb_request *req);
 struct TDB_DATA ltdb_key(struct ldb_module *module, const struct ldb_dn *dn);
 int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
 int ltdb_delete_noindex(struct ldb_module *module, const struct ldb_dn *dn);
diff --git a/source3/lib/ldb/modules/ldb_map.c b/source3/lib/ldb/modules/ldb_map.c
index 0c58687ddb..f9ae66a2aa 100644
--- a/source3/lib/ldb/modules/ldb_map.c
+++ b/source3/lib/ldb/modules/ldb_map.c
@@ -1233,11 +1233,13 @@ static int map_init_dns(struct ldb_module *module, struct ldb_map_context *data,
 	if (res->count == 0) {
 		ldb_debug(module->ldb, LDB_DEBUG_ERROR, "ldb_map: "
 			  "No results for '%s=%s'!\n", MAP_DN_NAME, name);
+		talloc_free(res);
 		return LDB_ERR_CONSTRAINT_VIOLATION;
 	}
 	if (res->count > 1) {
 		ldb_debug(module->ldb, LDB_DEBUG_ERROR, "ldb_map: "
 			  "Too many results for '%s=%s'!\n", MAP_DN_NAME, name);
+		talloc_free(res);
 		return LDB_ERR_CONSTRAINT_VIOLATION;
 	}
 
diff --git a/source3/lib/ldb/modules/ldb_map_outbound.c b/source3/lib/ldb/modules/ldb_map_outbound.c
index e1b207f6eb..cd33f29043 100644
--- a/source3/lib/ldb/modules/ldb_map_outbound.c
+++ b/source3/lib/ldb/modules/ldb_map_outbound.c
@@ -202,7 +202,16 @@ static int ldb_msg_replace(struct ldb_message *msg, const struct ldb_message_ele
 		}
 	}
 
-	*old = *el;			/* copy new element */
+	/* copy new element */
+	*old = *el;
+
+	/* and make sure we reference the contents */
+	if (!talloc_reference(msg->elements, el->name)) {
+		return -1;
+	}
+	if (!talloc_reference(msg->elements, el->values)) {
+		return -1;
+	}
 
 	return 0;
 }
diff --git a/source3/lib/ldb/modules/objectclass.c b/source3/lib/ldb/modules/objectclass.c
index 493ecdaad4..e4040a8e3d 100644
--- a/source3/lib/ldb/modules/objectclass.c
+++ b/source3/lib/ldb/modules/objectclass.c
@@ -464,7 +464,7 @@ static int objectclass_search_self(struct ldb_handle *h) {
 	ac->search_req->operation = LDB_SEARCH;
 	ac->search_req->op.search.base = ac->orig_req->op.mod.message->dn;
 	ac->search_req->op.search.scope = LDB_SCOPE_BASE;
-	ac->search_req->op.search.tree = ldb_parse_tree(ac->module->ldb, NULL);
+	ac->search_req->op.search.tree = ldb_parse_tree(ac->search_req, NULL);
 	if (ac->search_req->op.search.tree == NULL) {
 		ldb_set_errstring(ac->module->ldb, "objectclass: Internal error producing null search");
 		return LDB_ERR_OPERATIONS_ERROR;
diff --git a/source3/lib/ldb/samba/ldif_handlers.c b/source3/lib/ldb/samba/ldif_handlers.c
index b490c8f005..46eac2295d 100644
--- a/source3/lib/ldb/samba/ldif_handlers.c
+++ b/source3/lib/ldb/samba/ldif_handlers.c
@@ -296,7 +296,7 @@ static int ldif_canonicalise_objectCategory(struct ldb_context *ldb, void *mem_c
 					    const struct ldb_val *in, struct ldb_val *out)
 {
 	struct ldb_dn *dn1 = NULL;
-	char *oc1;
+	char *oc1, *oc2;
 
 	dn1 = ldb_dn_explode(mem_ctx, (char *)in->data);
 	if (dn1 == NULL) {
@@ -308,9 +308,11 @@ static int ldif_canonicalise_objectCategory(struct ldb_context *ldb, void *mem_c
 		return -1;
 	}
 
-	oc1 = ldb_casefold(ldb, mem_ctx, oc1);
-	out->data = (void *)oc1;
-	out->length = strlen(oc1);
+	oc2 = ldb_casefold(ldb, mem_ctx, oc1);
+	out->data = (void *)oc2;
+	out->length = strlen(oc2);
+	talloc_free(oc1);
+	talloc_free(dn1);
 	return 0;
 }
 
diff --git a/source3/lib/ldb/tests/test-generic.sh b/source3/lib/ldb/tests/test-generic.sh
index e4085c6d65..14337cc135 100755
--- a/source3/lib/ldb/tests/test-generic.sh
+++ b/source3/lib/ldb/tests/test-generic.sh
@@ -10,6 +10,12 @@ echo "LDB_URL: $LDB_URL"
 echo "Adding base elements"
 $VALGRIND ldbadd $LDBDIR/tests/test.ldif || exit 1
 
+echo "Adding again - should fail"
+ldbadd $LDBDIR/tests/test.ldif 2> /dev/null && {
+    echo "Should have failed to add again - gave $?"
+    exit 1
+}
+
 echo "Modifying elements"
 $VALGRIND ldbmodify $LDBDIR/tests/test-modify.ldif || exit 1
 
@@ -32,8 +38,11 @@ if [ $LDB_SPECIALS = 1 ]; then
  $VALGRIND ldbadd $LDBDIR/tests/test-index.ldif  || exit 1
 fi
 
-echo "Adding attributes"
-$VALGRIND ldbadd $LDBDIR/tests/test-wrong_attributes.ldif  || exit 1
+echo "Adding bad attributes - should fail"
+$VALGRIND ldbadd $LDBDIR/tests/test-wrong_attributes.ldif && {
+    echo "Should fhave failed - gave $?"
+    exit 1
+}
 
 echo "testing indexed search"
 $VALGRIND ldbsearch '(uid=uham)'  || exit 1
@@ -75,7 +84,7 @@ echo "Testing binary file attribute value"
 mkdir -p tests/tmp
 cp $LDBDIR/tests/samba4.png tests/tmp/samba4.png
 $VALGRIND ldbmodify $LDBDIR/tests/photo.ldif || exit 1
-count=`$VALGRIND ldbsearch '(cn=Ursula Hampster)' jpegPhoto | grep '^dn' | wc -l`
+count=`$VALGRIND ldbsearch '(cn=Hampster Ursula)' jpegPhoto | grep '^dn' | wc -l`
 if [ $count != 1 ]; then
     echo returned $count records - expected 1
     exit 1
@@ -88,7 +97,7 @@ echo "Testing compare"
 count=`$VALGRIND ldbsearch '(cn>=t)' cn | grep '^dn' | wc -l`
 if [ $count != 2 ]; then
     echo returned $count records - expected 2
-    echo "this fails on opsnLdap ..."
+    echo "this fails on openLdap ..."
 fi
 
 count=`$VALGRIND ldbsearch '(cn<=t)' cn | grep '^dn' | wc -l`
diff --git a/source3/lib/ldb/tests/test-tdb.sh b/source3/lib/ldb/tests/test-tdb.sh
index e1052d1651..7c4f5205b4 100755
--- a/source3/lib/ldb/tests/test-tdb.sh
+++ b/source3/lib/ldb/tests/test-tdb.sh
@@ -17,6 +17,13 @@ if [ -z "$LDBDIR" ]; then
     export LDBDIR
 fi
 
+cat <<EOF | $VALGRIND ldbadd || exit 1
+dn: @MODULES
+@LIST: rdn_name
+EOF
+
+$VALGRIND ldbadd $LDBDIR/tests/init.ldif || exit 1
+
 . $LDBDIR/tests/test-generic.sh
 
 . $LDBDIR/tests/test-extended.sh
diff --git a/source3/lib/ldb/tests/test.ldif b/source3/lib/ldb/tests/test.ldif
index ab8b81437a..e53fadc700 100644
--- a/source3/lib/ldb/tests/test.ldif
+++ b/source3/lib/ldb/tests/test.ldif
@@ -409,8 +409,3 @@ homephone: +1 313 555 8421
 pager: +1 313 555 2844
 facsimiletelephonenumber: +1 313 555 9700
 telephonenumber: +1 313 555 5331
-
-dn: ou=Ldb Test,ou=People,o=University of Michigan,c=TEST
-objectclass: organizationalUnit
-ou: Ldb Test
-
diff --git a/source3/lib/ldb/tools/ad2oLschema.c b/source3/lib/ldb/tools/ad2oLschema.c
index 8b1203bff0..62c6e01c2e 100644
--- a/source3/lib/ldb/tools/ad2oLschema.c
+++ b/source3/lib/ldb/tools/ad2oLschema.c
@@ -330,7 +330,7 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum convert_
 		int j;
 
 		/* We have been asked to skip some attributes/objectClasses */
-		if (str_list_check_ci(attrs_skip, name)) {
+		if (attrs_skip && str_list_check_ci(attrs_skip, name)) {
 			ret.skipped++;
 			continue;
 		}
@@ -436,7 +436,7 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum convert_
 		int j;
 
 		/* We have been asked to skip some attributes/objectClasses */
-		if (str_list_check_ci(attrs_skip, name)) {
+		if (attrs_skip && str_list_check_ci(attrs_skip, name)) {
 			ret.skipped++;
 			continue;
 		}
diff --git a/source3/lib/ldb/tools/ldbadd.c b/source3/lib/ldb/tools/ldbadd.c
index 6a0f510244..9595703e92 100644
--- a/source3/lib/ldb/tools/ldbadd.c
+++ b/source3/lib/ldb/tools/ldbadd.c
@@ -54,10 +54,10 @@ static void usage(void)
 /*
   add records from an opened file
 */
-static int process_file(struct ldb_context *ldb, FILE *f)
+static int process_file(struct ldb_context *ldb, FILE *f, int *count)
 {
 	struct ldb_ldif *ldif;
-	int ret, count=0;
+	int ret = LDB_SUCCESS;
 
 	while ((ldif = ldb_ldif_read_file(ldb, f))) {
 		if (ldif->changetype != LDB_CHANGETYPE_ADD &&
@@ -74,12 +74,12 @@ static int process_file(struct ldb_context *ldb, FILE *f)
 				ldb_errstring(ldb), ldb_dn_linearize(ldb, ldif->msg->dn));
 			failures++;
 		} else {
-			count++;
+			(*count)++;
 		}
 		ldb_ldif_read_free(ldb, ldif);
 	}
 
-	return count;
+	return ret;
 }
 
 
@@ -87,7 +87,7 @@ static int process_file(struct ldb_context *ldb, FILE *f)
 int main(int argc, const char **argv)
 {
 	struct ldb_context *ldb;
-	int i, count=0;
+	int i, ret=0, count=0;
 	struct ldb_cmdline *options;
 
 	ldb_global_init();
@@ -97,7 +97,7 @@ int main(int argc, const char **argv)
 	options = ldb_cmdline_process(ldb, argc, argv, usage);
 
 	if (options->argc == 0) {
-		count += process_file(ldb, stdin);
+		ret = process_file(ldb, stdin, &count);
 	} else {
 		for (i=0;i<options->argc;i++) {
 			const char *fname = options->argv[i];
@@ -107,7 +107,7 @@ int main(int argc, const char **argv)
 				perror(fname);
 				exit(1);
 			}
-			count += process_file(ldb, f);
+			ret = process_file(ldb, f, &count);
 			fclose(f);
 		}
 	}
@@ -116,5 +116,5 @@ int main(int argc, const char **argv)
 
 	printf("Added %d records with %d failures\n", count, failures);
 	
-	return 0;
+	return ret;
 }
diff --git a/source3/lib/ldb/tools/ldbdel.c b/source3/lib/ldb/tools/ldbdel.c
index aee911efaf..94f1da9903 100644
--- a/source3/lib/ldb/tools/ldbdel.c
+++ b/source3/lib/ldb/tools/ldbdel.c
@@ -76,7 +76,7 @@ static void usage(void)
 int main(int argc, const char **argv)
 {
 	struct ldb_context *ldb;
-	int ret, i;
+	int ret = 0, i;
 	struct ldb_cmdline *options;
 
 	ldb_global_init();
@@ -115,5 +115,5 @@ int main(int argc, const char **argv)
 
 	talloc_free(ldb);
 
-	return 0;
+	return ret;
 }
diff --git a/source3/lib/ldb/tools/ldbmodify.c b/source3/lib/ldb/tools/ldbmodify.c
index 24f9386266..962045ef7d 100644
--- a/source3/lib/ldb/tools/ldbmodify.c
+++ b/source3/lib/ldb/tools/ldbmodify.c
@@ -53,10 +53,10 @@ static void usage(void)
 /*
   process modifies for one file
 */
-static int process_file(struct ldb_context *ldb, FILE *f)
+static int process_file(struct ldb_context *ldb, FILE *f, int *count)
 {
 	struct ldb_ldif *ldif;
-	int ret = -1, count = 0;
+	int ret = LDB_SUCCESS;
 	
 	while ((ldif = ldb_ldif_read_file(ldb, f))) {
 		switch (ldif->changetype) {
@@ -71,24 +71,24 @@ static int process_file(struct ldb_context *ldb, FILE *f)
 			ret = ldb_modify(ldb, ldif->msg);
 			break;
 		}
-		if (ret != 0) {
+		if (ret != LDB_SUCCESS) {
 			fprintf(stderr, "ERR: \"%s\" on DN %s\n", 
 				ldb_errstring(ldb), ldb_dn_linearize(ldb, ldif->msg->dn));
 			failures++;
 		} else {
-			count++;
+			(*count)++;
 		}
 		ldb_ldif_read_free(ldb, ldif);
 	}
 
-	return count;
+	return ret;
 }
 
 int main(int argc, const char **argv)
 {
 	struct ldb_context *ldb;
 	int count=0;
-	int i;
+	int i, ret=LDB_SUCCESS;
 	struct ldb_cmdline *options;
 
 	ldb_global_init();
@@ -98,7 +98,7 @@ int main(int argc, const char **argv)
 	options = ldb_cmdline_process(ldb, argc, argv, usage);
 
 	if (options->argc == 0) {
-		count += process_file(ldb, stdin);
+		ret = process_file(ldb, stdin, &count);
 	} else {
 		for (i=0;i<options->argc;i++) {
 			const char *fname = options->argv[i];
@@ -108,7 +108,7 @@ int main(int argc, const char **argv)
 				perror(fname);
 				exit(1);
 			}
-			count += process_file(ldb, f);
+			ret = process_file(ldb, f, &count);
 		}
 	}
 
@@ -116,9 +116,5 @@ int main(int argc, const char **argv)
 
 	printf("Modified %d records with %d failures\n", count, failures);
 
-	if (failures != 0) {
-		return -1;
-	}
-	
-	return 0;
+	return ret;
 }
diff --git a/source3/lib/ldb/tools/ldbsearch.c b/source3/lib/ldb/tools/ldbsearch.c
index 23d8115c20..837dfc9088 100644
--- a/source3/lib/ldb/tools/ldbsearch.c
+++ b/source3/lib/ldb/tools/ldbsearch.c
@@ -215,7 +215,7 @@ static int do_search(struct ldb_context *ldb,
 	req->operation = LDB_SEARCH;
 	req->op.search.base = basedn;
 	req->op.search.scope = options->scope;
-	req->op.search.tree = ldb_parse_tree(ldb, expression);
+	req->op.search.tree = ldb_parse_tree(req, expression);
 	if (req->op.search.tree == NULL) return -1;
 	req->op.search.attrs = attrs;
 	req->controls = sctx->req_ctrls;