Rebase ldb code with all changes in samba master

4 files changed, 403 insertions, 54 deletions

diff --git a/ldb/ldb_tdb/ldb_index.c b/ldb/ldb_tdb/ldb_index.c
index 65711d9f..c4c23022 100644
--- a/ldb/ldb_tdb/ldb_index.c
+++ b/ldb/ldb_tdb/ldb_index.c
@@ -1,4 +1,4 @@
-/* 
+/*
    ldb database library
 
    Copyright (C) Andrew Tridgell  2004
@@ -6,7 +6,7 @@
      ** NOTE! The following LGPL license applies to the ldb
      ** library. This does NOT imply that all of Samba is released
      ** under the LGPL
-   
+
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
@@ -36,13 +36,335 @@
 #include "ldb_tdb.h"
 
 /*
+  the idxptr code is a bit unusual. The way it works is to replace
+  @IDX elements in records during a transaction with @IDXPTR
+  elements. The @IDXPTR elements don't contain the actual index entry
+  values, but contain a pointer to a linked list of values.
+
+  This means we are storing pointers in a database, which is normally
+  not allowed, but in this case we are storing them only for the
+  duration of a transaction, and re-writing them into the normal @IDX
+  format at the end of the transaction. That means no other processes
+  are ever exposed to the @IDXPTR values.
+
+  The advantage is that the linked list doesn't cause huge
+  fragmentation during a transaction. Without the @IDXPTR method we
+  often ended up with a ldb that was between 10x and 100x larger then
+  it needs to be due to massive fragmentation caused by re-writing
+  @INDEX records many times during indexing.
+ */
+struct ldb_index_pointer {
+	struct ldb_index_pointer *next, *prev;
+	struct ldb_val value;
+};
+
+struct ltdb_idxptr {
+	int num_dns;
+	const char **dn_list;
+	bool repack;
+};
+
+/*
+  add to the list of DNs that need to be fixed on transaction end
+ */
+static int ltdb_idxptr_add(struct ldb_module *module, const struct ldb_message *msg)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	ltdb->idxptr->dn_list = talloc_realloc(ltdb->idxptr, ltdb->idxptr->dn_list, 
+					       const char *, ltdb->idxptr->num_dns+1);
+	if (ltdb->idxptr->dn_list == NULL) {
+		ltdb->idxptr->num_dns = 0;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ltdb->idxptr->dn_list[ltdb->idxptr->num_dns] =
+		talloc_strdup(ltdb->idxptr->dn_list, ldb_dn_get_linearized(msg->dn));
+	if (ltdb->idxptr->dn_list[ltdb->idxptr->num_dns] == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ltdb->idxptr->num_dns++;
+	return LDB_SUCCESS;
+}
+
+/* free an idxptr record */
+static int ltdb_free_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_val val;
+	struct ldb_index_pointer *ptr;
+
+	if (el->num_values != 1) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	val = el->values[0];
+	if (val.length != sizeof(void *)) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ptr = *(struct ldb_index_pointer **)val.data;
+	if (talloc_get_type(ptr, struct ldb_index_pointer) != ptr) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	while (ptr) {
+		struct ldb_index_pointer *tmp = ptr;
+		DLIST_REMOVE(ptr, ptr);
+		talloc_free(tmp);
+	}
+
+	return LDB_SUCCESS;
+}
+
+
+/* convert from the IDXPTR format to a ldb_message_element format */
+static int ltdb_convert_from_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_val val;
+	struct ldb_index_pointer *ptr, *tmp;
+	int i;
+	struct ldb_val *val2;
+
+	if (el->num_values != 1) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	val = el->values[0];
+	if (val.length != sizeof(void *)) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ptr = *(struct ldb_index_pointer **)val.data;
+	if (talloc_get_type(ptr, struct ldb_index_pointer) != ptr) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	/* count the length of the list */
+	for (i=0, tmp = ptr; tmp; tmp=tmp->next) {
+		i++;
+	}
+
+	/* allocate the new values array */
+	val2 = talloc_realloc(NULL, el->values, struct ldb_val, i);
+	if (val2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	el->values = val2;
+	el->num_values = i;
+
+	/* populate the values array */
+	for (i=0, tmp = ptr; tmp; tmp=tmp->next, i++) {
+		el->values[i].length = tmp->value.length;
+		/* we need to over-allocate here as there are still some places
+		   in ldb that rely on null termination. */
+		el->values[i].data = talloc_size(el->values, tmp->value.length+1);
+		if (el->values[i].data == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		memcpy(el->values[i].data, tmp->value.data, tmp->value.length);
+		el->values[i].data[tmp->value.length] = 0;
+	}
+
+	/* update the name */
+	el->name = LTDB_IDX;
+
+       	return LDB_SUCCESS;
+}
+
+
+/* convert to the IDXPTR format from a ldb_message_element format */
+static int ltdb_convert_to_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_index_pointer *ptr, *tmp;
+	int i;
+	struct ldb_val *val2;
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+
+	ptr = NULL;
+
+	for (i=0;i<el->num_values;i++) {
+		tmp = talloc(ltdb->idxptr, struct ldb_index_pointer);
+		if (tmp == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		tmp->value = el->values[i];
+		tmp->value.data = talloc_memdup(tmp, tmp->value.data, tmp->value.length);
+		if (tmp->value.data == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		DLIST_ADD(ptr, tmp);
+	}
+
+	/* allocate the new values array */
+	val2 = talloc_realloc(NULL, el->values, struct ldb_val, 1);
+	if (val2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	el->values = val2;
+	el->num_values = 1;
+
+	el->values[0].data = talloc_memdup(el->values, &ptr, sizeof(ptr));
+	el->values[0].length = sizeof(ptr);
+
+	/* update the name */
+	el->name = LTDB_IDXPTR;
+
+       	return LDB_SUCCESS;
+}
+
+
+/* enable the idxptr mode when transactions start */
+int ltdb_index_transaction_start(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	ltdb->idxptr = talloc_zero(module, struct ltdb_idxptr);
+	return LDB_SUCCESS;
+}
+
+/*
+  a wrapper around ltdb_search_dn1() which translates pointer based index records
+  and maps them into normal ldb message structures
+ */
+static int ltdb_search_dn1_index(struct ldb_module *module,
+				struct ldb_dn *dn, struct ldb_message *msg)
+{
+	int ret, i;
+	ret = ltdb_search_dn1(module, dn, msg);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/* if this isn't a @INDEX record then don't munge it */
+	if (strncmp(ldb_dn_get_linearized(msg->dn), LTDB_INDEX ":", strlen(LTDB_INDEX) + 1) != 0) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	for (i=0;i<msg->num_elements;i++) {
+		struct ldb_message_element *el = &msg->elements[i];
+		if (strcmp(el->name, LTDB_IDXPTR) == 0) {
+			ret = ltdb_convert_from_idxptr(module, el);
+			if (ret != LDB_SUCCESS) {
+				return ret;
+			}
+		}
+	}
+
+	return ret;
+}
+
+
+
+/*
+  fixup the idxptr for one DN
+ */
+static int ltdb_idxptr_fix_dn(struct ldb_module *module, const char *strdn)
+{
+	struct ldb_dn *dn;
+	struct ldb_message *msg = ldb_msg_new(module);
+	int ret;
+
+	dn = ldb_dn_new(msg, module->ldb, strdn);
+	if (ltdb_search_dn1_index(module, dn, msg) == LDB_SUCCESS) {
+		ret = ltdb_store(module, msg, TDB_REPLACE);
+	}
+	talloc_free(msg);
+	return ret;
+}
+
+/* cleanup the idxptr mode when transaction commits */
+int ltdb_index_transaction_commit(struct ldb_module *module)
+{
+	int i;
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+
+	/* fix all the DNs that we have modified */
+	if (ltdb->idxptr) {
+		for (i=0;i<ltdb->idxptr->num_dns;i++) {
+			ltdb_idxptr_fix_dn(module, ltdb->idxptr->dn_list[i]);
+		}
+
+		if (ltdb->idxptr->repack) {
+			tdb_repack(ltdb->tdb);
+		}
+	}
+
+	talloc_free(ltdb->idxptr);
+	ltdb->idxptr = NULL;
+	return LDB_SUCCESS;
+}
+
+/* cleanup the idxptr mode when transaction cancels */
+int ltdb_index_transaction_cancel(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	talloc_free(ltdb->idxptr);
+	ltdb->idxptr = NULL;
+	return LDB_SUCCESS;
+}
+
+
+
+/* a wrapper around ltdb_store() for the index code which
+   stores in IDXPTR format when idxptr mode is enabled
+
+   WARNING: This modifies the msg which is passed in
+*/
+int ltdb_store_idxptr(struct ldb_module *module, const struct ldb_message *msg, int flgs)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	int ret;
+
+	if (ltdb->idxptr) {
+		int i;
+		struct ldb_message *msg2 = ldb_msg_new(module);
+
+		/* free any old pointer */
+		ret = ltdb_search_dn1(module, msg->dn, msg2);
+		if (ret == 0) {
+			for (i=0;i<msg2->num_elements;i++) {
+				struct ldb_message_element *el = &msg2->elements[i];
+				if (strcmp(el->name, LTDB_IDXPTR) == 0) {
+					ret = ltdb_free_idxptr(module, el);
+					if (ret != LDB_SUCCESS) {
+						return ret;
+					}
+				}
+			}
+		}
+		talloc_free(msg2);
+
+		for (i=0;i<msg->num_elements;i++) {
+			struct ldb_message_element *el = &msg->elements[i];
+			if (strcmp(el->name, LTDB_IDX) == 0) {
+				ret = ltdb_convert_to_idxptr(module, el);
+				if (ret != LDB_SUCCESS) {
+					return ret;
+				}
+			}
+		}
+
+		if (ltdb_idxptr_add(module, msg) != 0) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+	}
+
+	ret = ltdb_store(module, msg, flgs);
+	return ret;
+}
+
+
+/*
   find an element in a list, using the given comparison function and
   assuming that the list is already sorted using comp_fn
 
   return -1 if not found, or the index of the first occurance of needle if found
 */
-static int ldb_list_find(const void *needle, 
-			 const void *base, size_t nmemb, size_t size, 
+static int ldb_list_find(const void *needle,
+			 const void *base, size_t nmemb, size_t size,
 			 comparison_fn_t comp_fn)
 {
 	const char *base_p = (const char *)base;
@@ -183,7 +505,7 @@ static int list_cmp(const char **s1, const char **s2)
 /*
   return a list of dn's that might match a simple indexed search or
  */
-static int ltdb_index_dn_simple(struct ldb_module *module, 
+static int ltdb_index_dn_simple(struct ldb_module *module,
 				const struct ldb_parse_tree *tree,
 				const struct ldb_message *index_list,
 				struct dn_list *list)
@@ -213,7 +535,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, dn, msg);
+	ret = ltdb_search_dn1_index(module, dn, msg);
 	talloc_free(dn);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -235,7 +557,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
 		}
 
 		for (j=0;j<el->num_values;j++) {
-			list->dn[list->count] = 
+			list->dn[list->count] =
 				talloc_strdup(list->dn, (char *)el->values[j].data);
 			if (!list->dn[list->count]) {
 				talloc_free(msg);
@@ -260,7 +582,7 @@ static int list_union(struct ldb_context *, struct dn_list *, const struct dn_li
 /*
   return a list of dn's that might match a leaf indexed search
  */
-static int ltdb_index_dn_leaf(struct ldb_module *module, 
+static int ltdb_index_dn_leaf(struct ldb_module *module,
 			      const struct ldb_parse_tree *tree,
 			      const struct ldb_message *index_list,
 			      struct dn_list *list)
@@ -312,13 +634,13 @@ static int list_intersect(struct ldb_context *ldb,
 	list3->count = 0;
 
 	for (i=0;i<list->count;i++) {
-		if (ldb_list_find(list->dn[i], list2->dn, list2->count, 
+		if (ldb_list_find(list->dn[i], list2->dn, list2->count,
 			      sizeof(char *), (comparison_fn_t)strcmp) != -1) {
 			list3->dn[list3->count] = talloc_move(list3->dn, &list->dn[i]);
 			list3->count++;
 		} else {
 			talloc_free(list->dn[i]);
-		}		
+		}
 	}
 
 	talloc_free(list->dn);
@@ -335,7 +657,7 @@ static int list_intersect(struct ldb_context *ldb,
   list = list | list2
   relies on the lists being sorted
 */
-static int list_union(struct ldb_context *ldb, 
+static int list_union(struct ldb_context *ldb,
 		      struct dn_list *list, const struct dn_list *list2)
 {
 	unsigned int i;
@@ -354,14 +676,14 @@ static int list_union(struct ldb_context *ldb,
 	list->dn = d;
 
 	for (i=0;i<list2->count;i++) {
-		if (ldb_list_find(list2->dn[i], list->dn, count, 
+		if (ldb_list_find(list2->dn[i], list->dn, count,
 			      sizeof(char *), (comparison_fn_t)strcmp) == -1) {
 			list->dn[list->count] = talloc_strdup(list->dn, list2->dn[i]);
 			if (!list->dn[list->count]) {
 				return LDB_ERR_OPERATIONS_ERROR;
 			}
 			list->count++;
-		}		
+		}
 	}
 
 	if (list->count != count) {
@@ -371,7 +693,7 @@ static int list_union(struct ldb_context *ldb,
 	return LDB_ERR_NO_SUCH_OBJECT;
 }
 
-static int ltdb_index_dn(struct ldb_module *module, 
+static int ltdb_index_dn(struct ldb_module *module,
 			 const struct ldb_parse_tree *tree,
 			 const struct ldb_message *index_list,
 			 struct dn_list *list);
@@ -380,7 +702,7 @@ static int ltdb_index_dn(struct ldb_module *module,
 /*
   OR two index results
  */
-static int ltdb_index_dn_or(struct ldb_module *module, 
+static int ltdb_index_dn_or(struct ldb_module *module,
 			    const struct ldb_parse_tree *tree,
 			    const struct ldb_message *index_list,
 			    struct dn_list *list)
@@ -388,7 +710,7 @@ static int ltdb_index_dn_or(struct ldb_module *module,
 	struct ldb_context *ldb = module->ldb;
 	unsigned int i;
 	int ret;
-	
+
 	ret = LDB_ERR_OPERATIONS_ERROR;
 	list->dn = NULL;
 	list->count = 0;
@@ -445,7 +767,7 @@ static int ltdb_index_dn_or(struct ldb_module *module,
 /*
   NOT an index results
  */
-static int ltdb_index_dn_not(struct ldb_module *module, 
+static int ltdb_index_dn_not(struct ldb_module *module,
 			     const struct ldb_parse_tree *tree,
 			     const struct ldb_message *index_list,
 			     struct dn_list *list)
@@ -453,8 +775,8 @@ static int ltdb_index_dn_not(struct ldb_module *module,
 	/* the only way to do an indexed not would be if we could
 	   negate the not via another not or if we knew the total
 	   number of database elements so we could know that the
-	   existing expression covered the whole database. 
-	   
+	   existing expression covered the whole database.
+
 	   instead, we just give up, and rely on a full index scan
 	   (unless an outer & manages to reduce the list)
 	*/
@@ -464,7 +786,7 @@ static int ltdb_index_dn_not(struct ldb_module *module,
 /*
   AND two index results
  */
-static int ltdb_index_dn_and(struct ldb_module *module, 
+static int ltdb_index_dn_and(struct ldb_module *module,
 			     const struct ldb_parse_tree *tree,
 			     const struct ldb_message *index_list,
 			     struct dn_list *list)
@@ -472,7 +794,7 @@ static int ltdb_index_dn_and(struct ldb_module *module,
 	struct ldb_context *ldb = module->ldb;
 	unsigned int i;
 	int ret;
-	
+
 	ret = LDB_ERR_OPERATIONS_ERROR;
 	list->dn = NULL;
 	list->count = 0;
@@ -526,7 +848,7 @@ static int ltdb_index_dn_and(struct ldb_module *module,
 /*
   AND index results and ONE level special index
  */
-static int ltdb_index_dn_one(struct ldb_module *module, 
+static int ltdb_index_dn_one(struct ldb_module *module,
 			     struct ldb_dn *parent_dn,
 			     struct dn_list *list)
 {
@@ -537,13 +859,13 @@ static int ltdb_index_dn_one(struct ldb_module *module,
 	struct ldb_val val;
 	unsigned int i, j;
 	int ret;
-	
+
 	list2 = talloc_zero(module, struct dn_list);
 	if (list2 == NULL) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	/* the attribute is indexed. Pull the list of DNs that match the 
+	/* the attribute is indexed. Pull the list of DNs that match the
 	   search criterion */
 	val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(parent_dn));
 	val.length = strlen((char *)val.data);
@@ -559,7 +881,7 @@ static int ltdb_index_dn_one(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, key, msg);
+	ret = ltdb_search_dn1_index(module, key, msg);
 	talloc_free(key);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -624,7 +946,7 @@ static int ltdb_index_dn_one(struct ldb_module *module,
   return a list of dn's that might match a indexed search or
   an error. return LDB_ERR_NO_SUCH_OBJECT for no matches, or LDB_SUCCESS for matches
  */
-static int ltdb_index_dn(struct ldb_module *module, 
+static int ltdb_index_dn(struct ldb_module *module,
 			 const struct ldb_parse_tree *tree,
 			 const struct ldb_message *index_list,
 			 struct dn_list *list)
@@ -666,7 +988,7 @@ static int ltdb_index_dn(struct ldb_module *module,
   filter a candidate dn_list from an indexed search into a set of results
   extracting just the given attributes
 */
-static int ltdb_index_filter(const struct dn_list *dn_list, 
+static int ltdb_index_filter(const struct dn_list *dn_list,
 			     struct ltdb_context *ac)
 {
 	struct ldb_message *msg;
@@ -715,7 +1037,7 @@ static int ltdb_index_filter(const struct dn_list *dn_list,
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
 
-		ret = ldb_module_send_entry(ac->req, msg);
+		ret = ldb_module_send_entry(ac->req, msg, NULL);
 		if (ret != LDB_SUCCESS) {
 			ac->callback_failed = true;
 			return ret;
@@ -728,7 +1050,7 @@ static int ltdb_index_filter(const struct dn_list *dn_list,
 /*
   search the database with a LDAP-like expression using indexes
   returns -1 if an indexed search is not possible, in which
-  case the caller should call ltdb_search_full() 
+  case the caller should call ltdb_search_full()
 */
 int ltdb_search_indexed(struct ltdb_context *ac)
 {
@@ -804,14 +1126,14 @@ int ltdb_search_indexed(struct ltdb_context *ac)
 /*
   add a index element where this is the first indexed DN for this value
 */
-static int ltdb_index_add1_new(struct ldb_context *ldb, 
+static int ltdb_index_add1_new(struct ldb_context *ldb,
 			       struct ldb_message *msg,
 			       const char *dn)
 {
 	struct ldb_message_element *el;
 
 	/* add another entry */
-	el = talloc_realloc(msg, msg->elements, 
+	el = talloc_realloc(msg, msg->elements,
 			       struct ldb_message_element, msg->num_elements+1);
 	if (!el) {
 		return LDB_ERR_OPERATIONS_ERROR;
@@ -840,7 +1162,7 @@ static int ltdb_index_add1_new(struct ldb_context *ldb,
   add a index element where this is not the first indexed DN for this
   value
 */
-static int ltdb_index_add1_add(struct ldb_context *ldb, 
+static int ltdb_index_add1_add(struct ldb_context *ldb,
 			       struct ldb_message *msg,
 			       int idx,
 			       const char *dn)
@@ -856,7 +1178,7 @@ static int ltdb_index_add1_add(struct ldb_context *ldb,
 	}
 
 	v2 = talloc_realloc(msg->elements, msg->elements[idx].values,
-			      struct ldb_val, 
+			      struct ldb_val,
 			      msg->elements[idx].num_values+1);
 	if (!v2) {
 		return LDB_ERR_OPERATIONS_ERROR;
@@ -873,7 +1195,7 @@ static int ltdb_index_add1_add(struct ldb_context *ldb,
 /*
   add an index entry for one message element
 */
-static int ltdb_index_add1(struct ldb_module *module, const char *dn, 
+static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 			   struct ldb_message_element *el, int v_idx)
 {
 	struct ldb_context *ldb = module->ldb;
@@ -895,7 +1217,7 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 	}
 	talloc_steal(msg, dn_key);
 
-	ret = ltdb_search_dn1(module, dn_key, msg);
+	ret = ltdb_search_dn1_index(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(msg);
 		return ret;
@@ -920,7 +1242,7 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 	}
 
 	if (ret == LDB_SUCCESS) {
-		ret = ltdb_store(module, msg, TDB_REPLACE);
+		ret = ltdb_store_idxptr(module, msg, TDB_REPLACE);
 	}
 
 	talloc_free(msg);
@@ -945,7 +1267,7 @@ static int ltdb_index_add0(struct ldb_module *module, const char *dn,
 	}
 
 	for (i = 0; i < num_el; i++) {
-		ret = ldb_msg_find_idx(ltdb->cache->indexlist, elements[i].name, 
+		ret = ldb_msg_find_idx(ltdb->cache->indexlist, elements[i].name,
 				       NULL, LTDB_IDXATTR);
 		if (ret == -1) {
 			continue;
@@ -983,7 +1305,7 @@ int ltdb_index_add(struct ldb_module *module, const struct ldb_message *msg)
 /*
   delete an index entry for one message element
 */
-int ltdb_index_del_value(struct ldb_module *module, const char *dn, 
+int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 			 struct ldb_message_element *el, int v_idx)
 {
 	struct ldb_context *ldb = module->ldb;
@@ -1007,7 +1329,7 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, dn_key, msg);
+	ret = ltdb_search_dn1_index(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(dn_key);
 		return ret;
@@ -1022,18 +1344,24 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 
 	i = ldb_msg_find_idx(msg, dn, &j, LTDB_IDX);
 	if (i == -1) {
+		struct ldb_ldif ldif;
+
 		ldb_debug(ldb, LDB_DEBUG_ERROR,
 				"ERROR: dn %s not found in %s\n", dn,
 				ldb_dn_get_linearized(dn_key));
+		ldif.changetype = LDB_CHANGETYPE_NONE;
+		ldif.msg = msg;
+		ldb_ldif_write_file(module->ldb, stdout, &ldif);
+		sleep(100);
 		/* it ain't there. hmmm */
 		talloc_free(dn_key);
 		return LDB_SUCCESS;
 	}
 
 	if (j != msg->elements[i].num_values - 1) {
-		memmove(&msg->elements[i].values[j], 
-			&msg->elements[i].values[j+1], 
-			(msg->elements[i].num_values-(j+1)) * 
+		memmove(&msg->elements[i].values[j],
+			&msg->elements[i].values[j+1],
+			(msg->elements[i].num_values-(j+1)) *
 			sizeof(msg->elements[i].values[0]));
 	}
 	msg->elements[i].num_values--;
@@ -1041,7 +1369,7 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 	if (msg->elements[i].num_values == 0) {
 		ret = ltdb_delete_noindex(module, dn_key);
 	} else {
-		ret = ltdb_store(module, msg, TDB_REPLACE);
+		ret = ltdb_store_idxptr(module, msg, TDB_REPLACE);
 	}
 
 	talloc_free(dn_key);
@@ -1060,7 +1388,7 @@ int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg)
 	const char *dn;
 	unsigned int i, j;
 
-	/* find the list of indexed fields */	
+	/* find the list of indexed fields */
 	if (ltdb->cache->indexlist->num_elements == 0) {
 		/* no indexed fields */
 		return LDB_SUCCESS;
@@ -1092,7 +1420,7 @@ int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg)
 	return LDB_SUCCESS;
 }
 
-/* 
+/*
   handle special index for one level searches
 */
 int ltdb_index_one(struct ldb_module *module, const struct ldb_message *msg, int add)
@@ -1183,7 +1511,7 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
 		return -1;
 	}
 
-	/* check if the DN key has changed, perhaps due to the 
+	/* check if the DN key has changed, perhaps due to the
 	   case insensitivity of an element changing */
 	key2 = ltdb_key(module, msg->dn);
 	if (key2.dptr == NULL) {
@@ -1250,5 +1578,9 @@ int ltdb_reindex(struct ldb_module *module)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
+	if (ltdb->idxptr) {
+		ltdb->idxptr->repack = true;
+	}
+
 	return LDB_SUCCESS;
 }
diff --git a/ldb/ldb_tdb/ldb_search.c b/ldb/ldb_tdb/ldb_search.c
index 1c76411d..35149c4b 100644
--- a/ldb/ldb_tdb/ldb_search.c
+++ b/ldb/ldb_tdb/ldb_search.c
@@ -205,7 +205,7 @@ static struct ldb_message *ltdb_pull_attrs(struct ldb_module *module,
   return LDB_ERR_NO_SUCH_OBJECT on record-not-found
   and LDB_SUCCESS on success
 */
-int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
+static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
 {
 	struct ltdb_private *ltdb = (struct ltdb_private *)module->private_data;
 	TDB_DATA tdb_key, tdb_data;
@@ -418,7 +418,7 @@ static int search_func(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, voi
 		return -1;
 	}
 
-	ret = ldb_module_send_entry(ac->req, msg);
+	ret = ldb_module_send_entry(ac->req, msg, NULL);
 	if (ret != LDB_SUCCESS) {
 		ac->callback_failed = true;
 		/* the callback failed, abort the operation */
diff --git a/ldb/ldb_tdb/ldb_tdb.c b/ldb/ldb_tdb/ldb_tdb.c
index 2282f61d..ea460def 100644
--- a/ldb/ldb_tdb/ldb_tdb.c
+++ b/ldb/ldb_tdb/ldb_tdb.c
@@ -161,7 +161,7 @@ failed:
   check special dn's have valid attributes
   currently only @ATTRIBUTES is checked
 */
-int ltdb_check_special_dn(struct ldb_module *module,
+static int ltdb_check_special_dn(struct ldb_module *module,
 			  const struct ldb_message *msg)
 {
 	int i, j;
@@ -850,6 +850,8 @@ static int ltdb_start_trans(struct ldb_module *module)
 
 	ltdb->in_transaction++;
 
+	ltdb_index_transaction_start(module);
+
 	return LDB_SUCCESS;
 }
 
@@ -860,6 +862,10 @@ static int ltdb_end_trans(struct ldb_module *module)
 
 	ltdb->in_transaction--;
 
+	if (ltdb_index_transaction_commit(module) != 0) {
+		return ltdb_err_map(tdb_error(ltdb->tdb));
+	}
+
 	if (tdb_transaction_commit(ltdb->tdb) != 0) {
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
@@ -874,6 +880,10 @@ static int ltdb_del_trans(struct ldb_module *module)
 
 	ltdb->in_transaction--;
 
+	if (ltdb_index_transaction_cancel(module) != 0) {
+		return ltdb_err_map(tdb_error(ltdb->tdb));
+	}
+
 	if (tdb_transaction_cancel(ltdb->tdb) != 0) {
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
@@ -968,7 +978,7 @@ done:
 	return ret;
 }
 
-void ltdb_request_done(struct ldb_request *req, int error)
+static void ltdb_request_done(struct ldb_request *req, int error)
 {
 	struct ldb_reply *ares;
 
@@ -1076,12 +1086,14 @@ static void ltdb_callback(struct event_context *ev,
 	}
 
 	if (!ctx->callback_failed) {
+		/* Once we are done, we do not need timeout events */
+		talloc_free(ctx->timeout_event);
 		ltdb_request_done(ctx->req, ret);
 	}
 }
 
 static int ltdb_handle_request(struct ldb_module *module,
-				struct ldb_request *req)
+			       struct ldb_request *req)
 {
 	struct event_context *ev;
 	struct ltdb_context *ac;
@@ -1115,10 +1127,9 @@ static int ltdb_handle_request(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-
 	tv.tv_sec = req->starttime + req->timeout;
-	te = event_add_timed(ev, ac, tv, ltdb_timeout, ac);
-	if (NULL == te) {
+	ac->timeout_event = event_add_timed(ev, ac, tv, ltdb_timeout, ac);
+	if (NULL == ac->timeout_event) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
diff --git a/ldb/ldb_tdb/ldb_tdb.h b/ldb/ldb_tdb/ldb_tdb.h
index 61e90bcc..7b9a7650 100644
--- a/ldb/ldb_tdb/ldb_tdb.h
+++ b/ldb/ldb_tdb/ldb_tdb.h
@@ -33,6 +33,7 @@ struct ltdb_private {
 	int in_transaction;
 
 	bool check_base;
+	struct ltdb_idxptr *idxptr;
 };
 
 /*
@@ -50,12 +51,14 @@ struct ltdb_context {
 	struct ldb_dn *base;
 	enum ldb_scope scope;
 	const char * const *attrs;
+	struct timed_event *timeout_event;
 };
 
 /* special record types */
 #define LTDB_INDEX      "@INDEX"
 #define LTDB_INDEXLIST  "@INDEXLIST"
 #define LTDB_IDX        "@IDX"
+#define LTDB_IDXPTR     "@IDXPTR"
 #define LTDB_IDXATTR    "@IDXATTR"
 #define LTDB_IDXONE     "@IDXONE"
 #define LTDB_BASEINFO   "@BASEINFO"
@@ -84,6 +87,9 @@ int ltdb_index_add(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_one(struct ldb_module *module, const struct ldb_message *msg, int add);
 int ltdb_reindex(struct ldb_module *module);
+int ltdb_index_transaction_start(struct ldb_module *module);
+int ltdb_index_transaction_commit(struct ldb_module *module);
+int ltdb_index_transaction_cancel(struct ldb_module *module);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_pack.c  */