Merge branch 'master' of ssh://git.samba.org/data/git/samba into pyregistry

24 files changed, 1002 insertions, 123 deletions

diff --git a/source4/lib/ldb/common/ldb_attributes.c b/source4/lib/ldb/common/ldb_attributes.c
index 747f241781..48f9e11caf 100644
--- a/source4/lib/ldb/common/ldb_attributes.c
+++ b/source4/lib/ldb/common/ldb_attributes.c
@@ -225,3 +225,50 @@ int ldb_setup_wellknown_attributes(struct ldb_context *ldb)
 	return LDB_SUCCESS;
 }
 
+
+/*
+  add a extended dn syntax to the ldb_schema
+*/
+int ldb_dn_extended_add_syntax(struct ldb_context *ldb, 
+			       unsigned flags,
+			       const struct ldb_dn_extended_syntax *syntax)
+{
+	int n;
+	struct ldb_dn_extended_syntax *a;
+
+	if (!syntax) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	n = ldb->schema.num_dn_extended_syntax + 1;
+
+	a = talloc_realloc(ldb, ldb->schema.dn_extended_syntax,
+			   struct ldb_dn_extended_syntax, n);
+
+	if (!a) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	a[ldb->schema.num_dn_extended_syntax] = *syntax;
+	ldb->schema.dn_extended_syntax = a;
+
+	ldb->schema.num_dn_extended_syntax = n;
+
+	return 0;
+}
+
+/*
+  return the extended dn syntax for a given name
+*/
+const struct ldb_dn_extended_syntax *ldb_dn_extended_syntax_by_name(struct ldb_context *ldb,
+								    const char *name)
+{
+	int i;
+	for (i=0; i < ldb->schema.num_dn_extended_syntax; i++) {
+		if (ldb_attr_cmp(ldb->schema.dn_extended_syntax[i].name, name) == 0) {
+			return &ldb->schema.dn_extended_syntax[i];
+		}
+	}
+	return NULL;
+}
+
diff --git a/source4/lib/ldb/common/ldb_controls.c b/source4/lib/ldb/common/ldb_controls.c
index e3f8551407..6fad5012b6 100644
--- a/source4/lib/ldb/common/ldb_controls.c
+++ b/source4/lib/ldb/common/ldb_controls.c
@@ -53,6 +53,26 @@ struct ldb_control *ldb_request_get_control(struct ldb_request *req, const char
 	return NULL;
 }
 
+/* check if a control with the specified "oid" exist and return it */
+/* returns NULL if not found */
+struct ldb_control *ldb_reply_get_control(struct ldb_reply *rep, const char *oid)
+{
+	int i;
+
+	/* check if there's a paged request control */
+	if (rep->controls != NULL) {
+		for (i = 0; rep->controls[i]; i++) {
+			if (strcmp(oid, rep->controls[i]->oid) == 0) {
+				break;
+			}
+		}
+
+		return rep->controls[i];
+	}
+
+	return NULL;
+}
+
 /* saves the current controls list into the "saver" and replace the one in req with a new one excluding
 the "exclude" control */
 /* returns False on error */
diff --git a/source4/lib/ldb/common/ldb_dn.c b/source4/lib/ldb/common/ldb_dn.c
index e36aea4e69..13c01f4e70 100644
--- a/source4/lib/ldb/common/ldb_dn.c
+++ b/source4/lib/ldb/common/ldb_dn.c
@@ -52,6 +52,12 @@ struct ldb_dn_component {
 	struct ldb_val cf_value;
 };
 
+struct ldb_dn_extended_component {
+
+	char *name;
+	struct ldb_val value;
+};
+
 struct ldb_dn {
 
 	struct ldb_context *ldb;
@@ -63,11 +69,14 @@ struct ldb_dn {
 	bool valid_case;
 
 	char *linearized;
+	char *extended_linearized;
 	char *casefold;
 
 	unsigned int comp_num;
 	struct ldb_dn_component *components;
 
+	unsigned int extended_comp_num;
+	struct ldb_dn_extended_component *extended_components;
 };
 
 /* strdn may be NULL */
@@ -85,28 +94,34 @@ struct ldb_dn *ldb_dn_from_ldb_val(void *mem_ctx, struct ldb_context *ldb, const
 	if (strdn->data && strdn->length) {
 		if (strdn->data[0] == '@') {
 			dn->special = true;
+		} 
+		dn->extended_linearized = talloc_strndup(dn, (const char *)strdn->data, strdn->length);
+		LDB_DN_NULL_FAILED(dn->extended_linearized);
+	
+		if (strdn->data[0] == '<') {
+			const char *p_save, *p = dn->extended_linearized;
+			do {
+				p_save = p;
+				p = strstr(p, ">;");
+				if (p) {
+					p = p + 2;
+				}
+			} while (p);
+			
+			if (p_save == dn->extended_linearized) {
+				dn->linearized = talloc_strdup(dn, "");
+			} else {
+				dn->linearized = talloc_strdup(dn, p_save);
+			}
+			LDB_DN_NULL_FAILED(dn->linearized);
+		} else {
+			dn->linearized = dn->extended_linearized;
+			dn->extended_linearized = NULL;
 		}
-		if (strdn->length >= 6 && strncasecmp((const char *)strdn->data, "<GUID=", 6) == 0) {
-			/* this is special DN returned when the
-			 * exploded_dn control is used */
-			dn->special = true;
-			/* FIXME: add a GUID string to ldb_dn structure */
-		} else if (strdn->length >= 5 && strncasecmp((const char *)strdn->data, "<SID=", 5) == 0) {
-			/* this is special DN returned when the
-			 * exploded_dn control is used */
-			dn->special = true;
-			/* FIXME: add a SID string to ldb_dn structure */
-		} else if (strdn->length >= 8 && strncasecmp((const char *)strdn->data, "<WKGUID=", 8) == 0) {
-			/* this is special DN returned when the
-			 * exploded_dn control is used */
-			dn->special = true;
-			/* FIXME: add a WKGUID string to ldb_dn structure */
-		}
-		dn->linearized = talloc_strndup(dn, (const char *)strdn->data, strdn->length);
 	} else {
 		dn->linearized = talloc_strdup(dn, "");
+		LDB_DN_NULL_FAILED(dn->linearized);
 	}
-	LDB_DN_NULL_FAILED(dn->linearized);
 
 	return dn;
 
@@ -126,47 +141,21 @@ struct ldb_dn *ldb_dn_new(void *mem_ctx, struct ldb_context *ldb, const char *st
 
 struct ldb_dn *ldb_dn_new_fmt(void *mem_ctx, struct ldb_context *ldb, const char *new_fmt, ...)
 {
-	struct ldb_dn *dn;
 	char *strdn;
 	va_list ap;
 
 	if ( (! mem_ctx) || (! ldb)) return NULL;
 
-	dn = talloc_zero(mem_ctx, struct ldb_dn);
-	LDB_DN_NULL_FAILED(dn);
-
-	dn->ldb = ldb;
-
 	va_start(ap, new_fmt);
-	strdn = talloc_vasprintf(dn, new_fmt, ap);
+	strdn = talloc_vasprintf(mem_ctx, new_fmt, ap);
 	va_end(ap);
-	LDB_DN_NULL_FAILED(strdn);
-
-	if (strdn[0] == '@') {
-		dn->special = true;
-	}
-	if (strncasecmp(strdn, "<GUID=", 6) == 0) {
-		/* this is special DN returned when the
-		 * exploded_dn control is used */
-		dn->special = true;
-		/* FIXME: add a GUID string to ldb_dn structure */
-	} else if (strncasecmp(strdn, "<SID=", 5) == 0) {
-		/* this is special DN returned when the
-		 * exploded_dn control is used */
-		dn->special = true;
-		/* FIXME: add a SID string to ldb_dn structure */
-	} else if (strncasecmp(strdn, "<WKGUID=", 8) == 0) {
-		/* this is special DN returned when the
-		 * exploded_dn control is used */
-		dn->special = true;
-		/* FIXME: add a WKGUID string to ldb_dn structure */
-	}
-	dn->linearized = strdn;
 
-	return dn;
-
-failed:
-	talloc_free(dn);
+	if (strdn) {
+		struct ldb_dn *dn = ldb_dn_new(mem_ctx, ldb, strdn);
+		talloc_free(strdn);
+		return dn;
+	}
+	
 	return NULL;
 }
 
@@ -235,15 +224,19 @@ char *ldb_dn_escape_value(void *mem_ctx, struct ldb_val value)
 */
 static bool ldb_dn_explode(struct ldb_dn *dn)
 {
-	char *p, *data, *d, *dt, *t;
+	char *p, *ex_name, *ex_value, *data, *d, *dt, *t;
 	bool trim = false;
+	bool in_extended = false;
+	bool in_ex_name = false;
+	bool in_ex_value = false;
 	bool in_attr = false;
 	bool in_value = false;
 	bool in_quote = false;
 	bool is_oid = false;
 	bool escape = false;
 	unsigned x;
-	int l;
+	int l, ret;
+	char *parse_dn;
 
 	if ( ! dn || dn->invalid) return false;
 
@@ -251,12 +244,18 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 		return true;
 	}
 
-	if ( ! dn->linearized) {
+	if (dn->extended_linearized) {
+		parse_dn = dn->extended_linearized;
+	} else {
+		parse_dn = dn->linearized;
+	}
+
+	if ( ! parse_dn ) {
 		return false;
 	}
 
 	/* Empty DNs */
-	if (dn->linearized[0] == '\0') {
+	if (parse_dn[0] == '\0') {
 		return true;
 	}
 
@@ -268,6 +267,9 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 	/* make sure we free this if alloced previously before replacing */
 	talloc_free(dn->components);
 
+	talloc_free(dn->extended_components);
+	dn->extended_components = NULL;
+
 	/* in the common case we have 3 or more components */
 	/* make sure all components are zeroed, other functions depend on this */
 	dn->components = talloc_zero_array(dn, struct ldb_dn_component, 3);
@@ -277,19 +279,109 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 	dn->comp_num = 0;
 
 	/* Components data space is allocated here once */
-	data = talloc_array(dn->components, char, strlen(dn->linearized) + 1);
+	data = talloc_array(dn->components, char, strlen(parse_dn) + 1);
 	if (!data) {
 		return false;
 	}
 
-	p = dn->linearized;
-	in_attr = true;
+	p = parse_dn;
+	in_extended = true;
+	in_ex_name = false;
+	in_ex_value = false;
 	trim = true;
 	t = NULL;
 	d = dt = data;
 
 	while (*p) {
+		if (in_extended) {
+
+			if (!in_ex_name && !in_ex_value) {
+
+				if (p[0] == '<') {
+					p++;
+					ex_name = d;
+					in_ex_name = true;
+					continue;
+				} else if (p[0] == '\0') {
+					p++;
+					continue;
+				} else {
+					in_extended = false;
+					in_attr = true;
+					dt = d;
+
+					continue;
+				}
+			}
+			
+			if (in_ex_name && *p == '=') {
+				*d++ = '\0';
+				p++;
+				ex_value = d;
+				in_ex_name = false;
+				in_ex_value = true;
+				continue;
+			}
+
+			if (in_ex_value && *p == '>') {
+				const struct ldb_dn_extended_syntax *extended_syntax;
+				struct ldb_val ex_val = {
+					.data = ex_value,
+					.length = d - ex_value
+				};
+					
+				*d++ = '\0';
+				p++;
+				in_ex_value = false;
+
+				/* Process name and ex_value */
+
+				dn->extended_components = talloc_realloc(dn,
+									 dn->extended_components,
+									 struct ldb_dn_extended_component,
+									 dn->extended_comp_num + 1);
+				if ( ! dn->extended_components) {
+					/* ouch ! */
+					goto failed;
+				}
+
+				extended_syntax = ldb_dn_extended_syntax_by_name(dn->ldb, ex_name);
+				if (!extended_syntax) {
+					/* We don't know about this type of extended DN */
+					goto failed;
+				}
+
+				dn->extended_components[dn->extended_comp_num].name = talloc_strdup(dn->extended_components, ex_name);
+				if (!dn->extended_components[dn->extended_comp_num].name) {
+					/* ouch */
+					goto failed;
+				}
+				ret = extended_syntax->read_fn(dn->ldb, dn->extended_components,
+							       &ex_val, &dn->extended_components[dn->extended_comp_num].value);
+				if (ret != LDB_SUCCESS) {
+					dn->invalid = true;
+					goto failed;
+				}
+
+				dn->extended_comp_num++;
 
+				if (*p == '\0') {
+					/* We have reached the end (extended component only)! */
+					talloc_free(data);
+					return true;
+
+				} else if (*p == ';') {
+					p++;
+					continue;
+				} else {
+					dn->invalid = true;
+					goto failed;
+				}
+			}
+
+			*d++ = *p++;
+			continue;
+		}
 		if (in_attr) {
 			if (trim) {
 				if (*p == ' ') {
@@ -315,6 +407,7 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 					goto failed;
 				}
 				
+				/* Copy this character across from parse_dn, now we have trimmed out spaces */
 				*d++ = *p++;
 				continue;
 			}
@@ -339,6 +432,7 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 				trim = true;
 				l = 0;
 
+				/* Terminate this string in d (which is a copy of parse_dn with spaces trimmed) */
 				*d++ = '\0';
 				dn->components[dn->comp_num].name = talloc_strdup(dn->components, dt);
 				if ( ! dn->components[dn->comp_num].name) {
@@ -614,6 +708,74 @@ const char *ldb_dn_get_linearized(struct ldb_dn *dn)
 	return dn->linearized;
 }
 
+char *ldb_dn_get_extended_linearized(void *mem_ctx, struct ldb_dn *dn, int mode)
+{
+	const char *linearized = ldb_dn_get_linearized(dn);
+	char *p;
+	int i;
+
+	if (!linearized) {
+		return NULL;
+	}
+
+	if (!ldb_dn_has_extended(dn)) {
+		return talloc_strdup(mem_ctx, linearized);
+	}
+	
+	if (!ldb_dn_validate(dn)) {
+		return NULL;
+	}
+
+	for (i=0; i < dn->extended_comp_num; i++) {
+		struct ldb_val val;
+		int ret;
+		const struct ldb_dn_extended_syntax *extended_syntax;
+		const char *name = dn->extended_components[i].name;
+		
+		extended_syntax = ldb_dn_extended_syntax_by_name(dn->ldb, name);
+
+		if (mode == 1) {
+			ret = extended_syntax->write_clear_fn(dn->ldb, mem_ctx,
+							      &dn->extended_components[i].value,
+							      &val);
+		} else if (mode == 0) {
+			ret = extended_syntax->write_hex_fn(dn->ldb, mem_ctx,
+							      &dn->extended_components[i].value,
+							      &val);
+		} else {
+			ret = -1;
+		}
+
+		if (ret != LDB_SUCCESS) {
+			return NULL;
+		}
+
+		if (i == 0) {
+			p = talloc_asprintf(mem_ctx, "<%s=%s>", dn->extended_components[i].name, val.data);
+		} else {
+			p = talloc_asprintf_append(p, ";<%s=%s>",  dn->extended_components[i].name, val.data);
+		}
+
+		talloc_free(val.data);
+
+		if (!p) {
+			return NULL;
+		}
+	}
+
+	if (dn->extended_comp_num && *linearized) {
+		p = talloc_asprintf_append(p, ";%s", linearized);
+	}
+
+	if (!p) {
+		return NULL;
+	}
+
+	return p;
+}
+
+
+
 char *ldb_dn_alloc_linearized(void *mem_ctx, struct ldb_dn *dn)
 {
 	return talloc_strdup(mem_ctx, ldb_dn_get_linearized(dn));
@@ -909,6 +1071,30 @@ static struct ldb_dn_component ldb_dn_copy_component(void *mem_ctx, struct ldb_d
 	return dst;
 }
 
+static struct ldb_dn_extended_component ldb_dn_extended_copy_component(void *mem_ctx, struct ldb_dn_extended_component *src)
+{
+	struct ldb_dn_extended_component dst;
+
+	memset(&dst, 0, sizeof(dst));
+
+	if (src == NULL) {
+		return dst;
+	}
+
+	dst.value = ldb_val_dup(mem_ctx, &(src->value));
+	if (dst.value.data == NULL) {
+		return dst;
+	}
+
+	dst.name = talloc_strdup(mem_ctx, src->name);
+	if (dst.name == NULL) {
+		LDB_FREE(dst.value.data);
+		return dst;
+	}
+
+	return dst;
+}
+
 struct ldb_dn *ldb_dn_copy(void *mem_ctx, struct ldb_dn *dn)
 {
 	struct ldb_dn *new_dn;
@@ -942,6 +1128,24 @@ struct ldb_dn *ldb_dn_copy(void *mem_ctx, struct ldb_dn *dn)
 		}
 	}
 
+	if (dn->extended_components) {
+		int i;
+
+		new_dn->extended_components = talloc_zero_array(new_dn, struct ldb_dn_extended_component, dn->extended_comp_num);
+		if ( ! new_dn->extended_components) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+
+		for (i = 0; i < dn->extended_comp_num; i++) {
+			new_dn->extended_components[i] = ldb_dn_extended_copy_component(new_dn->extended_components, &dn->extended_components[i]);
+			if ( ! new_dn->extended_components[i].value.data) {
+				talloc_free(new_dn);
+				return NULL;
+			}
+		}
+	}
+
 	if (dn->casefold) {
 		new_dn->casefold = talloc_strdup(new_dn, dn->casefold);
 		if ( ! new_dn->casefold) {
@@ -958,6 +1162,14 @@ struct ldb_dn *ldb_dn_copy(void *mem_ctx, struct ldb_dn *dn)
 		}
 	}
 
+	if (dn->extended_linearized) {
+		new_dn->extended_linearized = talloc_strdup(new_dn, dn->extended_linearized);
+		if ( ! new_dn->extended_linearized) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+	}
+
 	return new_dn;
 }
 
@@ -1037,6 +1249,13 @@ bool ldb_dn_add_base(struct ldb_dn *dn, struct ldb_dn *base)
 		dn->linearized = t;
 	}
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	if (dn->extended_linearized) {
+		LDB_FREE(dn->extended_linearized);
+	}
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
 	return true;
 }
 
@@ -1149,6 +1368,12 @@ bool ldb_dn_add_child(struct ldb_dn *dn, struct ldb_dn *child)
 		dn->linearized = t;
 	}
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
+
 	return true;
 }
 
@@ -1218,6 +1443,12 @@ bool ldb_dn_remove_base_components(struct ldb_dn *dn, unsigned int num)
 	LDB_FREE(dn->casefold);
 	LDB_FREE(dn->linearized);
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
+
 	return true;
 }
 
@@ -1256,6 +1487,11 @@ bool ldb_dn_remove_child_components(struct ldb_dn *dn, unsigned int num)
 	LDB_FREE(dn->casefold);
 	LDB_FREE(dn->linearized);
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
 	return true;
 }
 
@@ -1273,6 +1509,11 @@ struct ldb_dn *ldb_dn_get_parent(void *mem_ctx, struct ldb_dn *dn)
 		return NULL;
 	}
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
 	return new_dn;
 }
 
@@ -1434,9 +1675,97 @@ int ldb_dn_set_component(struct ldb_dn *dn, int num, const char *name, const str
 	LDB_FREE(dn->casefold);
 	LDB_FREE(dn->linearized);
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	dn->extended_comp_num = 0;
+	LDB_FREE(dn->extended_components);
 	return LDB_SUCCESS;
 }
 
+const struct ldb_val *ldb_dn_get_extended_component(struct ldb_dn *dn, const char *name)
+{
+	int i;
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+	for (i=0; i < dn->extended_comp_num; i++) {
+		if (ldb_attr_cmp(dn->extended_components[i].name, name) == 0) {
+			return &dn->extended_components[i].value;
+		}
+	}
+	return NULL;
+}
+
+int ldb_dn_set_extended_component(struct ldb_dn *dn, const char *name, const struct ldb_val *val)
+{
+	struct ldb_dn_extended_component *p;
+	int i;
+				
+	if ( ! ldb_dn_validate(dn)) {
+		return LDB_ERR_OTHER;
+	}
+
+	for (i=0; i < dn->extended_comp_num; i++) {
+		if (ldb_attr_cmp(dn->extended_components[i].name, name) == 0) {
+			if (val) {
+				dn->extended_components[i].value = ldb_val_dup(dn->extended_components, val);
+
+				dn->extended_components[i].name = talloc_strdup(dn->extended_components, name);
+				if (!dn->extended_components[i].name || !dn->extended_components[i].value.data) {
+					dn->invalid = true;
+					return LDB_ERR_OPERATIONS_ERROR;
+				}
+	
+			} else {
+				if (i != (dn->extended_comp_num - 1)) {
+					memmove(&dn->extended_components[i], &dn->extended_components[i+1],
+						((dn->extended_comp_num-1) - i)*sizeof(*dn->extended_components));
+				}
+				dn->extended_comp_num--;
+				
+				dn->extended_components = talloc_realloc(dn,
+						   dn->extended_components,
+						   struct ldb_dn_extended_component,
+						   dn->extended_comp_num);
+				if (!dn->extended_components) {
+					dn->invalid = true;
+					return LDB_ERR_OPERATIONS_ERROR;
+				}
+				return 0;
+			}
+		}
+	}
+
+	p = dn->extended_components
+		= talloc_realloc(dn,
+				 dn->extended_components,
+				 struct ldb_dn_extended_component,
+				 dn->extended_comp_num + 1);
+	if (!dn->extended_components) {
+		dn->invalid = true;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	
+	p[dn->extended_comp_num].value = ldb_val_dup(dn->extended_components, val);
+	p[dn->extended_comp_num].name = talloc_strdup(p, name);
+	
+	if (!dn->extended_components[i].name || !dn->extended_components[i].value.data) {
+		dn->invalid = true;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	dn->extended_components = p;
+	dn->extended_comp_num++;
+	
+	return 0;
+}
+
+void ldb_dn_remove_extended_components(struct ldb_dn *dn)
+{
+	dn->extended_comp_num = 0;
+	LDB_FREE(dn->extended_components);	
+}
+
 bool ldb_dn_is_valid(struct ldb_dn *dn)
 {
 	if ( ! dn) return false;
@@ -1449,6 +1778,13 @@ bool ldb_dn_is_special(struct ldb_dn *dn)
 	return dn->special;
 }
 
+bool ldb_dn_has_extended(struct ldb_dn *dn)
+{
+	if ( ! dn || dn->invalid) return false;
+	if (dn->extended_linearized && (dn->extended_linearized[0] == '<')) return true;
+	return dn->extended_comp_num != 0;
+}
+
 bool ldb_dn_check_special(struct ldb_dn *dn, const char *check)
 {
 	if ( ! dn || dn->invalid) return false;
@@ -1458,6 +1794,7 @@ bool ldb_dn_check_special(struct ldb_dn *dn, const char *check)
 bool ldb_dn_is_null(struct ldb_dn *dn)
 {
 	if ( ! dn || dn->invalid) return false;
+	if (ldb_dn_has_extended(dn)) return false;
 	if (dn->linearized && (dn->linearized[0] == '\0')) return true;
 	return false;
 }
diff --git a/source4/lib/ldb/common/ldb_ldif.c b/source4/lib/ldb/common/ldb_ldif.c
index 69490e670b..619c10e11e 100644
--- a/source4/lib/ldb/common/ldb_ldif.c
+++ b/source4/lib/ldb/common/ldb_ldif.c
@@ -278,13 +278,15 @@ int ldb_ldif_write(struct ldb_context *ldb,
 	TALLOC_CTX *mem_ctx;
 	unsigned int i, j;
 	int total=0, ret;
+	char *p;
 	const struct ldb_message *msg;
 
 	mem_ctx = talloc_named_const(NULL, 0, "ldb_ldif_write");
 
 	msg = ldif->msg;
-
-	ret = fprintf_fn(private_data, "dn: %s\n", ldb_dn_get_linearized(msg->dn));
+	p = ldb_dn_get_extended_linearized(mem_ctx, msg->dn, 1);
+	ret = fprintf_fn(private_data, "dn: %s\n", p);
+	talloc_free(p);
 	CHECK_RET;
 
 	if (ldif->changetype != LDB_CHANGETYPE_NONE) {
diff --git a/source4/lib/ldb/common/ldb_modules.c b/source4/lib/ldb/common/ldb_modules.c
index ab0f4c51cc..8db28d262c 100644
--- a/source4/lib/ldb/common/ldb_modules.c
+++ b/source4/lib/ldb/common/ldb_modules.c
@@ -587,10 +587,13 @@ struct ldb_handle *ldb_handle_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb)
  *      req: the original request passed to your module
  *      msg: reply message (must be a talloc pointer, and it will be stolen
  *           on the ldb_reply that is sent to the callback)
+ * 	ctrls: controls to send in the reply  (must be a talloc pointer, and it will be stolen
+ *           on the ldb_reply that is sent to the callback)
  */
 
 int ldb_module_send_entry(struct ldb_request *req,
-			  struct ldb_message *msg)
+			  struct ldb_message *msg,
+			  struct ldb_control **ctrls)
 {
 	struct ldb_reply *ares;
 
@@ -602,6 +605,7 @@ int ldb_module_send_entry(struct ldb_request *req,
 	}
 	ares->type = LDB_REPLY_ENTRY;
 	ares->message = talloc_steal(ares, msg);
+	ares->controls = talloc_steal(ares, ctrls);
 	ares->error = LDB_SUCCESS;
 
 	return req->callback(req, ares);
diff --git a/source4/lib/ldb/include/ldb.h b/source4/lib/ldb/include/ldb.h
index 9bc5c183d8..e2ec869872 100644
--- a/source4/lib/ldb/include/ldb.h
+++ b/source4/lib/ldb/include/ldb.h
@@ -326,7 +326,6 @@ typedef int (*ldb_attr_comparison_t)(struct ldb_context *, TALLOC_CTX *mem_ctx,
   attribute handler structure
 
   attr			-> The attribute name
-  flags			-> LDB_ATTR_FLAG_*
   ldif_read_fn		-> convert from ldif to binary format
   ldif_write_fn		-> convert from binary to ldif format
   canonicalise_fn	-> canonicalise a value, for use by indexing and dn construction
@@ -350,6 +349,16 @@ struct ldb_schema_attribute {
 const struct ldb_schema_attribute *ldb_schema_attribute_by_name(struct ldb_context *ldb,
 								const char *name);
 
+struct ldb_dn_extended_syntax {
+	const char *name;
+	ldb_attr_handler_t read_fn;
+	ldb_attr_handler_t write_clear_fn;
+	ldb_attr_handler_t write_hex_fn;
+};
+
+const struct ldb_dn_extended_syntax *ldb_dn_extended_syntax_by_name(struct ldb_context *ldb,
+								    const char *name);
+
 /**
    The attribute is not returned by default
 */
@@ -1066,6 +1075,15 @@ int ldb_request_add_control(struct ldb_request *req, const char *oid, bool criti
 struct ldb_control *ldb_request_get_control(struct ldb_request *req, const char *oid);
 
 /**
+   check if a control with the specified "oid" exist and return it 
+  \param rep the reply struct where to add the control
+  \param oid the object identifier of the control as string
+
+  \return the control, NULL if not found 
+*/
+struct ldb_control *ldb_reply_get_control(struct ldb_reply *rep, const char *oid);
+
+/**
   Search the database
 
   This function searches the database, and returns 
@@ -1421,15 +1439,82 @@ int ldb_base64_decode(char *s);
 
 /* The following definitions come from lib/ldb/common/ldb_dn.c  */
 
+/**
+  Get the linear form of a DN (without any extended components)
+  
+  \param dn The DN to linearize
+*/
+
+const char *ldb_dn_get_linearized(struct ldb_dn *dn);
+
+/**
+  Allocate a copy of the linear form of a DN (without any extended components) onto the supplied memory context 
+  
+  \param dn The DN to linearize
+  \param mem_ctx TALLOC context to return result on
+*/
+
+char *ldb_dn_alloc_linearized(TALLOC_CTX *mem_ctx, struct ldb_dn *dn);
+
+/**
+  Get the linear form of a DN (with any extended components)
+  
+  \param mem_ctx TALLOC context to return result on
+  \param dn The DN to linearize
+  \param mode Style of extended DN to return (0 is HEX representation of binary form, 1 is a string form)
+*/
+char *ldb_dn_get_extended_linearized(void *mem_ctx, struct ldb_dn *dn, int mode);
+const struct ldb_val *ldb_dn_get_extended_component(struct ldb_dn *dn, const char *name);
+int ldb_dn_set_extended_component(struct ldb_dn *dn, const char *name, const struct ldb_val *val);
+
+void ldb_dn_remove_extended_components(struct ldb_dn *dn);
+bool ldb_dn_has_extended(struct ldb_dn *dn);
+
+int ldb_dn_extended_add_syntax(struct ldb_context *ldb, 
+			       unsigned flags,
+			       const struct ldb_dn_extended_syntax *syntax);
+
+/** 
+  Allocate a new DN from a string
+
+  \param mem_ctx TALLOC context to return resulting ldb_dn structure on
+  \param dn The new DN 
+
+  \note The DN will not be parsed at this time.  Use ldb_dn_validate to tell if the DN is syntacticly correct
+*/
+
 struct ldb_dn *ldb_dn_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, const char *dn);
+/** 
+  Allocate a new DN from a printf style format string and arguments
+
+  \param mem_ctx TALLOC context to return resulting ldb_dn structure on
+  \param new_fms The new DN as a format string (plus arguments)
+
+  \note The DN will not be parsed at this time.  Use ldb_dn_validate to tell if the DN is syntacticly correct
+*/
+
 struct ldb_dn *ldb_dn_new_fmt(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, const char *new_fmt, ...) PRINTF_ATTRIBUTE(3,4);
+/** 
+  Allocate a new DN from a struct ldb_val (useful to avoid buffer overrun)
+
+  \param mem_ctx TALLOC context to return resulting ldb_dn structure on
+  \param dn The new DN 
+
+  \note The DN will not be parsed at this time.  Use ldb_dn_validate to tell if the DN is syntacticly correct
+*/
+
 struct ldb_dn *ldb_dn_from_ldb_val(void *mem_ctx, struct ldb_context *ldb, const struct ldb_val *strdn);
+
+/**
+  Determine if this DN is syntactically valid 
+
+  \param dn The DN to validate
+*/
+
 bool ldb_dn_validate(struct ldb_dn *dn);
 
 char *ldb_dn_escape_value(TALLOC_CTX *mem_ctx, struct ldb_val value);
-const char *ldb_dn_get_linearized(struct ldb_dn *dn);
 const char *ldb_dn_get_casefold(struct ldb_dn *dn);
-char *ldb_dn_alloc_linearized(TALLOC_CTX *mem_ctx, struct ldb_dn *dn);
 char *ldb_dn_alloc_casefold(TALLOC_CTX *mem_ctx, struct ldb_dn *dn);
 
 int ldb_dn_compare_base(struct ldb_dn *base, struct ldb_dn *dn);
diff --git a/source4/lib/ldb/include/ldb_private.h b/source4/lib/ldb/include/ldb_private.h
index 90c4980017..c065288279 100644
--- a/source4/lib/ldb/include/ldb_private.h
+++ b/source4/lib/ldb/include/ldb_private.h
@@ -84,6 +84,9 @@ struct ldb_schema {
 	/* attribute handling table */
 	unsigned num_attributes;
 	struct ldb_schema_attribute *attributes;
+
+	unsigned num_dn_extended_syntax;
+	struct ldb_dn_extended_syntax *dn_extended_syntax;
 };
 
 /*
@@ -262,7 +265,8 @@ int ldb_register_backend(const char *url_prefix, ldb_connect_fn);
 struct ldb_handle *ldb_handle_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb);
 
 int ldb_module_send_entry(struct ldb_request *req,
-			  struct ldb_message *msg);
+			  struct ldb_message *msg,
+			  struct ldb_control **ctrls);
 
 int ldb_module_send_referral(struct ldb_request *req,
 					   char *ref);
diff --git a/source4/lib/ldb/ldb.i b/source4/lib/ldb/ldb.i
index 0f05c1fbab..7831d6da60 100644
--- a/source4/lib/ldb/ldb.i
+++ b/source4/lib/ldb/ldb.i
@@ -575,6 +575,8 @@ static void py_ldb_debug(void *context, enum ldb_debug_level level, const char *
         if (ldif == NULL) {
             return Py_None;
         } else {
+	    /* We don't want this attached to the 'ldb' any more */
+            talloc_steal(NULL, ldif);
             return Py_BuildValue((char *)"(iO)", ldif->changetype, 
                    SWIG_NewPointerObj(ldif->msg, SWIGTYPE_p_ldb_message, 0));
         }
diff --git a/source4/lib/ldb/ldb_ildap/ldb_ildap.c b/source4/lib/ldb/ldb_ildap/ldb_ildap.c
index 93e81d4244..b17d063c0c 100644
--- a/source4/lib/ldb/ldb_ildap/ldb_ildap.c
+++ b/source4/lib/ldb/ldb_ildap/ldb_ildap.c
@@ -311,7 +311,9 @@ static void ildb_callback(struct ldap_request *req)
 				ldbmsg->num_elements = search->num_attributes;
 				ldbmsg->elements = talloc_move(ldbmsg, &search->attributes);
 
-				ret = ldb_module_send_entry(ac->req, ldbmsg);
+				controls = talloc_steal(ac, msg->controls);
+				
+				ret = ldb_module_send_entry(ac->req, ldbmsg, controls);
 				if (ret != LDB_SUCCESS) {
 					callback_failed = true;
 				}
@@ -428,7 +430,7 @@ static int ildb_search(struct ildb_context *ac)
 	if (req->op.search.base == NULL) {
 		msg->r.SearchRequest.basedn = talloc_strdup(msg, "");
 	} else {
-		msg->r.SearchRequest.basedn  = ldb_dn_alloc_linearized(msg, req->op.search.base);
+		msg->r.SearchRequest.basedn  = ldb_dn_get_extended_linearized(msg, req->op.search.base, 0);
 	}
 	if (msg->r.SearchRequest.basedn == NULL) {
 		ldb_set_errstring(ac->module->ldb, "Unable to determine baseDN");
@@ -473,7 +475,7 @@ static int ildb_add(struct ildb_context *ac)
 
 	msg->type = LDAP_TAG_AddRequest;
 
-	msg->r.AddRequest.dn = ldb_dn_alloc_linearized(msg, req->op.add.message->dn);
+	msg->r.AddRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.add.message->dn, 0);
 	if (msg->r.AddRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
@@ -516,7 +518,7 @@ static int ildb_modify(struct ildb_context *ac)
 
 	msg->type = LDAP_TAG_ModifyRequest;
 
-	msg->r.ModifyRequest.dn = ldb_dn_alloc_linearized(msg, req->op.mod.message->dn);
+	msg->r.ModifyRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.mod.message->dn, 0);
 	if (msg->r.ModifyRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
@@ -557,7 +559,7 @@ static int ildb_delete(struct ildb_context *ac)
 
 	msg->type = LDAP_TAG_DelRequest;
 
-	msg->r.DelRequest.dn = ldb_dn_alloc_linearized(msg, req->op.del.dn);
+	msg->r.DelRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.del.dn, 0);
 	if (msg->r.DelRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
@@ -580,7 +582,7 @@ static int ildb_rename(struct ildb_context *ac)
 	}
 
 	msg->type = LDAP_TAG_ModifyDNRequest;
-	msg->r.ModifyDNRequest.dn = ldb_dn_alloc_linearized(msg, req->op.rename.olddn);
+	msg->r.ModifyDNRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.rename.olddn, 0);
 	if (msg->r.ModifyDNRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
diff --git a/source4/lib/ldb/ldb_ldap/ldb_ldap.c b/source4/lib/ldb/ldb_ldap/ldb_ldap.c
index 7caee10b47..9f8b3e9f35 100644
--- a/source4/lib/ldb/ldb_ldap/ldb_ldap.c
+++ b/source4/lib/ldb/ldb_ldap/ldb_ldap.c
@@ -515,7 +515,7 @@ static bool lldb_parse_result(struct lldb_context *ac, LDAPMessage *result)
 			}
 			if (berptr) ber_free(berptr, 0);
 
-			ret = ldb_module_send_entry(ac->req, ldbmsg);
+			ret = ldb_module_send_entry(ac->req, ldbmsg, NULL /* controls not yet supported */);
 			if (ret != LDB_SUCCESS) {
 
 				callback_failed = true;
diff --git a/source4/lib/ldb/ldb_map/ldb_map_outbound.c b/source4/lib/ldb/ldb_map/ldb_map_outbound.c
index 5f524a8be3..5588eaaf46 100644
--- a/source4/lib/ldb/ldb_map/ldb_map_outbound.c
+++ b/source4/lib/ldb/ldb_map/ldb_map_outbound.c
@@ -1077,7 +1077,7 @@ int map_return_entry(struct map_context *ac, struct ldb_reply *ares)
 		}
 	}
 
-	return ldb_module_send_entry(ac->req, ares->message);
+	return ldb_module_send_entry(ac->req, ares->message, ares->controls);
 }
 
 /* Search a record. */
diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c
index eedbda4170..0a10307e65 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_index.c
@@ -36,6 +36,328 @@
 #include "ldb_tdb.h"
 
 /*
+  the idxptr code is a bit unusual. The way it works is to replace
+  @IDX elements in records during a transaction with @IDXPTR
+  elements. The @IDXPTR elements don't contain the actual index entry
+  values, but contain a pointer to a linked list of values. 
+
+  This means we are storing pointers in a database, which is normally
+  not allowed, but in this case we are storing them only for the
+  duration of a transaction, and re-writing them into the normal @IDX
+  format at the end of the transaction. That means no other processes
+  are ever exposed to the @IDXPTR values.
+
+  The advantage is that the linked list doesn't cause huge
+  fragmentation during a transaction. Without the @IDXPTR method we
+  often ended up with a ldb that was between 10x and 100x larger then
+  it needs to be due to massive fragmentation caused by re-writing
+  @INDEX records many times during indexing.
+ */
+struct ldb_index_pointer {
+	struct ldb_index_pointer *next, *prev;
+	struct ldb_val value;
+};
+
+struct ltdb_idxptr {
+	int num_dns;
+	const char **dn_list;
+	bool repack;
+};
+
+/*
+  add to the list of DNs that need to be fixed on transaction end
+ */
+static int ltdb_idxptr_add(struct ldb_module *module, const struct ldb_message *msg)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	ltdb->idxptr->dn_list = talloc_realloc(ltdb->idxptr, ltdb->idxptr->dn_list, 
+					       const char *, ltdb->idxptr->num_dns+1);
+	if (ltdb->idxptr->dn_list == NULL) {
+		ltdb->idxptr->num_dns = 0;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ltdb->idxptr->dn_list[ltdb->idxptr->num_dns] = 
+		talloc_strdup(ltdb->idxptr->dn_list, ldb_dn_get_linearized(msg->dn));
+	if (ltdb->idxptr->dn_list[ltdb->idxptr->num_dns] == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ltdb->idxptr->num_dns++;
+	return LDB_SUCCESS;
+}
+
+/* free an idxptr record */
+static int ltdb_free_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_val val;
+	struct ldb_index_pointer *ptr;
+
+	if (el->num_values != 1) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	val = el->values[0];
+	if (val.length != sizeof(void *)) {
+		return LDB_ERR_OPERATIONS_ERROR;				
+	}
+			
+	ptr = *(struct ldb_index_pointer **)val.data;
+	if (talloc_get_type(ptr, struct ldb_index_pointer) != ptr) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+
+	while (ptr) {
+		struct ldb_index_pointer *tmp = ptr;
+		DLIST_REMOVE(ptr, ptr);
+		talloc_free(tmp);
+	}
+
+	return 0;
+}
+
+
+/* convert from the IDXPTR format to a ldb_message_element format */
+static int ltdb_convert_from_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_val val;
+	struct ldb_index_pointer *ptr, *tmp;
+	int i;
+	struct ldb_val *val2;
+
+	if (el->num_values != 1) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	val = el->values[0];
+	if (val.length != sizeof(void *)) {
+		return LDB_ERR_OPERATIONS_ERROR;				
+	}
+			
+	ptr = *(struct ldb_index_pointer **)val.data;
+	if (talloc_get_type(ptr, struct ldb_index_pointer) != ptr) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+
+	/* count the length of the list */
+	for (i=0, tmp = ptr; tmp; tmp=tmp->next) {
+		i++;
+	}
+
+	/* allocate the new values array */
+	val2 = talloc_realloc(NULL, el->values, struct ldb_val, i);
+	if (val2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+	el->values = val2;
+	el->num_values = i;
+
+	/* populate the values array */
+	for (i=0, tmp = ptr; tmp; tmp=tmp->next, i++) {
+		el->values[i].length = tmp->value.length;
+		/* we need to over-allocate here as there are still some places
+		   in ldb that rely on null termination. */
+		el->values[i].data = talloc_size(el->values, tmp->value.length+1);
+		if (el->values[i].data == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		memcpy(el->values[i].data, tmp->value.data, tmp->value.length);
+		el->values[i].data[tmp->value.length] = 0;
+	}	
+
+	/* update the name */
+	el->name = LTDB_IDX;
+	
+       	return LDB_SUCCESS;
+}
+
+
+/* convert to the IDXPTR format from a ldb_message_element format */
+static int ltdb_convert_to_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_index_pointer *ptr, *tmp;
+	int i;
+	struct ldb_val *val2;
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+
+	ptr = NULL;
+
+	for (i=0;i<el->num_values;i++) {
+		tmp = talloc(ltdb->idxptr, struct ldb_index_pointer);
+		if (tmp == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;		
+		}
+		tmp->value = el->values[i];
+		tmp->value.data = talloc_memdup(tmp, tmp->value.data, tmp->value.length);
+		if (tmp->value.data == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;		
+		}
+		DLIST_ADD(ptr, tmp);
+	}
+
+	/* allocate the new values array */
+	val2 = talloc_realloc(NULL, el->values, struct ldb_val, 1);
+	if (val2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+	el->values = val2;
+	el->num_values = 1;
+
+	el->values[0].data = talloc_memdup(el->values, &ptr, sizeof(ptr));
+	el->values[0].length = sizeof(ptr);
+
+	/* update the name */
+	el->name = LTDB_IDXPTR;
+
+       	return LDB_SUCCESS;	
+}
+
+
+/* enable the idxptr mode when transactions start */
+int ltdb_index_transaction_start(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	ltdb->idxptr = talloc_zero(module, struct ltdb_idxptr);
+	return 0;
+}
+
+/*
+  a wrapper around ltdb_search_dn1() which translates pointer based index records
+  and maps them into normal ldb message structures
+ */
+static int ltdb_search_dn1_index(struct ldb_module *module, 
+				struct ldb_dn *dn, struct ldb_message *msg)
+{
+	int ret, i;
+	ret = ltdb_search_dn1(module, dn, msg);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/* if this isn't a @INDEX record then don't munge it */
+	if (strncmp(ldb_dn_get_linearized(msg->dn), LTDB_INDEX ":", strlen(LTDB_INDEX) + 1) != 0) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	for (i=0;i<msg->num_elements;i++) {
+		struct ldb_message_element *el = &msg->elements[i];
+		if (strcmp(el->name, LTDB_IDXPTR) == 0) {
+			ret = ltdb_convert_from_idxptr(module, el);
+			if (ret != LDB_SUCCESS) {
+				return ret;
+			}
+		}
+	}
+
+	return ret;
+}
+
+
+
+/*
+  fixup the idxptr for one DN
+ */
+static int ltdb_idxptr_fix_dn(struct ldb_module *module, const char *strdn)
+{
+	struct ldb_dn *dn;
+	struct ldb_message *msg = ldb_msg_new(module);
+	int ret;
+	
+	dn = ldb_dn_new(msg, module->ldb, strdn);
+	if (ltdb_search_dn1_index(module, dn, msg) == LDB_SUCCESS) {
+		ret = ltdb_store(module, msg, TDB_REPLACE);
+	}
+	talloc_free(msg);
+	return ret;
+}
+
+/* cleanup the idxptr mode when transaction commits */
+int ltdb_index_transaction_commit(struct ldb_module *module)
+{
+	int i;
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+
+	/* fix all the DNs that we have modified */
+	if (ltdb->idxptr) {
+		for (i=0;i<ltdb->idxptr->num_dns;i++) {
+			ltdb_idxptr_fix_dn(module, ltdb->idxptr->dn_list[i]);
+		}
+
+		if (ltdb->idxptr->repack) {
+			tdb_repack(ltdb->tdb);
+		}
+	}
+
+	talloc_free(ltdb->idxptr);
+	ltdb->idxptr = NULL;
+	return 0;
+}
+
+/* cleanup the idxptr mode when transaction cancels */
+int ltdb_index_transaction_cancel(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	talloc_free(ltdb->idxptr);
+	ltdb->idxptr = NULL;
+	return 0;
+}
+
+			
+
+/* a wrapper around ltdb_store() for the index code which 
+   stores in IDXPTR format when idxptr mode is enabled 
+
+   WARNING: This modifies the msg which is passed in
+*/
+int ltdb_store_idxptr(struct ldb_module *module, const struct ldb_message *msg, int flgs)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	int ret;
+
+	if (ltdb->idxptr) {
+		int i;
+		struct ldb_message *msg2 = ldb_msg_new(module);
+
+		/* free any old pointer */
+		ret = ltdb_search_dn1(module, msg->dn, msg2);
+		if (ret == 0) {
+			for (i=0;i<msg2->num_elements;i++) {
+				struct ldb_message_element *el = &msg2->elements[i];
+				if (strcmp(el->name, LTDB_IDXPTR) == 0) {
+					ret = ltdb_free_idxptr(module, el);
+					if (ret != LDB_SUCCESS) {
+						return ret;
+					}
+				}
+			}
+		}
+		talloc_free(msg2);
+
+		for (i=0;i<msg->num_elements;i++) {
+			struct ldb_message_element *el = &msg->elements[i];
+			if (strcmp(el->name, LTDB_IDX) == 0) {
+				ret = ltdb_convert_to_idxptr(module, el);
+				if (ret != LDB_SUCCESS) {
+					return ret;
+				}
+			}
+		}
+
+		if (ltdb_idxptr_add(module, msg) != 0) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+	}
+
+	ret = ltdb_store(module, msg, flgs);
+	return ret;
+}
+
+
+/*
   find an element in a list, using the given comparison function and
   assuming that the list is already sorted using comp_fn
 
@@ -213,7 +535,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, dn, msg);
+	ret = ltdb_search_dn1_index(module, dn, msg);
 	talloc_free(dn);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -559,7 +881,7 @@ static int ltdb_index_dn_one(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, key, msg);
+	ret = ltdb_search_dn1_index(module, key, msg);
 	talloc_free(key);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -715,7 +1037,7 @@ static int ltdb_index_filter(const struct dn_list *dn_list,
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
 
-		ret = ldb_module_send_entry(ac->req, msg);
+		ret = ldb_module_send_entry(ac->req, msg, NULL);
 		if (ret != LDB_SUCCESS) {
 			ac->callback_failed = true;
 			return ret;
@@ -895,7 +1217,7 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 	}
 	talloc_steal(msg, dn_key);
 
-	ret = ltdb_search_dn1(module, dn_key, msg);
+	ret = ltdb_search_dn1_index(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(msg);
 		return ret;
@@ -920,7 +1242,7 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 	}
 
 	if (ret == LDB_SUCCESS) {
-		ret = ltdb_store(module, msg, TDB_REPLACE);
+		ret = ltdb_store_idxptr(module, msg, TDB_REPLACE);
 	}
 
 	talloc_free(msg);
@@ -1007,7 +1329,7 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, dn_key, msg);
+	ret = ltdb_search_dn1_index(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(dn_key);
 		return ret;
@@ -1022,9 +1344,15 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 
 	i = ldb_msg_find_idx(msg, dn, &j, LTDB_IDX);
 	if (i == -1) {
+		struct ldb_ldif ldif;
+
 		ldb_debug(ldb, LDB_DEBUG_ERROR,
 				"ERROR: dn %s not found in %s\n", dn,
 				ldb_dn_get_linearized(dn_key));
+		ldif.changetype = LDB_CHANGETYPE_NONE;
+		ldif.msg = msg;
+		ldb_ldif_write_file(module->ldb, stdout, &ldif);
+		sleep(100);
 		/* it ain't there. hmmm */
 		talloc_free(dn_key);
 		return LDB_SUCCESS;
@@ -1041,7 +1369,7 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 	if (msg->elements[i].num_values == 0) {
 		ret = ltdb_delete_noindex(module, dn_key);
 	} else {
-		ret = ltdb_store(module, msg, TDB_REPLACE);
+		ret = ltdb_store_idxptr(module, msg, TDB_REPLACE);
 	}
 
 	talloc_free(dn_key);
@@ -1250,8 +1578,8 @@ int ltdb_reindex(struct ldb_module *module)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	if (tdb_repack(ltdb->tdb) != 0) {
-		return LDB_ERR_OPERATIONS_ERROR;		
+	if (ltdb->idxptr) {
+		ltdb->idxptr->repack = true;
 	}
 
 	return LDB_SUCCESS;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_search.c b/source4/lib/ldb/ldb_tdb/ldb_search.c
index 6ab06c4e48..35149c4b77 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_search.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_search.c
@@ -418,7 +418,7 @@ static int search_func(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, voi
 		return -1;
 	}
 
-	ret = ldb_module_send_entry(ac->req, msg);
+	ret = ldb_module_send_entry(ac->req, msg, NULL);
 	if (ret != LDB_SUCCESS) {
 		ac->callback_failed = true;
 		/* the callback failed, abort the operation */
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
index 9e3ad80705..ea460def65 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -850,6 +850,8 @@ static int ltdb_start_trans(struct ldb_module *module)
 
 	ltdb->in_transaction++;
 
+	ltdb_index_transaction_start(module);
+
 	return LDB_SUCCESS;
 }
 
@@ -860,6 +862,10 @@ static int ltdb_end_trans(struct ldb_module *module)
 
 	ltdb->in_transaction--;
 
+	if (ltdb_index_transaction_commit(module) != 0) {
+		return ltdb_err_map(tdb_error(ltdb->tdb));
+	}
+
 	if (tdb_transaction_commit(ltdb->tdb) != 0) {
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
@@ -874,6 +880,10 @@ static int ltdb_del_trans(struct ldb_module *module)
 
 	ltdb->in_transaction--;
 
+	if (ltdb_index_transaction_cancel(module) != 0) {
+		return ltdb_err_map(tdb_error(ltdb->tdb));
+	}
+
 	if (tdb_transaction_cancel(ltdb->tdb) != 0) {
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
index c78a8172c7..7b9a76501f 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -33,6 +33,7 @@ struct ltdb_private {
 	int in_transaction;
 
 	bool check_base;
+	struct ltdb_idxptr *idxptr;
 };
 
 /*
@@ -57,6 +58,7 @@ struct ltdb_context {
 #define LTDB_INDEX      "@INDEX"
 #define LTDB_INDEXLIST  "@INDEXLIST"
 #define LTDB_IDX        "@IDX"
+#define LTDB_IDXPTR     "@IDXPTR"
 #define LTDB_IDXATTR    "@IDXATTR"
 #define LTDB_IDXONE     "@IDXONE"
 #define LTDB_BASEINFO   "@BASEINFO"
@@ -85,6 +87,9 @@ int ltdb_index_add(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_one(struct ldb_module *module, const struct ldb_message *msg, int add);
 int ltdb_reindex(struct ldb_module *module);
+int ltdb_index_transaction_start(struct ldb_module *module);
+int ltdb_index_transaction_commit(struct ldb_module *module);
+int ltdb_index_transaction_cancel(struct ldb_module *module);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_pack.c  */
 
diff --git a/source4/lib/ldb/modules/asq.c b/source4/lib/ldb/modules/asq.c
index 835715e7dc..c650970af4 100644
--- a/source4/lib/ldb/modules/asq.c
+++ b/source4/lib/ldb/modules/asq.c
@@ -178,7 +178,7 @@ static int asq_reqs_callback(struct ldb_request *req, struct ldb_reply *ares)
 	case LDB_REPLY_ENTRY:
 		/* pass the message up to the original callback as we
 		 * do not have to elaborate on it any further */
-		ret = ldb_module_send_entry(ac->req, ares->message);
+		ret = ldb_module_send_entry(ac->req, ares->message, ares->controls);
 		if (ret != LDB_SUCCESS) {
 			return ldb_module_done(ac->req, NULL, NULL, ret);
 		}
diff --git a/source4/lib/ldb/modules/operational.c b/source4/lib/ldb/modules/operational.c
index abb1d4ca1a..345441b5e1 100644
--- a/source4/lib/ldb/modules/operational.c
+++ b/source4/lib/ldb/modules/operational.c
@@ -206,7 +206,7 @@ static int operational_callback(struct ldb_request *req, struct ldb_reply *ares)
 			return ldb_module_done(ac->req, NULL, NULL,
 						LDB_ERR_OPERATIONS_ERROR);
 		}
-		return ldb_module_send_entry(ac->req, ares->message);
+		return ldb_module_send_entry(ac->req, ares->message, ares->controls);
 
 	case LDB_REPLY_REFERRAL:
 		/* ignore referrals */
diff --git a/source4/lib/ldb/modules/paged_results.c b/source4/lib/ldb/modules/paged_results.c
index 1a242f1be0..dfc565fef8 100644
--- a/source4/lib/ldb/modules/paged_results.c
+++ b/source4/lib/ldb/modules/paged_results.c
@@ -147,7 +147,7 @@ static int paged_results(struct paged_context *ac)
 
 	while (ac->store->num_entries > 0 && ac->size > 0) {
 		msg = ac->store->first;
-		ret = ldb_module_send_entry(ac->req, msg->r->message);
+		ret = ldb_module_send_entry(ac->req, msg->r->message, msg->r->controls);
 		if (ret != LDB_SUCCESS) {
 			return ret;
 		}
diff --git a/source4/lib/ldb/modules/paged_searches.c b/source4/lib/ldb/modules/paged_searches.c
index 7a728e3bb0..56f9b1cac3 100644
--- a/source4/lib/ldb/modules/paged_searches.c
+++ b/source4/lib/ldb/modules/paged_searches.c
@@ -161,7 +161,7 @@ static int ps_callback(struct ldb_request *req, struct ldb_reply *ares)
 
 	switch (ares->type) {
 	case LDB_REPLY_ENTRY:
-		ret = ldb_module_send_entry(ac->req, ares->message);
+		ret = ldb_module_send_entry(ac->req, ares->message, ares->controls);
 		if (ret != LDB_SUCCESS) {
 			return ldb_module_done(ac->req, NULL, NULL, ret);
 		}
diff --git a/source4/lib/ldb/modules/sort.c b/source4/lib/ldb/modules/sort.c
index 64d60f370c..25e56b24c8 100644
--- a/source4/lib/ldb/modules/sort.c
+++ b/source4/lib/ldb/modules/sort.c
@@ -152,7 +152,7 @@ static int server_sort_results(struct sort_context *ac)
 		ares->type = LDB_REPLY_ENTRY;
 		ares->message = talloc_move(ares, &ac->msgs[i]);
 
-		ret = ldb_module_send_entry(ac->req, ares->message);
+		ret = ldb_module_send_entry(ac->req, ares->message, ares->controls);
 		if (ret != LDB_SUCCESS) {
 			return ret;
 		}
diff --git a/source4/lib/ldb/tests/python/ldap.py b/source4/lib/ldb/tests/python/ldap.py
index 71fd98876e..c64d85a0e5 100755
--- a/source4/lib/ldb/tests/python/ldap.py
+++ b/source4/lib/ldb/tests/python/ldap.py
@@ -74,7 +74,11 @@ class BasicTests(unittest.TestCase):
 
         self.delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
         self.delete_force(self.ldb, "cn=ldaptestgroup,cn=users," + self.base_dn)
-
+        self.delete_force(self.ldb, "cn=ldaptestuser2,cn=users," + self.base_dn)
+        self.delete_force(self.ldb, "cn=ldaptestgroup2,cn=users," + self.base_dn)
+        self.delete_force(self.ldb, "cn=ldaptestutf8user èùéìòà ,cn=users," + self.base_dn)
+        self.delete_force(self.ldb, "cn=ldaptestutf8user2  èùéìòà ,cn=users," + self.base_dn)
+  
     def test_group_add_invalid_member(self):
         """Testing group add with invalid member"""
         try:
@@ -436,7 +440,13 @@ member: cn=ldaptestuser3,cn=users,""" + self.base_dn + """
 
         print "Testing Renames"
 
-        ldb.rename("cn=ldaptestuser2,cn=users," + self.base_dn, "cn=ldaptestuser3,cn=users," + self.base_dn)
+        attrs = ["objectGUID", "objectSid"]
+        print "Testing ldb.search for (&(cn=ldaptestUSer2)(objectClass=user))"
+        res_user = ldb.search(self.base_dn, expression="(&(cn=ldaptestUSer2)(objectClass=user))", scope=SCOPE_SUBTREE, attrs=attrs)
+        self.assertEquals(len(res_user), 1, "Could not find (&(cn=ldaptestUSer2)(objectClass=user))")
+
+        #Check rename works with extended/alternate DN forms 
+        ldb.rename("<SID=" + ldb.schema_format_value("objectSID", res_user[0]["objectSID"][0]) + ">" , "cn=ldaptestuser3,cn=users," + self.base_dn)
 
         ldb.rename("cn=ldaptestuser3,cn=users," + self.base_dn, "cn=ldaptestuser3,cn=users," + self.base_dn)
 
@@ -585,7 +595,7 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
 
         print "Testing ldb.search for (&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group)) to check subtree renames and linked attributes"
         res = ldb.search(self.base_dn, expression="(&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group))", scope=SCOPE_SUBTREE)
-        self.assertEquals(len(res), 1, "Could not find (&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group)), perhaps linked attributes are not conistant with subtree renames?")
+        self.assertEquals(len(res), 1, "Could not find (&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group)), perhaps linked attributes are not consistant with subtree renames?")
 
         print "Testing ldb.rename (into itself) of cn=ldaptestcontainer2," + self.base_dn + " to cn=ldaptestcontainer,cn=ldaptestcontainer2," + self.base_dn
         try:
@@ -737,8 +747,8 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
         self.assertEquals(len(res6), 1, "Could not find (&(cn=*daptestcomputer)(objectCategory=compuTER))")
 
         self.assertEquals(res[0].dn, res6[0].dn)
-
-        ldb.delete(res[0].dn)
+        
+        ldb.delete("<GUID=" + ldb.schema_format_value("objectGUID", res[0]["objectGUID"][0]) + ">")
 
         print "Testing ldb.search for (&(cn=ldaptest2computer)(objectClass=user))"
         res = ldb.search(expression="(&(cn=ldaptest2computer)(objectClass=user))")
@@ -754,25 +764,29 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
         self.assertEquals(int(res[0]["sAMAccountType"][0]), 805306369)
         self.assertEquals(int(res[0]["userAccountControl"][0]), 4096)
 
-        ldb.delete(res[0].dn)
+        ldb.delete("<SID=" + ldb.schema_format_value("objectSID", res[0]["objectSID"][0]) + ">")
 
-        attrs = ["cn", "name", "objectClass", "objectGUID", "whenCreated", "nTSecurityDescriptor", "memberOf", "allowedAttributes", "allowedAttributesEffective"]
+        attrs = ["cn", "name", "objectClass", "objectGUID", "objectSID", "whenCreated", "nTSecurityDescriptor", "memberOf", "allowedAttributes", "allowedAttributesEffective"]
         print "Testing ldb.search for (&(cn=ldaptestUSer2)(objectClass=user))"
-        res = ldb.search(self.base_dn, expression="(&(cn=ldaptestUSer2)(objectClass=user))", scope=SCOPE_SUBTREE, attrs=attrs)
-        self.assertEquals(len(res), 1, "Could not find (&(cn=ldaptestUSer2)(objectClass=user))")
-
-        self.assertEquals(res[0].dn, ("CN=ldaptestuser2,CN=Users," + self.base_dn))
-        self.assertEquals(res[0]["cn"], "ldaptestuser2")
-        self.assertEquals(res[0]["name"], "ldaptestuser2")
-        self.assertEquals(res[0]["objectClass"], ["top", "person", "organizationalPerson", "user"])
-        self.assertTrue("objectGUID" in res[0])
-        self.assertTrue("whenCreated" in res[0])
-        self.assertTrue("nTSecurityDescriptor" in res[0])
-        self.assertTrue("allowedAttributes" in res[0])
-        self.assertTrue("allowedAttributesEffective" in res[0])
-        self.assertEquals(res[0]["memberOf"][0].upper(), ("CN=ldaptestgroup2,CN=Users," + self.base_dn).upper())
-
-        attrs = ["cn", "name", "objectClass", "objectGUID", "whenCreated", "nTSecurityDescriptor", "member", "allowedAttributes", "allowedAttributesEffective"]
+        res_user = ldb.search(self.base_dn, expression="(&(cn=ldaptestUSer2)(objectClass=user))", scope=SCOPE_SUBTREE, attrs=attrs)
+        self.assertEquals(len(res_user), 1, "Could not find (&(cn=ldaptestUSer2)(objectClass=user))")
+
+        self.assertEquals(res_user[0].dn, ("CN=ldaptestuser2,CN=Users," + self.base_dn))
+        self.assertEquals(res_user[0]["cn"], "ldaptestuser2")
+        self.assertEquals(res_user[0]["name"], "ldaptestuser2")
+        self.assertEquals(res_user[0]["objectClass"], ["top", "person", "organizationalPerson", "user"])
+        self.assertTrue("objectSid" in res_user[0])
+        self.assertTrue("objectGUID" in res_user[0])
+        self.assertTrue("whenCreated" in res_user[0])
+        self.assertTrue("nTSecurityDescriptor" in res_user[0])
+        self.assertTrue("allowedAttributes" in res_user[0])
+        self.assertTrue("allowedAttributesEffective" in res_user[0])
+        self.assertEquals(res_user[0]["memberOf"][0].upper(), ("CN=ldaptestgroup2,CN=Users," + self.base_dn).upper())
+
+        ldaptestuser2_sid = res_user[0]["objectSid"][0]
+        ldaptestuser2_guid = res_user[0]["objectGUID"][0]
+
+        attrs = ["cn", "name", "objectClass", "objectGUID", "objectSID", "whenCreated", "nTSecurityDescriptor", "member", "allowedAttributes", "allowedAttributesEffective"]
         print "Testing ldb.search for (&(cn=ldaptestgroup2)(objectClass=group))"
         res = ldb.search(self.base_dn, expression="(&(cn=ldaptestgroup2)(objectClass=group))", scope=SCOPE_SUBTREE, attrs=attrs)
         self.assertEquals(len(res), 1, "Could not find (&(cn=ldaptestgroup2)(objectClass=group))")
@@ -781,7 +795,8 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
         self.assertEquals(res[0]["cn"], "ldaptestgroup2")
         self.assertEquals(res[0]["name"], "ldaptestgroup2")
         self.assertEquals(res[0]["objectClass"], ["top", "group"])
-        self.assertTrue("objectGuid" not in res[0])
+        self.assertTrue("objectGUID" in res[0])
+        self.assertTrue("objectSid" in res[0])
         self.assertTrue("whenCreated" in res[0])
         self.assertTrue("nTSecurityDescriptor" in res[0])
         self.assertTrue("allowedAttributes" in res[0])
@@ -791,6 +806,18 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
             memberUP.append(m.upper())
         self.assertTrue(("CN=ldaptestuser2,CN=Users," + self.base_dn).upper() in memberUP)
 
+        res = ldb.search(self.base_dn, expression="(&(cn=ldaptestgroup2)(objectClass=group))", scope=SCOPE_SUBTREE, attrs=attrs, controls=["extended_dn:1:1"])
+        self.assertEquals(len(res), 1, "Could not find (&(cn=ldaptestgroup2)(objectClass=group))")
+
+        print res[0]["member"]
+        memberUP = []
+        for m in res[0]["member"]:
+            memberUP.append(m.upper())
+        print ("<GUID=" + ldb.schema_format_value("objectGUID", ldaptestuser2_guid) + ">;<SID=" + ldb.schema_format_value("objectSid", ldaptestuser2_sid) + ">;CN=ldaptestuser2,CN=Users," + self.base_dn).upper()
+
+        self.assertTrue(("<GUID=" + ldb.schema_format_value("objectGUID", ldaptestuser2_guid) + ">;<SID=" + ldb.schema_format_value("objectSid", ldaptestuser2_sid) + ">;CN=ldaptestuser2,CN=Users," + self.base_dn).upper() in memberUP)
+
+        print "Testing Linked attribute behaviours"
         ldb.modify_ldif("""
 dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
 changetype: modify
@@ -799,9 +826,15 @@ member: CN=ldaptestuser2,CN=Users,""" + self.base_dn + """
 member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
 """)
         
-        print "Testing Linked attribute behaviours"
         ldb.modify_ldif("""
-dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
+dn: <GUID=""" + ldb.schema_format_value("objectGUID", res[0]["objectGUID"][0]) + """>
+changetype: modify
+replace: member
+member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
+""")
+        
+        ldb.modify_ldif("""
+dn: <SID=""" + ldb.schema_format_value("objectSid", res[0]["objectSid"][0]) + """>
 changetype: modify
 delete: member
 """)
@@ -810,7 +843,7 @@ delete: member
 dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
 changetype: modify
 add: member
-member: CN=ldaptestuser2,CN=Users,""" + self.base_dn + """
+member: <GUID=""" + ldb.schema_format_value("objectGUID", res[0]["objectGUID"][0]) + """>
 member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
 """)
         
@@ -824,7 +857,7 @@ replace: member
 dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
 changetype: modify
 add: member
-member: CN=ldaptestuser2,CN=Users,""" + self.base_dn + """
+member: <SID=""" + ldb.schema_format_value("objectSid", res_user[0]["objectSid"][0]) + """>
 member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
 """)
         
diff --git a/source4/lib/ldb/tools/ldbadd.c b/source4/lib/ldb/tools/ldbadd.c
index 3749aec629..f32a4fa9bc 100644
--- a/source4/lib/ldb/tools/ldbadd.c
+++ b/source4/lib/ldb/tools/ldbadd.c
@@ -93,7 +93,7 @@ int main(int argc, const char **argv)
 	options = ldb_cmdline_process(ldb, argc, argv, usage);
 
 	if (ldb_transaction_start(ldb) != 0) {
-		printf("Failed to start transaction\n");
+		printf("Failed to start transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
@@ -114,7 +114,7 @@ int main(int argc, const char **argv)
 	}
 
 	if (count != 0 && ldb_transaction_commit(ldb) != 0) {
-		printf("Failed to commit transaction\n");
+		printf("Failed to commit transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
diff --git a/source4/lib/ldb/tools/ldbedit.c b/source4/lib/ldb/tools/ldbedit.c
index b18aea1b10..1a684c5c2d 100644
--- a/source4/lib/ldb/tools/ldbedit.c
+++ b/source4/lib/ldb/tools/ldbedit.c
@@ -113,7 +113,7 @@ static int merge_edits(struct ldb_context *ldb,
 	int adds=0, modifies=0, deletes=0;
 
 	if (ldb_transaction_start(ldb) != 0) {
-		fprintf(stderr, "Failed to start transaction\n");
+		fprintf(stderr, "Failed to start transaction: %s\n", ldb_errstring(ldb));
 		return -1;
 	}
 
@@ -156,7 +156,7 @@ static int merge_edits(struct ldb_context *ldb,
 	}
 
 	if (ldb_transaction_commit(ldb) != 0) {
-		fprintf(stderr, "Failed to commit transaction\n");
+		fprintf(stderr, "Failed to commit transaction: %s\n", ldb_errstring(ldb));
 		return -1;
 	}
 
diff --git a/source4/lib/ldb/tools/ldbmodify.c b/source4/lib/ldb/tools/ldbmodify.c
index 8b6309e016..d73937cea4 100644
--- a/source4/lib/ldb/tools/ldbmodify.c
+++ b/source4/lib/ldb/tools/ldbmodify.c
@@ -91,13 +91,13 @@ int main(int argc, const char **argv)
 
 	ldb = ldb_init(NULL, NULL);
 
+	options = ldb_cmdline_process(ldb, argc, argv, usage);
+
 	if (ldb_transaction_start(ldb) != 0) {
-		printf("Failed to start transaction\n");
+		printf("Failed to start transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
-	options = ldb_cmdline_process(ldb, argc, argv, usage);
-
 	if (options->argc == 0) {
 		ret = process_file(ldb, stdin, &count);
 	} else {
@@ -114,7 +114,7 @@ int main(int argc, const char **argv)
 	}
 
 	if (count != 0 && ldb_transaction_commit(ldb) != 0) {
-		printf("Failed to commit transaction\n");
+		printf("Failed to commit transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}