From e294c4799bf6ad8da043aadf8341391644147056 Mon Sep 17 00:00:00 2001
From: Andrew Tridgell <tridge@samba.org>
Date: Tue, 16 Dec 2008 18:45:28 +1100
Subject: an experimental patch for fixing ldb bloat

ldb indexing can cause huge files, and huge memory usage. This
experiment allows us to keep indexes in memory during a transaction,
then to write the indexes to disk when the transaction completes. The
result is that the db is much smaller (we have seen improvements of
about 100x in file size) and memory usage during large transactions is
also greatly reduced

Note that this patch uses the unusual strategy of putting pointers
into a ldb (and thus into a tdb). This works because the pointers are
only there during a transaction, so the pointers are not exposed to
any other users of the database. The pointers allow us to avoid some
really bad allocation problems with tdb record allocation during the
re-indexing.
---
 source4/lib/ldb/ldb_tdb/ldb_index.c | 346 +++++++++++++++++++++++++++++++++++-
 source4/lib/ldb/ldb_tdb/ldb_tdb.c   |  10 ++
 source4/lib/ldb/ldb_tdb/ldb_tdb.h   |   5 +
 3 files changed, 352 insertions(+), 9 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c
index eedbda4170..de0e9a49d6 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_index.c
@@ -35,6 +35,328 @@
 
 #include "ldb_tdb.h"
 
+/*
+  the idxptr code is a bit unusual. The way it works is to replace
+  @IDX elements in records during a transaction with @IDXPTR
+  elements. The @IDXPTR elements don't contain the actual index entry
+  values, but contain a pointer to a linked list of values. 
+
+  This means we are storing pointers in a database, which is normally
+  not allowed, but in this case we are storing them only for the
+  duration of a transaction, and re-writing them into the normal @IDX
+  format at the end of the transaction. That means no other processes
+  are ever exposed to the @IDXPTR values.
+
+  The advantage is that the linked list doesn't cause huge
+  fragmentation during a transaction. Without the @IDXPTR method we
+  often ended up with a ldb that was between 10x and 100x larger then
+  it needs to be due to massive fragmentation caused by re-writing
+  @INDEX records many times during indexing.
+ */
+struct ldb_index_pointer {
+	struct ldb_index_pointer *next, *prev;
+	struct ldb_val value;
+};
+
+struct ltdb_idxptr {
+	int num_dns;
+	const char **dn_list;
+	bool repack;
+};
+
+/*
+  add to the list of DNs that need to be fixed on transaction end
+ */
+static int ltdb_idxptr_add(struct ldb_module *module, const struct ldb_message *msg)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	ltdb->idxptr->dn_list = talloc_realloc(ltdb->idxptr, ltdb->idxptr->dn_list, 
+					       const char *, ltdb->idxptr->num_dns+1);
+	if (ltdb->idxptr->dn_list == NULL) {
+		ltdb->idxptr->num_dns = 0;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ltdb->idxptr->dn_list[ltdb->idxptr->num_dns] = 
+		talloc_strdup(ltdb->idxptr->dn_list, ldb_dn_get_linearized(msg->dn));
+	if (ltdb->idxptr->dn_list[ltdb->idxptr->num_dns] == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ltdb->idxptr->num_dns++;
+	return LDB_SUCCESS;
+}
+
+/* free an idxptr record */
+static int ltdb_free_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_val val;
+	struct ldb_index_pointer *ptr;
+
+	if (el->num_values != 1) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	val = el->values[0];
+	if (val.length != sizeof(void *)) {
+		return LDB_ERR_OPERATIONS_ERROR;				
+	}
+			
+	ptr = *(struct ldb_index_pointer **)val.data;
+	if (talloc_get_type(ptr, struct ldb_index_pointer) != ptr) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+
+	while (ptr) {
+		struct ldb_index_pointer *tmp = ptr;
+		DLIST_REMOVE(ptr, ptr);
+		talloc_free(tmp);
+	}
+
+	return 0;
+}
+
+
+/* convert from the IDXPTR format to a ldb_message_element format */
+static int ltdb_convert_from_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_val val;
+	struct ldb_index_pointer *ptr, *tmp;
+	int i;
+	struct ldb_val *val2;
+
+	if (el->num_values != 1) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	val = el->values[0];
+	if (val.length != sizeof(void *)) {
+		return LDB_ERR_OPERATIONS_ERROR;				
+	}
+			
+	ptr = *(struct ldb_index_pointer **)val.data;
+	if (talloc_get_type(ptr, struct ldb_index_pointer) != ptr) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+
+	/* count the length of the list */
+	for (i=0, tmp = ptr; tmp; tmp=tmp->next) {
+		i++;
+	}
+
+	/* allocate the new values array */
+	val2 = talloc_realloc(NULL, el->values, struct ldb_val, i);
+	if (val2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+	el->values = val2;
+	el->num_values = i;
+
+	/* populate the values array */
+	for (i=0, tmp = ptr; tmp; tmp=tmp->next, i++) {
+		el->values[i].length = tmp->value.length;
+		/* we need to over-allocate here as there are still some places
+		   in ldb that rely on null termination. */
+		el->values[i].data = talloc_size(el->values, tmp->value.length+1);
+		if (el->values[i].data == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		memcpy(el->values[i].data, tmp->value.data, tmp->value.length);
+		el->values[i].data[tmp->value.length] = 0;
+	}	
+
+	/* update the name */
+	el->name = LTDB_IDX;
+	
+       	return LDB_SUCCESS;
+}
+
+
+/* convert to the IDXPTR format from a ldb_message_element format */
+static int ltdb_convert_to_idxptr(struct ldb_module *module, struct ldb_message_element *el)
+{
+	struct ldb_index_pointer *ptr, *tmp;
+	int i;
+	struct ldb_val *val2;
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+
+	ptr = NULL;
+
+	for (i=0;i<el->num_values;i++) {
+		tmp = talloc(ltdb->idxptr, struct ldb_index_pointer);
+		if (tmp == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;		
+		}
+		tmp->value = el->values[i];
+		tmp->value.data = talloc_memdup(tmp, tmp->value.data, tmp->value.length);
+		if (tmp->value.data == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;		
+		}
+		DLIST_ADD(ptr, tmp);
+	}
+
+	/* allocate the new values array */
+	val2 = talloc_realloc(NULL, el->values, struct ldb_val, 1);
+	if (val2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;		
+	}
+	el->values = val2;
+	el->num_values = 1;
+
+	el->values[0].data = talloc_memdup(el->values, &ptr, sizeof(ptr));
+	el->values[0].length = sizeof(ptr);
+
+	/* update the name */
+	el->name = LTDB_IDXPTR;
+
+       	return LDB_SUCCESS;	
+}
+
+
+/* enable the idxptr mode when transactions start */
+int ltdb_index_transaction_start(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	ltdb->idxptr = talloc_zero(module, struct ltdb_idxptr);
+	return 0;
+}
+
+/*
+  a wrapper around ltdb_search_dn1() which translates pointer based index records
+  and maps them into normal ldb message structures
+ */
+static int ltdb_search_dn1_wrap(struct ldb_module *module, 
+				struct ldb_dn *dn, struct ldb_message *msg)
+{
+	int ret, i;
+	ret = ltdb_search_dn1(module, dn, msg);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/* if this isn't a @INDEX record then don't munge it */
+	if (strncmp(ldb_dn_get_linearized(msg->dn), LTDB_INDEX ":", strlen(LTDB_INDEX) + 1) != 0) {
+		return ret;
+	}
+
+	for (i=0;i<msg->num_elements;i++) {
+		struct ldb_message_element *el = &msg->elements[i];
+		if (strcmp(el->name, LTDB_IDXPTR) == 0) {
+			ret = ltdb_convert_from_idxptr(module, el);
+			if (ret != LDB_SUCCESS) {
+				return ret;
+			}
+		}
+	}
+
+	return ret;
+}
+
+
+
+/*
+  fixup the idxptr for one DN
+ */
+static int ltdb_idxptr_fix_dn(struct ldb_module *module, const char *strdn)
+{
+	struct ldb_dn *dn;
+	struct ldb_message *msg = ldb_msg_new(module);
+	int ret;
+	
+	dn = ldb_dn_new(msg, module->ldb, strdn);
+	if (ltdb_search_dn1_wrap(module, dn, msg) == LDB_SUCCESS) {
+		ret = ltdb_store(module, msg, TDB_REPLACE);
+	}
+	talloc_free(msg);
+	return ret;
+}
+
+/* cleanup the idxptr mode when transaction commits */
+int ltdb_index_transaction_commit(struct ldb_module *module)
+{
+	int i;
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+
+	/* fix all the DNs that we have modified */
+	if (ltdb->idxptr) {
+		for (i=0;i<ltdb->idxptr->num_dns;i++) {
+			ltdb_idxptr_fix_dn(module, ltdb->idxptr->dn_list[i]);
+		}
+
+		if (ltdb->idxptr->repack) {
+			tdb_repack(ltdb->tdb);
+		}
+	}
+
+	talloc_free(ltdb->idxptr);
+	ltdb->idxptr = NULL;
+	return 0;
+}
+
+/* cleanup the idxptr mode when transaction cancels */
+int ltdb_index_transaction_cancel(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	talloc_free(ltdb->idxptr);
+	ltdb->idxptr = NULL;
+	return 0;
+}
+
+			
+
+/* a wrapper around ltdb_store() for the index code which 
+   stores in IDXPTR format when idxptr mode is enabled 
+
+   WARNING: This modifies the msg which is passed in
+*/
+int ltdb_store_idxptr(struct ldb_module *module, const struct ldb_message *msg, int flgs)
+{
+	struct ltdb_private *ltdb =
+		talloc_get_type(module->private_data, struct ltdb_private);
+	int ret;
+
+	if (ltdb->idxptr) {
+		int i;
+		struct ldb_message *msg2 = ldb_msg_new(module);
+
+		/* free any old pointer */
+		ret = ltdb_search_dn1(module, msg->dn, msg2);
+		if (ret == 0) {
+			for (i=0;i<msg2->num_elements;i++) {
+				struct ldb_message_element *el = &msg2->elements[i];
+				if (strcmp(el->name, LTDB_IDXPTR) == 0) {
+					ret = ltdb_free_idxptr(module, el);
+					if (ret != LDB_SUCCESS) {
+						return ret;
+					}
+				}
+			}
+		}
+		talloc_free(msg2);
+
+		for (i=0;i<msg->num_elements;i++) {
+			struct ldb_message_element *el = &msg->elements[i];
+			if (strcmp(el->name, LTDB_IDX) == 0) {
+				ret = ltdb_convert_to_idxptr(module, el);
+				if (ret != LDB_SUCCESS) {
+					return ret;
+				}
+			}
+		}
+
+		if (ltdb_idxptr_add(module, msg) != 0) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+	}
+
+	ret = ltdb_store(module, msg, flgs);
+	return ret;
+}
+
+
 /*
   find an element in a list, using the given comparison function and
   assuming that the list is already sorted using comp_fn
@@ -213,7 +535,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, dn, msg);
+	ret = ltdb_search_dn1_wrap(module, dn, msg);
 	talloc_free(dn);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -559,7 +881,7 @@ static int ltdb_index_dn_one(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, key, msg);
+	ret = ltdb_search_dn1_wrap(module, key, msg);
 	talloc_free(key);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -687,7 +1009,7 @@ static int ltdb_index_filter(const struct dn_list *dn_list,
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
 
-		ret = ltdb_search_dn1(ac->module, dn, msg);
+		ret = ltdb_search_dn1_wrap(ac->module, dn, msg);
 		talloc_free(dn);
 		if (ret == LDB_ERR_NO_SUCH_OBJECT) {
 			/* the record has disappeared? yes, this can happen */
@@ -895,7 +1217,7 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 	}
 	talloc_steal(msg, dn_key);
 
-	ret = ltdb_search_dn1(module, dn_key, msg);
+	ret = ltdb_search_dn1_wrap(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(msg);
 		return ret;
@@ -920,7 +1242,7 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 	}
 
 	if (ret == LDB_SUCCESS) {
-		ret = ltdb_store(module, msg, TDB_REPLACE);
+		ret = ltdb_store_idxptr(module, msg, TDB_REPLACE);
 	}
 
 	talloc_free(msg);
@@ -1007,7 +1329,7 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1(module, dn_key, msg);
+	ret = ltdb_search_dn1_wrap(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(dn_key);
 		return ret;
@@ -1022,9 +1344,15 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 
 	i = ldb_msg_find_idx(msg, dn, &j, LTDB_IDX);
 	if (i == -1) {
+		struct ldb_ldif ldif;
+
 		ldb_debug(ldb, LDB_DEBUG_ERROR,
 				"ERROR: dn %s not found in %s\n", dn,
 				ldb_dn_get_linearized(dn_key));
+		ldif.changetype = LDB_CHANGETYPE_NONE;
+		ldif.msg = msg;
+		ldb_ldif_write_file(module->ldb, stdout, &ldif);
+		sleep(100);
 		/* it ain't there. hmmm */
 		talloc_free(dn_key);
 		return LDB_SUCCESS;
@@ -1041,7 +1369,7 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 	if (msg->elements[i].num_values == 0) {
 		ret = ltdb_delete_noindex(module, dn_key);
 	} else {
-		ret = ltdb_store(module, msg, TDB_REPLACE);
+		ret = ltdb_store_idxptr(module, msg, TDB_REPLACE);
 	}
 
 	talloc_free(dn_key);
@@ -1250,8 +1578,8 @@ int ltdb_reindex(struct ldb_module *module)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	if (tdb_repack(ltdb->tdb) != 0) {
-		return LDB_ERR_OPERATIONS_ERROR;		
+	if (ltdb->idxptr) {
+		ltdb->idxptr->repack = true;
 	}
 
 	return LDB_SUCCESS;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
index 9e3ad80705..ea460def65 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -850,6 +850,8 @@ static int ltdb_start_trans(struct ldb_module *module)
 
 	ltdb->in_transaction++;
 
+	ltdb_index_transaction_start(module);
+
 	return LDB_SUCCESS;
 }
 
@@ -860,6 +862,10 @@ static int ltdb_end_trans(struct ldb_module *module)
 
 	ltdb->in_transaction--;
 
+	if (ltdb_index_transaction_commit(module) != 0) {
+		return ltdb_err_map(tdb_error(ltdb->tdb));
+	}
+
 	if (tdb_transaction_commit(ltdb->tdb) != 0) {
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
@@ -874,6 +880,10 @@ static int ltdb_del_trans(struct ldb_module *module)
 
 	ltdb->in_transaction--;
 
+	if (ltdb_index_transaction_cancel(module) != 0) {
+		return ltdb_err_map(tdb_error(ltdb->tdb));
+	}
+
 	if (tdb_transaction_cancel(ltdb->tdb) != 0) {
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
index c78a8172c7..7b9a76501f 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -33,6 +33,7 @@ struct ltdb_private {
 	int in_transaction;
 
 	bool check_base;
+	struct ltdb_idxptr *idxptr;
 };
 
 /*
@@ -57,6 +58,7 @@ struct ltdb_context {
 #define LTDB_INDEX      "@INDEX"
 #define LTDB_INDEXLIST  "@INDEXLIST"
 #define LTDB_IDX        "@IDX"
+#define LTDB_IDXPTR     "@IDXPTR"
 #define LTDB_IDXATTR    "@IDXATTR"
 #define LTDB_IDXONE     "@IDXONE"
 #define LTDB_BASEINFO   "@BASEINFO"
@@ -85,6 +87,9 @@ int ltdb_index_add(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_one(struct ldb_module *module, const struct ldb_message *msg, int add);
 int ltdb_reindex(struct ldb_module *module);
+int ltdb_index_transaction_start(struct ldb_module *module);
+int ltdb_index_transaction_commit(struct ldb_module *module);
+int ltdb_index_transaction_cancel(struct ldb_module *module);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_pack.c  */
 
-- 
cgit 


From aa3eab70d5f9415dca7ccc37d87e9a0ca82855c7 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Tue, 16 Dec 2008 08:19:07 +0100
Subject: s4:ldb: add infrastructure for extended dn handlers

This introduces a new set of pluggable syntax, for use on the
extended DN, and uses them when parsing the DN.

If the DN appears to be in the extended form, we no longer return the
full DN 'as is', but only return the normal part from
ldb_dn_get_linearized().

When validating/parsing the DN we validate not only the format of the
DN, but also the contents of the GUID or SID (to ensure they are
plausable).

We also have functions to set and get the extended components on the DN.

For now, extended_dn_get_linearized() returns a newly constructed and
allocated string each time.

Signed-off-by: Stefan Metzmacher <metze@samba.org>
---
 source4/lib/ldb/common/ldb_attributes.c |  47 ++++
 source4/lib/ldb/common/ldb_dn.c         | 453 ++++++++++++++++++++++++++++----
 source4/lib/ldb/include/ldb.h           |  81 +++++-
 source4/lib/ldb/include/ldb_private.h   |   3 +
 4 files changed, 524 insertions(+), 60 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/common/ldb_attributes.c b/source4/lib/ldb/common/ldb_attributes.c
index 747f241781..48f9e11caf 100644
--- a/source4/lib/ldb/common/ldb_attributes.c
+++ b/source4/lib/ldb/common/ldb_attributes.c
@@ -225,3 +225,50 @@ int ldb_setup_wellknown_attributes(struct ldb_context *ldb)
 	return LDB_SUCCESS;
 }
 
+
+/*
+  add a extended dn syntax to the ldb_schema
+*/
+int ldb_dn_extended_add_syntax(struct ldb_context *ldb, 
+			       unsigned flags,
+			       const struct ldb_dn_extended_syntax *syntax)
+{
+	int n;
+	struct ldb_dn_extended_syntax *a;
+
+	if (!syntax) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	n = ldb->schema.num_dn_extended_syntax + 1;
+
+	a = talloc_realloc(ldb, ldb->schema.dn_extended_syntax,
+			   struct ldb_dn_extended_syntax, n);
+
+	if (!a) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	a[ldb->schema.num_dn_extended_syntax] = *syntax;
+	ldb->schema.dn_extended_syntax = a;
+
+	ldb->schema.num_dn_extended_syntax = n;
+
+	return 0;
+}
+
+/*
+  return the extended dn syntax for a given name
+*/
+const struct ldb_dn_extended_syntax *ldb_dn_extended_syntax_by_name(struct ldb_context *ldb,
+								    const char *name)
+{
+	int i;
+	for (i=0; i < ldb->schema.num_dn_extended_syntax; i++) {
+		if (ldb_attr_cmp(ldb->schema.dn_extended_syntax[i].name, name) == 0) {
+			return &ldb->schema.dn_extended_syntax[i];
+		}
+	}
+	return NULL;
+}
+
diff --git a/source4/lib/ldb/common/ldb_dn.c b/source4/lib/ldb/common/ldb_dn.c
index e36aea4e69..13c01f4e70 100644
--- a/source4/lib/ldb/common/ldb_dn.c
+++ b/source4/lib/ldb/common/ldb_dn.c
@@ -52,6 +52,12 @@ struct ldb_dn_component {
 	struct ldb_val cf_value;
 };
 
+struct ldb_dn_extended_component {
+
+	char *name;
+	struct ldb_val value;
+};
+
 struct ldb_dn {
 
 	struct ldb_context *ldb;
@@ -63,11 +69,14 @@ struct ldb_dn {
 	bool valid_case;
 
 	char *linearized;
+	char *extended_linearized;
 	char *casefold;
 
 	unsigned int comp_num;
 	struct ldb_dn_component *components;
 
+	unsigned int extended_comp_num;
+	struct ldb_dn_extended_component *extended_components;
 };
 
 /* strdn may be NULL */
@@ -85,28 +94,34 @@ struct ldb_dn *ldb_dn_from_ldb_val(void *mem_ctx, struct ldb_context *ldb, const
 	if (strdn->data && strdn->length) {
 		if (strdn->data[0] == '@') {
 			dn->special = true;
+		} 
+		dn->extended_linearized = talloc_strndup(dn, (const char *)strdn->data, strdn->length);
+		LDB_DN_NULL_FAILED(dn->extended_linearized);
+	
+		if (strdn->data[0] == '<') {
+			const char *p_save, *p = dn->extended_linearized;
+			do {
+				p_save = p;
+				p = strstr(p, ">;");
+				if (p) {
+					p = p + 2;
+				}
+			} while (p);
+			
+			if (p_save == dn->extended_linearized) {
+				dn->linearized = talloc_strdup(dn, "");
+			} else {
+				dn->linearized = talloc_strdup(dn, p_save);
+			}
+			LDB_DN_NULL_FAILED(dn->linearized);
+		} else {
+			dn->linearized = dn->extended_linearized;
+			dn->extended_linearized = NULL;
 		}
-		if (strdn->length >= 6 && strncasecmp((const char *)strdn->data, "<GUID=", 6) == 0) {
-			/* this is special DN returned when the
-			 * exploded_dn control is used */
-			dn->special = true;
-			/* FIXME: add a GUID string to ldb_dn structure */
-		} else if (strdn->length >= 5 && strncasecmp((const char *)strdn->data, "<SID=", 5) == 0) {
-			/* this is special DN returned when the
-			 * exploded_dn control is used */
-			dn->special = true;
-			/* FIXME: add a SID string to ldb_dn structure */
-		} else if (strdn->length >= 8 && strncasecmp((const char *)strdn->data, "<WKGUID=", 8) == 0) {
-			/* this is special DN returned when the
-			 * exploded_dn control is used */
-			dn->special = true;
-			/* FIXME: add a WKGUID string to ldb_dn structure */
-		}
-		dn->linearized = talloc_strndup(dn, (const char *)strdn->data, strdn->length);
 	} else {
 		dn->linearized = talloc_strdup(dn, "");
+		LDB_DN_NULL_FAILED(dn->linearized);
 	}
-	LDB_DN_NULL_FAILED(dn->linearized);
 
 	return dn;
 
@@ -126,47 +141,21 @@ struct ldb_dn *ldb_dn_new(void *mem_ctx, struct ldb_context *ldb, const char *st
 
 struct ldb_dn *ldb_dn_new_fmt(void *mem_ctx, struct ldb_context *ldb, const char *new_fmt, ...)
 {
-	struct ldb_dn *dn;
 	char *strdn;
 	va_list ap;
 
 	if ( (! mem_ctx) || (! ldb)) return NULL;
 
-	dn = talloc_zero(mem_ctx, struct ldb_dn);
-	LDB_DN_NULL_FAILED(dn);
-
-	dn->ldb = ldb;
-
 	va_start(ap, new_fmt);
-	strdn = talloc_vasprintf(dn, new_fmt, ap);
+	strdn = talloc_vasprintf(mem_ctx, new_fmt, ap);
 	va_end(ap);
-	LDB_DN_NULL_FAILED(strdn);
-
-	if (strdn[0] == '@') {
-		dn->special = true;
-	}
-	if (strncasecmp(strdn, "<GUID=", 6) == 0) {
-		/* this is special DN returned when the
-		 * exploded_dn control is used */
-		dn->special = true;
-		/* FIXME: add a GUID string to ldb_dn structure */
-	} else if (strncasecmp(strdn, "<SID=", 5) == 0) {
-		/* this is special DN returned when the
-		 * exploded_dn control is used */
-		dn->special = true;
-		/* FIXME: add a SID string to ldb_dn structure */
-	} else if (strncasecmp(strdn, "<WKGUID=", 8) == 0) {
-		/* this is special DN returned when the
-		 * exploded_dn control is used */
-		dn->special = true;
-		/* FIXME: add a WKGUID string to ldb_dn structure */
-	}
-	dn->linearized = strdn;
 
-	return dn;
-
-failed:
-	talloc_free(dn);
+	if (strdn) {
+		struct ldb_dn *dn = ldb_dn_new(mem_ctx, ldb, strdn);
+		talloc_free(strdn);
+		return dn;
+	}
+	
 	return NULL;
 }
 
@@ -235,15 +224,19 @@ char *ldb_dn_escape_value(void *mem_ctx, struct ldb_val value)
 */
 static bool ldb_dn_explode(struct ldb_dn *dn)
 {
-	char *p, *data, *d, *dt, *t;
+	char *p, *ex_name, *ex_value, *data, *d, *dt, *t;
 	bool trim = false;
+	bool in_extended = false;
+	bool in_ex_name = false;
+	bool in_ex_value = false;
 	bool in_attr = false;
 	bool in_value = false;
 	bool in_quote = false;
 	bool is_oid = false;
 	bool escape = false;
 	unsigned x;
-	int l;
+	int l, ret;
+	char *parse_dn;
 
 	if ( ! dn || dn->invalid) return false;
 
@@ -251,12 +244,18 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 		return true;
 	}
 
-	if ( ! dn->linearized) {
+	if (dn->extended_linearized) {
+		parse_dn = dn->extended_linearized;
+	} else {
+		parse_dn = dn->linearized;
+	}
+
+	if ( ! parse_dn ) {
 		return false;
 	}
 
 	/* Empty DNs */
-	if (dn->linearized[0] == '\0') {
+	if (parse_dn[0] == '\0') {
 		return true;
 	}
 
@@ -268,6 +267,9 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 	/* make sure we free this if alloced previously before replacing */
 	talloc_free(dn->components);
 
+	talloc_free(dn->extended_components);
+	dn->extended_components = NULL;
+
 	/* in the common case we have 3 or more components */
 	/* make sure all components are zeroed, other functions depend on this */
 	dn->components = talloc_zero_array(dn, struct ldb_dn_component, 3);
@@ -277,19 +279,109 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 	dn->comp_num = 0;
 
 	/* Components data space is allocated here once */
-	data = talloc_array(dn->components, char, strlen(dn->linearized) + 1);
+	data = talloc_array(dn->components, char, strlen(parse_dn) + 1);
 	if (!data) {
 		return false;
 	}
 
-	p = dn->linearized;
-	in_attr = true;
+	p = parse_dn;
+	in_extended = true;
+	in_ex_name = false;
+	in_ex_value = false;
 	trim = true;
 	t = NULL;
 	d = dt = data;
 
 	while (*p) {
+		if (in_extended) {
+
+			if (!in_ex_name && !in_ex_value) {
+
+				if (p[0] == '<') {
+					p++;
+					ex_name = d;
+					in_ex_name = true;
+					continue;
+				} else if (p[0] == '\0') {
+					p++;
+					continue;
+				} else {
+					in_extended = false;
+					in_attr = true;
+					dt = d;
+
+					continue;
+				}
+			}
+			
+			if (in_ex_name && *p == '=') {
+				*d++ = '\0';
+				p++;
+				ex_value = d;
+				in_ex_name = false;
+				in_ex_value = true;
+				continue;
+			}
+
+			if (in_ex_value && *p == '>') {
+				const struct ldb_dn_extended_syntax *extended_syntax;
+				struct ldb_val ex_val = {
+					.data = ex_value,
+					.length = d - ex_value
+				};
+					
+				*d++ = '\0';
+				p++;
+				in_ex_value = false;
+
+				/* Process name and ex_value */
+
+				dn->extended_components = talloc_realloc(dn,
+									 dn->extended_components,
+									 struct ldb_dn_extended_component,
+									 dn->extended_comp_num + 1);
+				if ( ! dn->extended_components) {
+					/* ouch ! */
+					goto failed;
+				}
+
+				extended_syntax = ldb_dn_extended_syntax_by_name(dn->ldb, ex_name);
+				if (!extended_syntax) {
+					/* We don't know about this type of extended DN */
+					goto failed;
+				}
+
+				dn->extended_components[dn->extended_comp_num].name = talloc_strdup(dn->extended_components, ex_name);
+				if (!dn->extended_components[dn->extended_comp_num].name) {
+					/* ouch */
+					goto failed;
+				}
+				ret = extended_syntax->read_fn(dn->ldb, dn->extended_components,
+							       &ex_val, &dn->extended_components[dn->extended_comp_num].value);
+				if (ret != LDB_SUCCESS) {
+					dn->invalid = true;
+					goto failed;
+				}
+
+				dn->extended_comp_num++;
 
+				if (*p == '\0') {
+					/* We have reached the end (extended component only)! */
+					talloc_free(data);
+					return true;
+
+				} else if (*p == ';') {
+					p++;
+					continue;
+				} else {
+					dn->invalid = true;
+					goto failed;
+				}
+			}
+
+			*d++ = *p++;
+			continue;
+		}
 		if (in_attr) {
 			if (trim) {
 				if (*p == ' ') {
@@ -315,6 +407,7 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 					goto failed;
 				}
 				
+				/* Copy this character across from parse_dn, now we have trimmed out spaces */
 				*d++ = *p++;
 				continue;
 			}
@@ -339,6 +432,7 @@ static bool ldb_dn_explode(struct ldb_dn *dn)
 				trim = true;
 				l = 0;
 
+				/* Terminate this string in d (which is a copy of parse_dn with spaces trimmed) */
 				*d++ = '\0';
 				dn->components[dn->comp_num].name = talloc_strdup(dn->components, dt);
 				if ( ! dn->components[dn->comp_num].name) {
@@ -614,6 +708,74 @@ const char *ldb_dn_get_linearized(struct ldb_dn *dn)
 	return dn->linearized;
 }
 
+char *ldb_dn_get_extended_linearized(void *mem_ctx, struct ldb_dn *dn, int mode)
+{
+	const char *linearized = ldb_dn_get_linearized(dn);
+	char *p;
+	int i;
+
+	if (!linearized) {
+		return NULL;
+	}
+
+	if (!ldb_dn_has_extended(dn)) {
+		return talloc_strdup(mem_ctx, linearized);
+	}
+	
+	if (!ldb_dn_validate(dn)) {
+		return NULL;
+	}
+
+	for (i=0; i < dn->extended_comp_num; i++) {
+		struct ldb_val val;
+		int ret;
+		const struct ldb_dn_extended_syntax *extended_syntax;
+		const char *name = dn->extended_components[i].name;
+		
+		extended_syntax = ldb_dn_extended_syntax_by_name(dn->ldb, name);
+
+		if (mode == 1) {
+			ret = extended_syntax->write_clear_fn(dn->ldb, mem_ctx,
+							      &dn->extended_components[i].value,
+							      &val);
+		} else if (mode == 0) {
+			ret = extended_syntax->write_hex_fn(dn->ldb, mem_ctx,
+							      &dn->extended_components[i].value,
+							      &val);
+		} else {
+			ret = -1;
+		}
+
+		if (ret != LDB_SUCCESS) {
+			return NULL;
+		}
+
+		if (i == 0) {
+			p = talloc_asprintf(mem_ctx, "<%s=%s>", dn->extended_components[i].name, val.data);
+		} else {
+			p = talloc_asprintf_append(p, ";<%s=%s>",  dn->extended_components[i].name, val.data);
+		}
+
+		talloc_free(val.data);
+
+		if (!p) {
+			return NULL;
+		}
+	}
+
+	if (dn->extended_comp_num && *linearized) {
+		p = talloc_asprintf_append(p, ";%s", linearized);
+	}
+
+	if (!p) {
+		return NULL;
+	}
+
+	return p;
+}
+
+
+
 char *ldb_dn_alloc_linearized(void *mem_ctx, struct ldb_dn *dn)
 {
 	return talloc_strdup(mem_ctx, ldb_dn_get_linearized(dn));
@@ -909,6 +1071,30 @@ static struct ldb_dn_component ldb_dn_copy_component(void *mem_ctx, struct ldb_d
 	return dst;
 }
 
+static struct ldb_dn_extended_component ldb_dn_extended_copy_component(void *mem_ctx, struct ldb_dn_extended_component *src)
+{
+	struct ldb_dn_extended_component dst;
+
+	memset(&dst, 0, sizeof(dst));
+
+	if (src == NULL) {
+		return dst;
+	}
+
+	dst.value = ldb_val_dup(mem_ctx, &(src->value));
+	if (dst.value.data == NULL) {
+		return dst;
+	}
+
+	dst.name = talloc_strdup(mem_ctx, src->name);
+	if (dst.name == NULL) {
+		LDB_FREE(dst.value.data);
+		return dst;
+	}
+
+	return dst;
+}
+
 struct ldb_dn *ldb_dn_copy(void *mem_ctx, struct ldb_dn *dn)
 {
 	struct ldb_dn *new_dn;
@@ -942,6 +1128,24 @@ struct ldb_dn *ldb_dn_copy(void *mem_ctx, struct ldb_dn *dn)
 		}
 	}
 
+	if (dn->extended_components) {
+		int i;
+
+		new_dn->extended_components = talloc_zero_array(new_dn, struct ldb_dn_extended_component, dn->extended_comp_num);
+		if ( ! new_dn->extended_components) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+
+		for (i = 0; i < dn->extended_comp_num; i++) {
+			new_dn->extended_components[i] = ldb_dn_extended_copy_component(new_dn->extended_components, &dn->extended_components[i]);
+			if ( ! new_dn->extended_components[i].value.data) {
+				talloc_free(new_dn);
+				return NULL;
+			}
+		}
+	}
+
 	if (dn->casefold) {
 		new_dn->casefold = talloc_strdup(new_dn, dn->casefold);
 		if ( ! new_dn->casefold) {
@@ -958,6 +1162,14 @@ struct ldb_dn *ldb_dn_copy(void *mem_ctx, struct ldb_dn *dn)
 		}
 	}
 
+	if (dn->extended_linearized) {
+		new_dn->extended_linearized = talloc_strdup(new_dn, dn->extended_linearized);
+		if ( ! new_dn->extended_linearized) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+	}
+
 	return new_dn;
 }
 
@@ -1037,6 +1249,13 @@ bool ldb_dn_add_base(struct ldb_dn *dn, struct ldb_dn *base)
 		dn->linearized = t;
 	}
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	if (dn->extended_linearized) {
+		LDB_FREE(dn->extended_linearized);
+	}
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
 	return true;
 }
 
@@ -1149,6 +1368,12 @@ bool ldb_dn_add_child(struct ldb_dn *dn, struct ldb_dn *child)
 		dn->linearized = t;
 	}
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
+
 	return true;
 }
 
@@ -1218,6 +1443,12 @@ bool ldb_dn_remove_base_components(struct ldb_dn *dn, unsigned int num)
 	LDB_FREE(dn->casefold);
 	LDB_FREE(dn->linearized);
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
+
 	return true;
 }
 
@@ -1256,6 +1487,11 @@ bool ldb_dn_remove_child_components(struct ldb_dn *dn, unsigned int num)
 	LDB_FREE(dn->casefold);
 	LDB_FREE(dn->linearized);
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
 	return true;
 }
 
@@ -1273,6 +1509,11 @@ struct ldb_dn *ldb_dn_get_parent(void *mem_ctx, struct ldb_dn *dn)
 		return NULL;
 	}
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	LDB_FREE(dn->extended_components);
+	dn->extended_comp_num = 0;
 	return new_dn;
 }
 
@@ -1434,9 +1675,97 @@ int ldb_dn_set_component(struct ldb_dn *dn, int num, const char *name, const str
 	LDB_FREE(dn->casefold);
 	LDB_FREE(dn->linearized);
 
+	/* Wipe the extended_linearized DN, as the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->extended_linearized);
+
+	dn->extended_comp_num = 0;
+	LDB_FREE(dn->extended_components);
 	return LDB_SUCCESS;
 }
 
+const struct ldb_val *ldb_dn_get_extended_component(struct ldb_dn *dn, const char *name)
+{
+	int i;
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+	for (i=0; i < dn->extended_comp_num; i++) {
+		if (ldb_attr_cmp(dn->extended_components[i].name, name) == 0) {
+			return &dn->extended_components[i].value;
+		}
+	}
+	return NULL;
+}
+
+int ldb_dn_set_extended_component(struct ldb_dn *dn, const char *name, const struct ldb_val *val)
+{
+	struct ldb_dn_extended_component *p;
+	int i;
+				
+	if ( ! ldb_dn_validate(dn)) {
+		return LDB_ERR_OTHER;
+	}
+
+	for (i=0; i < dn->extended_comp_num; i++) {
+		if (ldb_attr_cmp(dn->extended_components[i].name, name) == 0) {
+			if (val) {
+				dn->extended_components[i].value = ldb_val_dup(dn->extended_components, val);
+
+				dn->extended_components[i].name = talloc_strdup(dn->extended_components, name);
+				if (!dn->extended_components[i].name || !dn->extended_components[i].value.data) {
+					dn->invalid = true;
+					return LDB_ERR_OPERATIONS_ERROR;
+				}
+	
+			} else {
+				if (i != (dn->extended_comp_num - 1)) {
+					memmove(&dn->extended_components[i], &dn->extended_components[i+1],
+						((dn->extended_comp_num-1) - i)*sizeof(*dn->extended_components));
+				}
+				dn->extended_comp_num--;
+				
+				dn->extended_components = talloc_realloc(dn,
+						   dn->extended_components,
+						   struct ldb_dn_extended_component,
+						   dn->extended_comp_num);
+				if (!dn->extended_components) {
+					dn->invalid = true;
+					return LDB_ERR_OPERATIONS_ERROR;
+				}
+				return 0;
+			}
+		}
+	}
+
+	p = dn->extended_components
+		= talloc_realloc(dn,
+				 dn->extended_components,
+				 struct ldb_dn_extended_component,
+				 dn->extended_comp_num + 1);
+	if (!dn->extended_components) {
+		dn->invalid = true;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	
+	p[dn->extended_comp_num].value = ldb_val_dup(dn->extended_components, val);
+	p[dn->extended_comp_num].name = talloc_strdup(p, name);
+	
+	if (!dn->extended_components[i].name || !dn->extended_components[i].value.data) {
+		dn->invalid = true;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	dn->extended_components = p;
+	dn->extended_comp_num++;
+	
+	return 0;
+}
+
+void ldb_dn_remove_extended_components(struct ldb_dn *dn)
+{
+	dn->extended_comp_num = 0;
+	LDB_FREE(dn->extended_components);	
+}
+
 bool ldb_dn_is_valid(struct ldb_dn *dn)
 {
 	if ( ! dn) return false;
@@ -1449,6 +1778,13 @@ bool ldb_dn_is_special(struct ldb_dn *dn)
 	return dn->special;
 }
 
+bool ldb_dn_has_extended(struct ldb_dn *dn)
+{
+	if ( ! dn || dn->invalid) return false;
+	if (dn->extended_linearized && (dn->extended_linearized[0] == '<')) return true;
+	return dn->extended_comp_num != 0;
+}
+
 bool ldb_dn_check_special(struct ldb_dn *dn, const char *check)
 {
 	if ( ! dn || dn->invalid) return false;
@@ -1458,6 +1794,7 @@ bool ldb_dn_check_special(struct ldb_dn *dn, const char *check)
 bool ldb_dn_is_null(struct ldb_dn *dn)
 {
 	if ( ! dn || dn->invalid) return false;
+	if (ldb_dn_has_extended(dn)) return false;
 	if (dn->linearized && (dn->linearized[0] == '\0')) return true;
 	return false;
 }
diff --git a/source4/lib/ldb/include/ldb.h b/source4/lib/ldb/include/ldb.h
index 9bc5c183d8..cd7eb06b4d 100644
--- a/source4/lib/ldb/include/ldb.h
+++ b/source4/lib/ldb/include/ldb.h
@@ -350,6 +350,16 @@ struct ldb_schema_attribute {
 const struct ldb_schema_attribute *ldb_schema_attribute_by_name(struct ldb_context *ldb,
 								const char *name);
 
+struct ldb_dn_extended_syntax {
+	const char *name;
+	ldb_attr_handler_t read_fn;
+	ldb_attr_handler_t write_clear_fn;
+	ldb_attr_handler_t write_hex_fn;
+};
+
+const struct ldb_dn_extended_syntax *ldb_dn_extended_syntax_by_name(struct ldb_context *ldb,
+								    const char *name);
+
 /**
    The attribute is not returned by default
 */
@@ -1421,15 +1431,82 @@ int ldb_base64_decode(char *s);
 
 /* The following definitions come from lib/ldb/common/ldb_dn.c  */
 
+/**
+  Get the linear form of a DN (without any extended components)
+  
+  \param dn The DN to linearize
+*/
+
+const char *ldb_dn_get_linearized(struct ldb_dn *dn);
+
+/**
+  Allocate a copy of the linear form of a DN (without any extended components) onto the supplied memory context 
+  
+  \param dn The DN to linearize
+  \param mem_ctx TALLOC context to return result on
+*/
+
+char *ldb_dn_alloc_linearized(TALLOC_CTX *mem_ctx, struct ldb_dn *dn);
+
+/**
+  Get the linear form of a DN (with any extended components)
+  
+  \param mem_ctx TALLOC context to return result on
+  \param dn The DN to linearize
+  \param mode Style of extended DN to return (0 is HEX representation of binary form, 1 is a string form)
+*/
+char *ldb_dn_get_extended_linearized(void *mem_ctx, struct ldb_dn *dn, int mode);
+const struct ldb_val *ldb_dn_get_extended_component(struct ldb_dn *dn, const char *name);
+int ldb_dn_set_extended_component(struct ldb_dn *dn, const char *name, const struct ldb_val *val);
+
+void ldb_dn_remove_extended_components(struct ldb_dn *dn);
+bool ldb_dn_has_extended(struct ldb_dn *dn);
+
+int ldb_dn_extended_add_syntax(struct ldb_context *ldb, 
+			       unsigned flags,
+			       const struct ldb_dn_extended_syntax *syntax);
+
+/** 
+  Allocate a new DN from a string
+
+  \param mem_ctx TALLOC context to return resulting ldb_dn structure on
+  \param dn The new DN 
+
+  \note The DN will not be parsed at this time.  Use ldb_dn_validate to tell if the DN is syntacticly correct
+*/
+
 struct ldb_dn *ldb_dn_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, const char *dn);
+/** 
+  Allocate a new DN from a printf style format string and arguments
+
+  \param mem_ctx TALLOC context to return resulting ldb_dn structure on
+  \param new_fms The new DN as a format string (plus arguments)
+
+  \note The DN will not be parsed at this time.  Use ldb_dn_validate to tell if the DN is syntacticly correct
+*/
+
 struct ldb_dn *ldb_dn_new_fmt(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, const char *new_fmt, ...) PRINTF_ATTRIBUTE(3,4);
+/** 
+  Allocate a new DN from a struct ldb_val (useful to avoid buffer overrun)
+
+  \param mem_ctx TALLOC context to return resulting ldb_dn structure on
+  \param dn The new DN 
+
+  \note The DN will not be parsed at this time.  Use ldb_dn_validate to tell if the DN is syntacticly correct
+*/
+
 struct ldb_dn *ldb_dn_from_ldb_val(void *mem_ctx, struct ldb_context *ldb, const struct ldb_val *strdn);
+
+/**
+  Determine if this DN is syntactically valid 
+
+  \param dn The DN to validate
+*/
+
 bool ldb_dn_validate(struct ldb_dn *dn);
 
 char *ldb_dn_escape_value(TALLOC_CTX *mem_ctx, struct ldb_val value);
-const char *ldb_dn_get_linearized(struct ldb_dn *dn);
 const char *ldb_dn_get_casefold(struct ldb_dn *dn);
-char *ldb_dn_alloc_linearized(TALLOC_CTX *mem_ctx, struct ldb_dn *dn);
 char *ldb_dn_alloc_casefold(TALLOC_CTX *mem_ctx, struct ldb_dn *dn);
 
 int ldb_dn_compare_base(struct ldb_dn *base, struct ldb_dn *dn);
diff --git a/source4/lib/ldb/include/ldb_private.h b/source4/lib/ldb/include/ldb_private.h
index 90c4980017..d1d3587eb7 100644
--- a/source4/lib/ldb/include/ldb_private.h
+++ b/source4/lib/ldb/include/ldb_private.h
@@ -84,6 +84,9 @@ struct ldb_schema {
 	/* attribute handling table */
 	unsigned num_attributes;
 	struct ldb_schema_attribute *attributes;
+
+	unsigned num_dn_extended_syntax;
+	struct ldb_dn_extended_syntax *dn_extended_syntax;
 };
 
 /*
-- 
cgit 


From 7a7573d57016b466eef7a48bc189289744ca4c7c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Tue, 16 Dec 2008 08:25:44 +0100
Subject: s4:ldb_ildap: try to pass extended DNs to the server

Whenever we pass a DN to the LDAP server, we now use
ldb_dn_get_extended_linearized().  This allows us to send the extended
DN if set, and therefore allows searches of the form
'<GUID=aaa45ea0-94cd-45e9-8753-abe455d9a8f1>'.

We actually use the '0' format (GUID=aaa45ea094cd45e98753abe455d9a8f1)
because it is more widely supported (by Win2k in particular).

Signed-off-by: Stefan Metzmacher <metze@samba.org>
---
 source4/lib/ldb/ldb_ildap/ldb_ildap.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/ldb_ildap/ldb_ildap.c b/source4/lib/ldb/ldb_ildap/ldb_ildap.c
index 93e81d4244..ae79bdfa48 100644
--- a/source4/lib/ldb/ldb_ildap/ldb_ildap.c
+++ b/source4/lib/ldb/ldb_ildap/ldb_ildap.c
@@ -428,7 +428,7 @@ static int ildb_search(struct ildb_context *ac)
 	if (req->op.search.base == NULL) {
 		msg->r.SearchRequest.basedn = talloc_strdup(msg, "");
 	} else {
-		msg->r.SearchRequest.basedn  = ldb_dn_alloc_linearized(msg, req->op.search.base);
+		msg->r.SearchRequest.basedn  = ldb_dn_get_extended_linearized(msg, req->op.search.base, 0);
 	}
 	if (msg->r.SearchRequest.basedn == NULL) {
 		ldb_set_errstring(ac->module->ldb, "Unable to determine baseDN");
@@ -473,7 +473,7 @@ static int ildb_add(struct ildb_context *ac)
 
 	msg->type = LDAP_TAG_AddRequest;
 
-	msg->r.AddRequest.dn = ldb_dn_alloc_linearized(msg, req->op.add.message->dn);
+	msg->r.AddRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.add.message->dn, 0);
 	if (msg->r.AddRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
@@ -516,7 +516,7 @@ static int ildb_modify(struct ildb_context *ac)
 
 	msg->type = LDAP_TAG_ModifyRequest;
 
-	msg->r.ModifyRequest.dn = ldb_dn_alloc_linearized(msg, req->op.mod.message->dn);
+	msg->r.ModifyRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.mod.message->dn, 0);
 	if (msg->r.ModifyRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
@@ -557,7 +557,7 @@ static int ildb_delete(struct ildb_context *ac)
 
 	msg->type = LDAP_TAG_DelRequest;
 
-	msg->r.DelRequest.dn = ldb_dn_alloc_linearized(msg, req->op.del.dn);
+	msg->r.DelRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.del.dn, 0);
 	if (msg->r.DelRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
@@ -580,7 +580,7 @@ static int ildb_rename(struct ildb_context *ac)
 	}
 
 	msg->type = LDAP_TAG_ModifyDNRequest;
-	msg->r.ModifyDNRequest.dn = ldb_dn_alloc_linearized(msg, req->op.rename.olddn);
+	msg->r.ModifyDNRequest.dn = ldb_dn_get_extended_linearized(msg, req->op.rename.olddn, 0);
 	if (msg->r.ModifyDNRequest.dn == NULL) {
 		talloc_free(msg);
 		return LDB_ERR_INVALID_DN_SYNTAX;
-- 
cgit 


From f7c53e0addb13dd085ed102b8afaec2122e078ea Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Tue, 16 Dec 2008 08:33:32 +0100
Subject: s4:ldb: use try to print the extended dn in the ldif output

This allows searches with the extended DN control to still print the
extended DN in ldif output (it would otherwise be parsed and hidden in
the structure).

Signed-off-by: Stefan Metzmacher <metze@samba.org>
---
 source4/lib/ldb/common/ldb_ldif.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/common/ldb_ldif.c b/source4/lib/ldb/common/ldb_ldif.c
index 69490e670b..619c10e11e 100644
--- a/source4/lib/ldb/common/ldb_ldif.c
+++ b/source4/lib/ldb/common/ldb_ldif.c
@@ -278,13 +278,15 @@ int ldb_ldif_write(struct ldb_context *ldb,
 	TALLOC_CTX *mem_ctx;
 	unsigned int i, j;
 	int total=0, ret;
+	char *p;
 	const struct ldb_message *msg;
 
 	mem_ctx = talloc_named_const(NULL, 0, "ldb_ldif_write");
 
 	msg = ldif->msg;
-
-	ret = fprintf_fn(private_data, "dn: %s\n", ldb_dn_get_linearized(msg->dn));
+	p = ldb_dn_get_extended_linearized(mem_ctx, msg->dn, 1);
+	ret = fprintf_fn(private_data, "dn: %s\n", p);
+	talloc_free(p);
 	CHECK_RET;
 
 	if (ldif->changetype != LDB_CHANGETYPE_NONE) {
-- 
cgit 


From ced158d25066a236fba36c0e8391de1eab301a17 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Tue, 16 Dec 2008 08:34:48 +0100
Subject: s4:ldb.i: hang the dn on the NULL context as the python destructor
 will free it

This fixes a bug in the ldb.i python wrapper, that showed up under valgrind.

Signed-off-by: Stefan Metzmacher <metze@samba.org>
---
 source4/lib/ldb/ldb.i | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/ldb.i b/source4/lib/ldb/ldb.i
index 0f05c1fbab..7831d6da60 100644
--- a/source4/lib/ldb/ldb.i
+++ b/source4/lib/ldb/ldb.i
@@ -575,6 +575,8 @@ static void py_ldb_debug(void *context, enum ldb_debug_level level, const char *
         if (ldif == NULL) {
             return Py_None;
         } else {
+	    /* We don't want this attached to the 'ldb' any more */
+            talloc_steal(NULL, ldif);
             return Py_BuildValue((char *)"(iO)", ldif->changetype, 
                    SWIG_NewPointerObj(ldif->msg, SWIGTYPE_p_ldb_message, 0));
         }
-- 
cgit 


From 596fe759e1fed835173146a74ac9986066acc48e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Tue, 16 Dec 2008 08:59:05 +0100
Subject: s4:ldb: make it possible to return per entry controls

Signed-off-by: Stefan Metzmacher <metze@samba.org>
---
 source4/lib/ldb/common/ldb_controls.c      | 20 ++++++++++++++++++++
 source4/lib/ldb/common/ldb_modules.c       |  6 +++++-
 source4/lib/ldb/include/ldb.h              | 10 +++++++++-
 source4/lib/ldb/include/ldb_private.h      |  3 ++-
 source4/lib/ldb/ldb_ildap/ldb_ildap.c      |  4 +++-
 source4/lib/ldb/ldb_ldap/ldb_ldap.c        |  2 +-
 source4/lib/ldb/ldb_map/ldb_map_outbound.c |  2 +-
 source4/lib/ldb/ldb_tdb/ldb_index.c        |  2 +-
 source4/lib/ldb/ldb_tdb/ldb_search.c       |  2 +-
 source4/lib/ldb/modules/asq.c              |  2 +-
 source4/lib/ldb/modules/operational.c      |  2 +-
 source4/lib/ldb/modules/paged_results.c    |  2 +-
 source4/lib/ldb/modules/paged_searches.c   |  2 +-
 source4/lib/ldb/modules/sort.c             |  2 +-
 14 files changed, 48 insertions(+), 13 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/common/ldb_controls.c b/source4/lib/ldb/common/ldb_controls.c
index e3f8551407..6fad5012b6 100644
--- a/source4/lib/ldb/common/ldb_controls.c
+++ b/source4/lib/ldb/common/ldb_controls.c
@@ -53,6 +53,26 @@ struct ldb_control *ldb_request_get_control(struct ldb_request *req, const char
 	return NULL;
 }
 
+/* check if a control with the specified "oid" exist and return it */
+/* returns NULL if not found */
+struct ldb_control *ldb_reply_get_control(struct ldb_reply *rep, const char *oid)
+{
+	int i;
+
+	/* check if there's a paged request control */
+	if (rep->controls != NULL) {
+		for (i = 0; rep->controls[i]; i++) {
+			if (strcmp(oid, rep->controls[i]->oid) == 0) {
+				break;
+			}
+		}
+
+		return rep->controls[i];
+	}
+
+	return NULL;
+}
+
 /* saves the current controls list into the "saver" and replace the one in req with a new one excluding
 the "exclude" control */
 /* returns False on error */
diff --git a/source4/lib/ldb/common/ldb_modules.c b/source4/lib/ldb/common/ldb_modules.c
index ab0f4c51cc..8db28d262c 100644
--- a/source4/lib/ldb/common/ldb_modules.c
+++ b/source4/lib/ldb/common/ldb_modules.c
@@ -587,10 +587,13 @@ struct ldb_handle *ldb_handle_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb)
  *      req: the original request passed to your module
  *      msg: reply message (must be a talloc pointer, and it will be stolen
  *           on the ldb_reply that is sent to the callback)
+ * 	ctrls: controls to send in the reply  (must be a talloc pointer, and it will be stolen
+ *           on the ldb_reply that is sent to the callback)
  */
 
 int ldb_module_send_entry(struct ldb_request *req,
-			  struct ldb_message *msg)
+			  struct ldb_message *msg,
+			  struct ldb_control **ctrls)
 {
 	struct ldb_reply *ares;
 
@@ -602,6 +605,7 @@ int ldb_module_send_entry(struct ldb_request *req,
 	}
 	ares->type = LDB_REPLY_ENTRY;
 	ares->message = talloc_steal(ares, msg);
+	ares->controls = talloc_steal(ares, ctrls);
 	ares->error = LDB_SUCCESS;
 
 	return req->callback(req, ares);
diff --git a/source4/lib/ldb/include/ldb.h b/source4/lib/ldb/include/ldb.h
index cd7eb06b4d..e2ec869872 100644
--- a/source4/lib/ldb/include/ldb.h
+++ b/source4/lib/ldb/include/ldb.h
@@ -326,7 +326,6 @@ typedef int (*ldb_attr_comparison_t)(struct ldb_context *, TALLOC_CTX *mem_ctx,
   attribute handler structure
 
   attr			-> The attribute name
-  flags			-> LDB_ATTR_FLAG_*
   ldif_read_fn		-> convert from ldif to binary format
   ldif_write_fn		-> convert from binary to ldif format
   canonicalise_fn	-> canonicalise a value, for use by indexing and dn construction
@@ -1075,6 +1074,15 @@ int ldb_request_add_control(struct ldb_request *req, const char *oid, bool criti
 */
 struct ldb_control *ldb_request_get_control(struct ldb_request *req, const char *oid);
 
+/**
+   check if a control with the specified "oid" exist and return it 
+  \param rep the reply struct where to add the control
+  \param oid the object identifier of the control as string
+
+  \return the control, NULL if not found 
+*/
+struct ldb_control *ldb_reply_get_control(struct ldb_reply *rep, const char *oid);
+
 /**
   Search the database
 
diff --git a/source4/lib/ldb/include/ldb_private.h b/source4/lib/ldb/include/ldb_private.h
index d1d3587eb7..c065288279 100644
--- a/source4/lib/ldb/include/ldb_private.h
+++ b/source4/lib/ldb/include/ldb_private.h
@@ -265,7 +265,8 @@ int ldb_register_backend(const char *url_prefix, ldb_connect_fn);
 struct ldb_handle *ldb_handle_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb);
 
 int ldb_module_send_entry(struct ldb_request *req,
-			  struct ldb_message *msg);
+			  struct ldb_message *msg,
+			  struct ldb_control **ctrls);
 
 int ldb_module_send_referral(struct ldb_request *req,
 					   char *ref);
diff --git a/source4/lib/ldb/ldb_ildap/ldb_ildap.c b/source4/lib/ldb/ldb_ildap/ldb_ildap.c
index ae79bdfa48..b17d063c0c 100644
--- a/source4/lib/ldb/ldb_ildap/ldb_ildap.c
+++ b/source4/lib/ldb/ldb_ildap/ldb_ildap.c
@@ -311,7 +311,9 @@ static void ildb_callback(struct ldap_request *req)
 				ldbmsg->num_elements = search->num_attributes;
 				ldbmsg->elements = talloc_move(ldbmsg, &search->attributes);
 
-				ret = ldb_module_send_entry(ac->req, ldbmsg);
+				controls = talloc_steal(ac, msg->controls);
+				
+				ret = ldb_module_send_entry(ac->req, ldbmsg, controls);
 				if (ret != LDB_SUCCESS) {
 					callback_failed = true;
 				}
diff --git a/source4/lib/ldb/ldb_ldap/ldb_ldap.c b/source4/lib/ldb/ldb_ldap/ldb_ldap.c
index 7caee10b47..9f8b3e9f35 100644
--- a/source4/lib/ldb/ldb_ldap/ldb_ldap.c
+++ b/source4/lib/ldb/ldb_ldap/ldb_ldap.c
@@ -515,7 +515,7 @@ static bool lldb_parse_result(struct lldb_context *ac, LDAPMessage *result)
 			}
 			if (berptr) ber_free(berptr, 0);
 
-			ret = ldb_module_send_entry(ac->req, ldbmsg);
+			ret = ldb_module_send_entry(ac->req, ldbmsg, NULL /* controls not yet supported */);
 			if (ret != LDB_SUCCESS) {
 
 				callback_failed = true;
diff --git a/source4/lib/ldb/ldb_map/ldb_map_outbound.c b/source4/lib/ldb/ldb_map/ldb_map_outbound.c
index 5f524a8be3..5588eaaf46 100644
--- a/source4/lib/ldb/ldb_map/ldb_map_outbound.c
+++ b/source4/lib/ldb/ldb_map/ldb_map_outbound.c
@@ -1077,7 +1077,7 @@ int map_return_entry(struct map_context *ac, struct ldb_reply *ares)
 		}
 	}
 
-	return ldb_module_send_entry(ac->req, ares->message);
+	return ldb_module_send_entry(ac->req, ares->message, ares->controls);
 }
 
 /* Search a record. */
diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c
index de0e9a49d6..d0b9fa27ce 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_index.c
@@ -1037,7 +1037,7 @@ static int ltdb_index_filter(const struct dn_list *dn_list,
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
 
-		ret = ldb_module_send_entry(ac->req, msg);
+		ret = ldb_module_send_entry(ac->req, msg, NULL);
 		if (ret != LDB_SUCCESS) {
 			ac->callback_failed = true;
 			return ret;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_search.c b/source4/lib/ldb/ldb_tdb/ldb_search.c
index 6ab06c4e48..35149c4b77 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_search.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_search.c
@@ -418,7 +418,7 @@ static int search_func(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, voi
 		return -1;
 	}
 
-	ret = ldb_module_send_entry(ac->req, msg);
+	ret = ldb_module_send_entry(ac->req, msg, NULL);
 	if (ret != LDB_SUCCESS) {
 		ac->callback_failed = true;
 		/* the callback failed, abort the operation */
diff --git a/source4/lib/ldb/modules/asq.c b/source4/lib/ldb/modules/asq.c
index 835715e7dc..c650970af4 100644
--- a/source4/lib/ldb/modules/asq.c
+++ b/source4/lib/ldb/modules/asq.c
@@ -178,7 +178,7 @@ static int asq_reqs_callback(struct ldb_request *req, struct ldb_reply *ares)
 	case LDB_REPLY_ENTRY:
 		/* pass the message up to the original callback as we
 		 * do not have to elaborate on it any further */
-		ret = ldb_module_send_entry(ac->req, ares->message);
+		ret = ldb_module_send_entry(ac->req, ares->message, ares->controls);
 		if (ret != LDB_SUCCESS) {
 			return ldb_module_done(ac->req, NULL, NULL, ret);
 		}
diff --git a/source4/lib/ldb/modules/operational.c b/source4/lib/ldb/modules/operational.c
index abb1d4ca1a..345441b5e1 100644
--- a/source4/lib/ldb/modules/operational.c
+++ b/source4/lib/ldb/modules/operational.c
@@ -206,7 +206,7 @@ static int operational_callback(struct ldb_request *req, struct ldb_reply *ares)
 			return ldb_module_done(ac->req, NULL, NULL,
 						LDB_ERR_OPERATIONS_ERROR);
 		}
-		return ldb_module_send_entry(ac->req, ares->message);
+		return ldb_module_send_entry(ac->req, ares->message, ares->controls);
 
 	case LDB_REPLY_REFERRAL:
 		/* ignore referrals */
diff --git a/source4/lib/ldb/modules/paged_results.c b/source4/lib/ldb/modules/paged_results.c
index 1a242f1be0..dfc565fef8 100644
--- a/source4/lib/ldb/modules/paged_results.c
+++ b/source4/lib/ldb/modules/paged_results.c
@@ -147,7 +147,7 @@ static int paged_results(struct paged_context *ac)
 
 	while (ac->store->num_entries > 0 && ac->size > 0) {
 		msg = ac->store->first;
-		ret = ldb_module_send_entry(ac->req, msg->r->message);
+		ret = ldb_module_send_entry(ac->req, msg->r->message, msg->r->controls);
 		if (ret != LDB_SUCCESS) {
 			return ret;
 		}
diff --git a/source4/lib/ldb/modules/paged_searches.c b/source4/lib/ldb/modules/paged_searches.c
index 7a728e3bb0..56f9b1cac3 100644
--- a/source4/lib/ldb/modules/paged_searches.c
+++ b/source4/lib/ldb/modules/paged_searches.c
@@ -161,7 +161,7 @@ static int ps_callback(struct ldb_request *req, struct ldb_reply *ares)
 
 	switch (ares->type) {
 	case LDB_REPLY_ENTRY:
-		ret = ldb_module_send_entry(ac->req, ares->message);
+		ret = ldb_module_send_entry(ac->req, ares->message, ares->controls);
 		if (ret != LDB_SUCCESS) {
 			return ldb_module_done(ac->req, NULL, NULL, ret);
 		}
diff --git a/source4/lib/ldb/modules/sort.c b/source4/lib/ldb/modules/sort.c
index 64d60f370c..25e56b24c8 100644
--- a/source4/lib/ldb/modules/sort.c
+++ b/source4/lib/ldb/modules/sort.c
@@ -152,7 +152,7 @@ static int server_sort_results(struct sort_context *ac)
 		ares->type = LDB_REPLY_ENTRY;
 		ares->message = talloc_move(ares, &ac->msgs[i]);
 
-		ret = ldb_module_send_entry(ac->req, ares->message);
+		ret = ldb_module_send_entry(ac->req, ares->message, ares->controls);
 		if (ret != LDB_SUCCESS) {
 			return ret;
 		}
-- 
cgit 


From eb3714acfad21be9de0a2424f1f4943e7ff5f0b7 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Tue, 16 Dec 2008 09:11:11 +0100
Subject: s4:ldb: add some python tests for extended dns

Signed-off-by: Stefan Metzmacher <metze@samba.org>
---
 source4/lib/ldb/tests/python/ldap.py | 87 +++++++++++++++++++++++++-----------
 1 file changed, 60 insertions(+), 27 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/tests/python/ldap.py b/source4/lib/ldb/tests/python/ldap.py
index 71fd98876e..c64d85a0e5 100755
--- a/source4/lib/ldb/tests/python/ldap.py
+++ b/source4/lib/ldb/tests/python/ldap.py
@@ -74,7 +74,11 @@ class BasicTests(unittest.TestCase):
 
         self.delete_force(self.ldb, "cn=ldaptestuser,cn=users," + self.base_dn)
         self.delete_force(self.ldb, "cn=ldaptestgroup,cn=users," + self.base_dn)
-
+        self.delete_force(self.ldb, "cn=ldaptestuser2,cn=users," + self.base_dn)
+        self.delete_force(self.ldb, "cn=ldaptestgroup2,cn=users," + self.base_dn)
+        self.delete_force(self.ldb, "cn=ldaptestutf8user èùéìòà ,cn=users," + self.base_dn)
+        self.delete_force(self.ldb, "cn=ldaptestutf8user2  èùéìòà ,cn=users," + self.base_dn)
+  
     def test_group_add_invalid_member(self):
         """Testing group add with invalid member"""
         try:
@@ -436,7 +440,13 @@ member: cn=ldaptestuser3,cn=users,""" + self.base_dn + """
 
         print "Testing Renames"
 
-        ldb.rename("cn=ldaptestuser2,cn=users," + self.base_dn, "cn=ldaptestuser3,cn=users," + self.base_dn)
+        attrs = ["objectGUID", "objectSid"]
+        print "Testing ldb.search for (&(cn=ldaptestUSer2)(objectClass=user))"
+        res_user = ldb.search(self.base_dn, expression="(&(cn=ldaptestUSer2)(objectClass=user))", scope=SCOPE_SUBTREE, attrs=attrs)
+        self.assertEquals(len(res_user), 1, "Could not find (&(cn=ldaptestUSer2)(objectClass=user))")
+
+        #Check rename works with extended/alternate DN forms 
+        ldb.rename("<SID=" + ldb.schema_format_value("objectSID", res_user[0]["objectSID"][0]) + ">" , "cn=ldaptestuser3,cn=users," + self.base_dn)
 
         ldb.rename("cn=ldaptestuser3,cn=users," + self.base_dn, "cn=ldaptestuser3,cn=users," + self.base_dn)
 
@@ -585,7 +595,7 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
 
         print "Testing ldb.search for (&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group)) to check subtree renames and linked attributes"
         res = ldb.search(self.base_dn, expression="(&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group))", scope=SCOPE_SUBTREE)
-        self.assertEquals(len(res), 1, "Could not find (&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group)), perhaps linked attributes are not conistant with subtree renames?")
+        self.assertEquals(len(res), 1, "Could not find (&(member=CN=ldaptestuser4,CN=ldaptestcontainer2," + self.base_dn + ")(objectclass=group)), perhaps linked attributes are not consistant with subtree renames?")
 
         print "Testing ldb.rename (into itself) of cn=ldaptestcontainer2," + self.base_dn + " to cn=ldaptestcontainer,cn=ldaptestcontainer2," + self.base_dn
         try:
@@ -737,8 +747,8 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
         self.assertEquals(len(res6), 1, "Could not find (&(cn=*daptestcomputer)(objectCategory=compuTER))")
 
         self.assertEquals(res[0].dn, res6[0].dn)
-
-        ldb.delete(res[0].dn)
+        
+        ldb.delete("<GUID=" + ldb.schema_format_value("objectGUID", res[0]["objectGUID"][0]) + ">")
 
         print "Testing ldb.search for (&(cn=ldaptest2computer)(objectClass=user))"
         res = ldb.search(expression="(&(cn=ldaptest2computer)(objectClass=user))")
@@ -754,25 +764,29 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
         self.assertEquals(int(res[0]["sAMAccountType"][0]), 805306369)
         self.assertEquals(int(res[0]["userAccountControl"][0]), 4096)
 
-        ldb.delete(res[0].dn)
+        ldb.delete("<SID=" + ldb.schema_format_value("objectSID", res[0]["objectSID"][0]) + ">")
 
-        attrs = ["cn", "name", "objectClass", "objectGUID", "whenCreated", "nTSecurityDescriptor", "memberOf", "allowedAttributes", "allowedAttributesEffective"]
+        attrs = ["cn", "name", "objectClass", "objectGUID", "objectSID", "whenCreated", "nTSecurityDescriptor", "memberOf", "allowedAttributes", "allowedAttributesEffective"]
         print "Testing ldb.search for (&(cn=ldaptestUSer2)(objectClass=user))"
-        res = ldb.search(self.base_dn, expression="(&(cn=ldaptestUSer2)(objectClass=user))", scope=SCOPE_SUBTREE, attrs=attrs)
-        self.assertEquals(len(res), 1, "Could not find (&(cn=ldaptestUSer2)(objectClass=user))")
-
-        self.assertEquals(res[0].dn, ("CN=ldaptestuser2,CN=Users," + self.base_dn))
-        self.assertEquals(res[0]["cn"], "ldaptestuser2")
-        self.assertEquals(res[0]["name"], "ldaptestuser2")
-        self.assertEquals(res[0]["objectClass"], ["top", "person", "organizationalPerson", "user"])
-        self.assertTrue("objectGUID" in res[0])
-        self.assertTrue("whenCreated" in res[0])
-        self.assertTrue("nTSecurityDescriptor" in res[0])
-        self.assertTrue("allowedAttributes" in res[0])
-        self.assertTrue("allowedAttributesEffective" in res[0])
-        self.assertEquals(res[0]["memberOf"][0].upper(), ("CN=ldaptestgroup2,CN=Users," + self.base_dn).upper())
-
-        attrs = ["cn", "name", "objectClass", "objectGUID", "whenCreated", "nTSecurityDescriptor", "member", "allowedAttributes", "allowedAttributesEffective"]
+        res_user = ldb.search(self.base_dn, expression="(&(cn=ldaptestUSer2)(objectClass=user))", scope=SCOPE_SUBTREE, attrs=attrs)
+        self.assertEquals(len(res_user), 1, "Could not find (&(cn=ldaptestUSer2)(objectClass=user))")
+
+        self.assertEquals(res_user[0].dn, ("CN=ldaptestuser2,CN=Users," + self.base_dn))
+        self.assertEquals(res_user[0]["cn"], "ldaptestuser2")
+        self.assertEquals(res_user[0]["name"], "ldaptestuser2")
+        self.assertEquals(res_user[0]["objectClass"], ["top", "person", "organizationalPerson", "user"])
+        self.assertTrue("objectSid" in res_user[0])
+        self.assertTrue("objectGUID" in res_user[0])
+        self.assertTrue("whenCreated" in res_user[0])
+        self.assertTrue("nTSecurityDescriptor" in res_user[0])
+        self.assertTrue("allowedAttributes" in res_user[0])
+        self.assertTrue("allowedAttributesEffective" in res_user[0])
+        self.assertEquals(res_user[0]["memberOf"][0].upper(), ("CN=ldaptestgroup2,CN=Users," + self.base_dn).upper())
+
+        ldaptestuser2_sid = res_user[0]["objectSid"][0]
+        ldaptestuser2_guid = res_user[0]["objectGUID"][0]
+
+        attrs = ["cn", "name", "objectClass", "objectGUID", "objectSID", "whenCreated", "nTSecurityDescriptor", "member", "allowedAttributes", "allowedAttributesEffective"]
         print "Testing ldb.search for (&(cn=ldaptestgroup2)(objectClass=group))"
         res = ldb.search(self.base_dn, expression="(&(cn=ldaptestgroup2)(objectClass=group))", scope=SCOPE_SUBTREE, attrs=attrs)
         self.assertEquals(len(res), 1, "Could not find (&(cn=ldaptestgroup2)(objectClass=group))")
@@ -781,7 +795,8 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
         self.assertEquals(res[0]["cn"], "ldaptestgroup2")
         self.assertEquals(res[0]["name"], "ldaptestgroup2")
         self.assertEquals(res[0]["objectClass"], ["top", "group"])
-        self.assertTrue("objectGuid" not in res[0])
+        self.assertTrue("objectGUID" in res[0])
+        self.assertTrue("objectSid" in res[0])
         self.assertTrue("whenCreated" in res[0])
         self.assertTrue("nTSecurityDescriptor" in res[0])
         self.assertTrue("allowedAttributes" in res[0])
@@ -791,6 +806,18 @@ member: cn=ldaptestuser4,cn=ldaptestcontainer,""" + self.base_dn + """
             memberUP.append(m.upper())
         self.assertTrue(("CN=ldaptestuser2,CN=Users," + self.base_dn).upper() in memberUP)
 
+        res = ldb.search(self.base_dn, expression="(&(cn=ldaptestgroup2)(objectClass=group))", scope=SCOPE_SUBTREE, attrs=attrs, controls=["extended_dn:1:1"])
+        self.assertEquals(len(res), 1, "Could not find (&(cn=ldaptestgroup2)(objectClass=group))")
+
+        print res[0]["member"]
+        memberUP = []
+        for m in res[0]["member"]:
+            memberUP.append(m.upper())
+        print ("<GUID=" + ldb.schema_format_value("objectGUID", ldaptestuser2_guid) + ">;<SID=" + ldb.schema_format_value("objectSid", ldaptestuser2_sid) + ">;CN=ldaptestuser2,CN=Users," + self.base_dn).upper()
+
+        self.assertTrue(("<GUID=" + ldb.schema_format_value("objectGUID", ldaptestuser2_guid) + ">;<SID=" + ldb.schema_format_value("objectSid", ldaptestuser2_sid) + ">;CN=ldaptestuser2,CN=Users," + self.base_dn).upper() in memberUP)
+
+        print "Testing Linked attribute behaviours"
         ldb.modify_ldif("""
 dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
 changetype: modify
@@ -799,9 +826,15 @@ member: CN=ldaptestuser2,CN=Users,""" + self.base_dn + """
 member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
 """)
         
-        print "Testing Linked attribute behaviours"
         ldb.modify_ldif("""
-dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
+dn: <GUID=""" + ldb.schema_format_value("objectGUID", res[0]["objectGUID"][0]) + """>
+changetype: modify
+replace: member
+member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
+""")
+        
+        ldb.modify_ldif("""
+dn: <SID=""" + ldb.schema_format_value("objectSid", res[0]["objectSid"][0]) + """>
 changetype: modify
 delete: member
 """)
@@ -810,7 +843,7 @@ delete: member
 dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
 changetype: modify
 add: member
-member: CN=ldaptestuser2,CN=Users,""" + self.base_dn + """
+member: <GUID=""" + ldb.schema_format_value("objectGUID", res[0]["objectGUID"][0]) + """>
 member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
 """)
         
@@ -824,7 +857,7 @@ replace: member
 dn: cn=ldaptestgroup2,cn=users,""" + self.base_dn + """
 changetype: modify
 add: member
-member: CN=ldaptestuser2,CN=Users,""" + self.base_dn + """
+member: <SID=""" + ldb.schema_format_value("objectSid", res_user[0]["objectSid"][0]) + """>
 member: CN=ldaptestutf8user èùéìòà,CN=Users,""" + self.base_dn + """
 """)
         
-- 
cgit 


From 8deded43ba0ab6a31739df5ca927eed48b81393a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Thu, 18 Dec 2008 14:12:19 +1100
Subject: Rename ldb index pointer wrapper function

This also asserts that it is used only for index records.

Andrew Bartlett
---
 source4/lib/ldb/ldb_tdb/ldb_index.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c
index d0b9fa27ce..0a10307e65 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_index.c
@@ -226,7 +226,7 @@ int ltdb_index_transaction_start(struct ldb_module *module)
   a wrapper around ltdb_search_dn1() which translates pointer based index records
   and maps them into normal ldb message structures
  */
-static int ltdb_search_dn1_wrap(struct ldb_module *module, 
+static int ltdb_search_dn1_index(struct ldb_module *module, 
 				struct ldb_dn *dn, struct ldb_message *msg)
 {
 	int ret, i;
@@ -237,7 +237,7 @@ static int ltdb_search_dn1_wrap(struct ldb_module *module,
 
 	/* if this isn't a @INDEX record then don't munge it */
 	if (strncmp(ldb_dn_get_linearized(msg->dn), LTDB_INDEX ":", strlen(LTDB_INDEX) + 1) != 0) {
-		return ret;
+		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
 	for (i=0;i<msg->num_elements;i++) {
@@ -265,7 +265,7 @@ static int ltdb_idxptr_fix_dn(struct ldb_module *module, const char *strdn)
 	int ret;
 	
 	dn = ldb_dn_new(msg, module->ldb, strdn);
-	if (ltdb_search_dn1_wrap(module, dn, msg) == LDB_SUCCESS) {
+	if (ltdb_search_dn1_index(module, dn, msg) == LDB_SUCCESS) {
 		ret = ltdb_store(module, msg, TDB_REPLACE);
 	}
 	talloc_free(msg);
@@ -535,7 +535,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1_wrap(module, dn, msg);
+	ret = ltdb_search_dn1_index(module, dn, msg);
 	talloc_free(dn);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -881,7 +881,7 @@ static int ltdb_index_dn_one(struct ldb_module *module,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1_wrap(module, key, msg);
+	ret = ltdb_search_dn1_index(module, key, msg);
 	talloc_free(key);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -1009,7 +1009,7 @@ static int ltdb_index_filter(const struct dn_list *dn_list,
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
 
-		ret = ltdb_search_dn1_wrap(ac->module, dn, msg);
+		ret = ltdb_search_dn1(ac->module, dn, msg);
 		talloc_free(dn);
 		if (ret == LDB_ERR_NO_SUCH_OBJECT) {
 			/* the record has disappeared? yes, this can happen */
@@ -1217,7 +1217,7 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
 	}
 	talloc_steal(msg, dn_key);
 
-	ret = ltdb_search_dn1_wrap(module, dn_key, msg);
+	ret = ltdb_search_dn1_index(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(msg);
 		return ret;
@@ -1329,7 +1329,7 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	ret = ltdb_search_dn1_wrap(module, dn_key, msg);
+	ret = ltdb_search_dn1_index(module, dn_key, msg);
 	if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
 		talloc_free(dn_key);
 		return ret;
-- 
cgit 


From a9cf61dbdd4cf914215723b17912f584fff151c0 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Thu, 18 Dec 2008 14:30:11 +1100
Subject: Print error strings when transactions fail in ldb tools

---
 source4/lib/ldb/tools/ldbadd.c    | 4 ++--
 source4/lib/ldb/tools/ldbedit.c   | 4 ++--
 source4/lib/ldb/tools/ldbmodify.c | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/tools/ldbadd.c b/source4/lib/ldb/tools/ldbadd.c
index 3749aec629..f32a4fa9bc 100644
--- a/source4/lib/ldb/tools/ldbadd.c
+++ b/source4/lib/ldb/tools/ldbadd.c
@@ -93,7 +93,7 @@ int main(int argc, const char **argv)
 	options = ldb_cmdline_process(ldb, argc, argv, usage);
 
 	if (ldb_transaction_start(ldb) != 0) {
-		printf("Failed to start transaction\n");
+		printf("Failed to start transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
@@ -114,7 +114,7 @@ int main(int argc, const char **argv)
 	}
 
 	if (count != 0 && ldb_transaction_commit(ldb) != 0) {
-		printf("Failed to commit transaction\n");
+		printf("Failed to commit transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
diff --git a/source4/lib/ldb/tools/ldbedit.c b/source4/lib/ldb/tools/ldbedit.c
index b18aea1b10..1a684c5c2d 100644
--- a/source4/lib/ldb/tools/ldbedit.c
+++ b/source4/lib/ldb/tools/ldbedit.c
@@ -113,7 +113,7 @@ static int merge_edits(struct ldb_context *ldb,
 	int adds=0, modifies=0, deletes=0;
 
 	if (ldb_transaction_start(ldb) != 0) {
-		fprintf(stderr, "Failed to start transaction\n");
+		fprintf(stderr, "Failed to start transaction: %s\n", ldb_errstring(ldb));
 		return -1;
 	}
 
@@ -156,7 +156,7 @@ static int merge_edits(struct ldb_context *ldb,
 	}
 
 	if (ldb_transaction_commit(ldb) != 0) {
-		fprintf(stderr, "Failed to commit transaction\n");
+		fprintf(stderr, "Failed to commit transaction: %s\n", ldb_errstring(ldb));
 		return -1;
 	}
 
diff --git a/source4/lib/ldb/tools/ldbmodify.c b/source4/lib/ldb/tools/ldbmodify.c
index 8b6309e016..0288e4e688 100644
--- a/source4/lib/ldb/tools/ldbmodify.c
+++ b/source4/lib/ldb/tools/ldbmodify.c
@@ -92,7 +92,7 @@ int main(int argc, const char **argv)
 	ldb = ldb_init(NULL, NULL);
 
 	if (ldb_transaction_start(ldb) != 0) {
-		printf("Failed to start transaction\n");
+		printf("Failed to start transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
@@ -114,7 +114,7 @@ int main(int argc, const char **argv)
 	}
 
 	if (count != 0 && ldb_transaction_commit(ldb) != 0) {
-		printf("Failed to commit transaction\n");
+		printf("Failed to commit transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
-- 
cgit 


From 1a399db01c90753f0f1c70b2987b925c828b1c53 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@samba.org>
Date: Thu, 18 Dec 2008 14:31:52 +1100
Subject: Parse options (and open the database) before starting transactions

---
 source4/lib/ldb/tools/ldbmodify.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'source4/lib/ldb')

diff --git a/source4/lib/ldb/tools/ldbmodify.c b/source4/lib/ldb/tools/ldbmodify.c
index 0288e4e688..d73937cea4 100644
--- a/source4/lib/ldb/tools/ldbmodify.c
+++ b/source4/lib/ldb/tools/ldbmodify.c
@@ -91,13 +91,13 @@ int main(int argc, const char **argv)
 
 	ldb = ldb_init(NULL, NULL);
 
+	options = ldb_cmdline_process(ldb, argc, argv, usage);
+
 	if (ldb_transaction_start(ldb) != 0) {
 		printf("Failed to start transaction: %s\n", ldb_errstring(ldb));
 		exit(1);
 	}
 
-	options = ldb_cmdline_process(ldb, argc, argv, usage);
-
 	if (options->argc == 0) {
 		ret = process_file(ldb, stdin, &count);
 	} else {
-- 
cgit