Merge 2610c05b5b95cc7036b3d6dfb894c6cfbdb68483 as Samba-4.0alpha16

15 files changed, 238 insertions, 115 deletions

diff --git a/source4/lib/ldb/common/ldb_controls.c b/source4/lib/ldb/common/ldb_controls.c
index 5048b6deac..b3ef243493 100644
--- a/source4/lib/ldb/common/ldb_controls.c
+++ b/source4/lib/ldb/common/ldb_controls.c
@@ -482,10 +482,10 @@ struct ldb_control *ldb_parse_control_from_string(struct ldb_context *ldb, TALLO
 		}
 
 		/* w2k3 seems to ignore the parameter,
-			* but w2k sends a wrong cookie when this value is to small
-			* this would cause looping forever, while getting
-			* the same data and same cookie forever
-			*/
+		 * but w2k sends a wrong cookie when this value is to small
+		 * this would cause looping forever, while getting
+		 * the same data and same cookie forever
+		 */
 		if (max_attrs == 0) max_attrs = 0x0FFFFFFF;
 
 		ctrl->oid = LDB_CONTROL_DIRSYNC_OID;
diff --git a/source4/lib/ldb/common/ldb_msg.c b/source4/lib/ldb/common/ldb_msg.c
index 9c5a279b8a..28c414e6b2 100644
--- a/source4/lib/ldb/common/ldb_msg.c
+++ b/source4/lib/ldb/common/ldb_msg.c
@@ -130,7 +130,6 @@ static int _ldb_msg_add_el(struct ldb_message *msg,
 	els = talloc_realloc(msg, msg->elements,
 			     struct ldb_message_element, msg->num_elements + 1);
 	if (!els) {
-		errno = ENOMEM;
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -164,7 +163,6 @@ int ldb_msg_add_empty(struct ldb_message *msg,
 	el->flags = flags;
 	el->name = talloc_strdup(msg->elements, attr_name);
 	if (!el->name) {
-		errno = ENOMEM;
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -229,7 +227,6 @@ int ldb_msg_add_value(struct ldb_message *msg,
 	vals = talloc_realloc(msg->elements, el->values, struct ldb_val,
 			      el->num_values+1);
 	if (!vals) {
-		errno = ENOMEM;
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 	el->values = vals;
diff --git a/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c b/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
index e5b43fd0c9..223868a6c0 100644
--- a/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
+++ b/source4/lib/ldb/ldb_sqlite3/ldb_sqlite3.c
@@ -1136,7 +1136,7 @@ static int lsql_modify(struct lsql_context *ctx)
 	for (i = 0; i < msg->num_elements; i++) {
 		const struct ldb_message_element *el = &msg->elements[i];
 		const struct ldb_schema_attribute *a;
-		int flags = el->flags & LDB_FLAG_MOD_MASK;
+		unsigned int flags = el->flags & LDB_FLAG_MOD_MASK;
 		char *attr;
 		char *mod;
 		unsigned int j;
@@ -1596,7 +1596,8 @@ static const struct ldb_module_ops lsqlite3_ops = {
  */
 
 static int initialize(struct lsqlite3_private *lsqlite3,
-		      struct ldb_context *ldb, const char *url, int flags)
+		      struct ldb_context *ldb, const char *url,
+		      unsigned int flags)
 {
 	TALLOC_CTX *local_ctx;
         long long queryInt;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_cache.c b/source4/lib/ldb/ldb_tdb/ldb_cache.c
index 697f7427a4..e54ceaaa98 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_cache.c
@@ -345,9 +345,6 @@ int ltdb_cache_load(struct ldb_module *module)
 		ltdb->check_base = false;
 	}
 
-	talloc_free(ltdb->cache->last_attribute.name);
-	memset(&ltdb->cache->last_attribute, 0, sizeof(ltdb->cache->last_attribute));
-
 	talloc_free(ltdb->cache->indexlist);
 	ltdb_attributes_unload(module); /* calls internally "talloc_free" */
 
diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c
index 02e4acbbde..24cc93feb9 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_index.c
@@ -155,7 +155,7 @@ static int ltdb_dn_list_load(struct ldb_module *module,
 	key.dptr = discard_const_p(unsigned char, ldb_dn_get_linearized(dn));
 	key.dsize = strlen((char *)key.dptr);
 
-	rec = tdb_fetch(ltdb->idxptr->itdb, key);
+	rec = tdb_fetch_compat(ltdb->idxptr->itdb, key);
 	if (rec.dptr == NULL) {
 		goto normal_index;
 	}
@@ -261,7 +261,7 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
 	}
 
 	if (ltdb->idxptr->itdb == NULL) {
-		ltdb->idxptr->itdb = tdb_open(NULL, 1000, TDB_INTERNAL, O_RDWR, 0);
+		ltdb->idxptr->itdb = tdb_open_compat(NULL, 1000, TDB_INTERNAL, O_RDWR, 0, NULL, NULL);
 		if (ltdb->idxptr->itdb == NULL) {
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
@@ -270,7 +270,7 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
 	key.dptr = discard_const_p(unsigned char, ldb_dn_get_linearized(dn));
 	key.dsize = strlen((char *)key.dptr);
 
-	rec = tdb_fetch(ltdb->idxptr->itdb, key);
+	rec = tdb_fetch_compat(ltdb->idxptr->itdb, key);
 	if (rec.dptr != NULL) {
 		list2 = ltdb_index_idxptr(module, rec, false);
 		if (list2 == NULL) {
@@ -294,7 +294,7 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
 	rec.dsize = sizeof(void *);
 
 	ret = tdb_store(ltdb->idxptr->itdb, key, rec, TDB_INSERT);
-	if (ret == -1) {
+	if (ret != 0) {
 		return ltdb_err_map(tdb_error(ltdb->idxptr->itdb));
 	}
 	return LDB_SUCCESS;
@@ -1569,7 +1569,7 @@ int ltdb_reindex(struct ldb_module *module)
 	 * putting NULL entries in the in-memory tdb
 	 */
 	ret = tdb_traverse(ltdb->tdb, delete_index, module);
-	if (ret == -1) {
+	if (ret < 0) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -1583,7 +1583,7 @@ int ltdb_reindex(struct ldb_module *module)
 
 	/* now traverse adding any indexes for normal LDB records */
 	ret = tdb_traverse(ltdb->tdb, re_index, &ctx);
-	if (ret == -1) {
+	if (ret < 0) {
 		struct ldb_context *ldb = ldb_module_get_ctx(module);
 		ldb_asprintf_errstring(ldb, "reindexing traverse failed: %s", ldb_errstring(ldb));
 		return LDB_ERR_OPERATIONS_ERROR;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_pack.c b/source4/lib/ldb/ldb_tdb/ldb_pack.c
index 8ab07cd347..7c13065aee 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_pack.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_pack.c
@@ -74,7 +74,7 @@ static int attribute_storable_values(const struct ldb_message_element *el)
 */
 int ltdb_pack_data(struct ldb_module *module,
 		   const struct ldb_message *message,
-		   struct TDB_DATA *data)
+		   TDB_DATA *data)
 {
 	struct ldb_context *ldb;
 	unsigned int i, j, real_elements=0;
@@ -155,7 +155,7 @@ int ltdb_pack_data(struct ldb_module *module,
   Free with ltdb_unpack_data_free()
 */
 int ltdb_unpack_data(struct ldb_module *module,
-		     const struct TDB_DATA *data,
+		     const TDB_DATA *data,
 		     struct ldb_message *message)
 {
 	struct ldb_context *ldb;
diff --git a/source4/lib/ldb/ldb_tdb/ldb_search.c b/source4/lib/ldb/ldb_tdb/ldb_search.c
index a49751de15..46e2d74998 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_search.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_search.c
@@ -32,6 +32,7 @@
  */
 
 #include "ldb_tdb.h"
+#include <lib/tdb_compat/tdb_compat.h>
 
 /*
   add one element to a message
@@ -223,7 +224,7 @@ static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	tdb_data = tdb_fetch(ltdb->tdb, tdb_key);
+	tdb_data = tdb_fetch_compat(ltdb->tdb, tdb_key);
 	talloc_free(tdb_key.dptr);
 	if (!tdb_data.dptr) {
 		return LDB_ERR_NO_SUCH_OBJECT;
@@ -255,7 +256,7 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
-	tdb_data = tdb_fetch(ltdb->tdb, tdb_key);
+	tdb_data = tdb_fetch_compat(ltdb->tdb, tdb_key);
 	talloc_free(tdb_key.dptr);
 	if (!tdb_data.dptr) {
 		return LDB_ERR_NO_SUCH_OBJECT;
@@ -479,7 +480,7 @@ static int ltdb_search_full(struct ltdb_context *ctx)
 		ret = tdb_traverse_read(ltdb->tdb, search_func, ctx);
 	}
 
-	if (ret == -1) {
+	if (ret < 0) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
index 2f7f222086..0d4be49123 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -50,6 +50,7 @@
  */
 
 #include "ldb_tdb.h"
+#include <lib/tdb_compat/tdb_compat.h>
 
 
 /*
@@ -67,9 +68,13 @@ int ltdb_err_map(enum TDB_ERROR tdb_code)
 	case TDB_ERR_IO:
 		return LDB_ERR_PROTOCOL_ERROR;
 	case TDB_ERR_LOCK:
+#ifndef BUILD_TDB2
 	case TDB_ERR_NOLOCK:
+#endif
 		return LDB_ERR_BUSY;
+#ifndef BUILD_TDB2
 	case TDB_ERR_LOCK_TIMEOUT:
+#endif
 		return LDB_ERR_TIME_LIMIT_EXCEEDED;
 	case TDB_ERR_EXISTS:
 		return LDB_ERR_ENTRY_ALREADY_EXISTS;
@@ -110,7 +115,8 @@ int ltdb_unlock_read(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
-		return tdb_unlockall_read(ltdb->tdb);
+		tdb_unlockall_read(ltdb->tdb);
+		return 0;
 	}
 	ltdb->read_lock_count--;
 	return 0;
@@ -124,7 +130,7 @@ int ltdb_unlock_read(struct ldb_module *module)
   note that the key for a record can depend on whether the
   dn refers to a case sensitive index record or not
 */
-struct TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn)
+TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn)
 {
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 	TDB_DATA key;
@@ -263,7 +269,7 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
 	}
 
 	ret = tdb_store(ltdb->tdb, tdb_key, tdb_data, flgs);
-	if (ret == -1) {
+	if (ret != 0) {
 		ret = ltdb_err_map(tdb_error(ltdb->tdb));
 		goto done;
 	}
@@ -653,7 +659,7 @@ int ltdb_modify_internal(struct ldb_module *module,
 		return LDB_ERR_OTHER;
 	}
 
-	tdb_data = tdb_fetch(ltdb->tdb, tdb_key);
+	tdb_data = tdb_fetch_compat(ltdb->tdb, tdb_key);
 	if (!tdb_data.dptr) {
 		talloc_free(tdb_key.dptr);
 		return ltdb_err_map(tdb_error(ltdb->tdb));
@@ -1072,10 +1078,7 @@ static int ltdb_del_trans(struct ldb_module *module)
 		return ltdb_err_map(tdb_error(ltdb->tdb));
 	}
 
-	if (tdb_transaction_cancel(ltdb->tdb) != 0) {
-		return ltdb_err_map(tdb_error(ltdb->tdb));
-	}
-
+	tdb_transaction_cancel(ltdb->tdb);
 	return LDB_SUCCESS;
 }
 
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
index 33313b00da..96ad43fbd6 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -1,7 +1,7 @@
 #include "replace.h"
 #include "system/filesys.h"
 #include "system/time.h"
-#include "tdb.h"
+#include "tdb_compat.h"
 #include "ldb_module.h"
 
 /* this private structure is used by the ltdb backend in the
@@ -21,11 +21,6 @@ struct ltdb_private {
 		struct ldb_message *attributes;
 		bool one_level_indexes;
 		bool attribute_indexes;
-
-		struct {
-			char *name;
-			int flags;
-		} last_attribute;
 	} *cache;
 
 	int in_transaction;
@@ -107,11 +102,11 @@ int ltdb_index_transaction_cancel(struct ldb_module *module);
 
 int ltdb_pack_data(struct ldb_module *module,
 		   const struct ldb_message *message,
-		   struct TDB_DATA *data);
+		   TDB_DATA *data);
 void ltdb_unpack_data_free(struct ldb_module *module,
 			   struct ldb_message *message);
 int ltdb_unpack_data(struct ldb_module *module,
-		     const struct TDB_DATA *data,
+		     const TDB_DATA *data,
 		     struct ldb_message *message);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_search.c  */
@@ -132,7 +127,7 @@ int ltdb_search(struct ltdb_context *ctx);
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_tdb.c  */
 int ltdb_lock_read(struct ldb_module *module);
 int ltdb_unlock_read(struct ldb_module *module);
-struct TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn);
+TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn);
 int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
 int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
 int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn);
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c b/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
index b9f3e79f20..16a037a6c3 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
@@ -24,29 +24,30 @@
 #include "ldb_tdb.h"
 #include "dlinklist.h"
 
-/*
-  the purpose of this code is to work around the braindead posix locking
-  rules, to allow us to have a ldb open more than once while allowing 
-  locking to work
-*/
-
-struct ltdb_wrap {
-	struct ltdb_wrap *next, *prev;
-	struct tdb_context *tdb;
-	dev_t device;
-	ino_t inode;
-};
-
-static struct ltdb_wrap *tdb_list;
-
-/* destroy the last connection to a tdb */
-static int ltdb_wrap_destructor(struct ltdb_wrap *w)
+/* FIXME: TDB2 does this internally, so no need to wrap multiple opens! */
+#if BUILD_TDB2
+static void ltdb_log_fn(struct tdb_context *tdb,
+			enum tdb_log_level level,
+			const char *message,
+			struct ldb_context *ldb)
 {
-	tdb_close(w->tdb);
-	DLIST_REMOVE(tdb_list, w);
-	return 0;
-}				 
+	enum ldb_debug_level ldb_level;
+	const char *name = tdb_name(tdb);
+
+	switch (level) {
+	case TDB_LOG_WARNING:
+		ldb_level = LDB_DEBUG_WARNING;
+	case TDB_LOG_USE_ERROR:
+	case TDB_LOG_ERROR:
+		ldb_level = LDB_DEBUG_FATAL;
+		break;
+	default:
+		ldb_level = LDB_DEBUG_FATAL;
+	}
 
+	ldb_debug(ldb, ldb_level, "ltdb: tdb(%s): %s", name, message);
+}
+#else /* !TDB2 */
 static void ltdb_log_fn(struct tdb_context *tdb, enum tdb_debug_level level, const char *fmt, ...) PRINTF_ATTRIBUTE(3, 4);
 static void ltdb_log_fn(struct tdb_context *tdb, enum tdb_debug_level level, const char *fmt, ...)
 {
@@ -83,6 +84,32 @@ static void ltdb_log_fn(struct tdb_context *tdb, enum tdb_debug_level level, con
 	ldb_debug(ldb, ldb_level, "ltdb: tdb(%s): %s", name, message);
 	talloc_free(message);
 }
+#endif
+
+/*
+  the purpose of this code is to work around the braindead posix locking
+  rules, to allow us to have a ldb open more than once while allowing
+  locking to work
+
+  TDB2 handles multiple opens, so we don't have this problem there.
+*/
+
+struct ltdb_wrap {
+	struct ltdb_wrap *next, *prev;
+	struct tdb_context *tdb;
+	dev_t device;
+	ino_t inode;
+};
+
+static struct ltdb_wrap *tdb_list;
+
+/* destroy the last connection to a tdb */
+static int ltdb_wrap_destructor(struct ltdb_wrap *w)
+{
+	tdb_close(w->tdb);
+	DLIST_REMOVE(tdb_list, w);
+	return 0;
+}
 
 /*
   wrapped connection to a tdb database. The caller should _not_ free
@@ -98,10 +125,6 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
 {
 	struct ltdb_wrap *w;
 	struct stat st;
-	struct tdb_logging_context log_ctx;
-
-	log_ctx.log_fn = ltdb_log_fn;
-	log_ctx.log_private = ldb;
 
 	if (stat(path, &st) == 0) {
 		for (w=tdb_list;w;w=w->next) {
@@ -119,7 +142,7 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 
-	w->tdb = tdb_open_ex(path, hash_size, tdb_flags, open_flags, mode, &log_ctx, NULL);
+	w->tdb = tdb_open_compat(path, hash_size, tdb_flags, open_flags, mode, ltdb_log_fn, ldb);
 	if (w->tdb == NULL) {
 		talloc_free(w);
 		return NULL;
@@ -140,4 +163,3 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
 	
 	return w->tdb;
 }
-
diff --git a/source4/lib/ldb/pyldb.c b/source4/lib/ldb/pyldb.c
index 5fcc5a64b6..e2a2e7180e 100644
--- a/source4/lib/ldb/pyldb.c
+++ b/source4/lib/ldb/pyldb.c
@@ -7,6 +7,8 @@
    Copyright (C) 2006 Simo Sorce <idra@samba.org>
    Copyright (C) 2007-2010 Jelmer Vernooij <jelmer@samba.org>
    Copyright (C) 2009-2010 Matthias Dieter Wallnöfer
+   Copyright (C) 2009-2011 Andrew Tridgell
+   Copyright (C) 2009-2011 Andrew Bartlett
 
     ** NOTE! The following LGPL license applies to the ldb
     ** library. This does NOT imply that all of Samba is released
@@ -48,7 +50,7 @@ static PyObject *PyLdbModule_FromModule(struct ldb_module *mod);
 static struct ldb_message_element *PyObject_AsMessageElement(
 						      TALLOC_CTX *mem_ctx,
 						      PyObject *set_obj,
-						      int flags,
+						      unsigned int flags,
 						      const char *attr_name);
 
 /* There's no Py_ssize_t in 2.4, apparently */
@@ -384,6 +386,62 @@ static PyObject *py_ldb_dn_canonical_ex_str(PyLdbDnObject *self)
 	return PyString_FromString(ldb_dn_canonical_ex_string(self->dn, self->dn));
 }
 
+static PyObject *py_ldb_dn_extended_str(PyLdbDnObject *self, PyObject *args, PyObject *kwargs)
+{
+	const char * const kwnames[] = { "mode", NULL };
+	int mode = 1;
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i",
+					 discard_const_p(char *, kwnames),
+					 &mode))
+		return NULL;
+	return PyString_FromString(ldb_dn_get_extended_linearized(self->dn, self->dn, mode));
+}
+
+static PyObject *py_ldb_dn_get_extended_component(PyLdbDnObject *self, PyObject *args)
+{
+	char *name;
+	const struct ldb_val *val;
+
+	if (!PyArg_ParseTuple(args, "s", &name))
+		return NULL;
+	val = ldb_dn_get_extended_component(self->dn, name);
+	if (val == NULL) {
+		Py_RETURN_NONE;
+	}
+
+	return PyString_FromStringAndSize((const char *)val->data, val->length);
+}
+
+static PyObject *py_ldb_dn_set_extended_component(PyLdbDnObject *self, PyObject *args)
+{
+	char *name;
+	PyObject *value;
+	int err;
+
+	if (!PyArg_ParseTuple(args, "sO", &name, &value))
+		return NULL;
+
+	if (value == Py_None) {
+		err = ldb_dn_set_extended_component(self->dn, name, NULL);
+	} else {
+		struct ldb_val val;
+		if (!PyString_Check(value)) {
+			PyErr_SetString(PyExc_TypeError, "Expected a string argument");
+			return NULL;
+		}
+		val.data = (uint8_t *)PyString_AsString(value);
+		val.length = PyString_Size(value);
+		err = ldb_dn_set_extended_component(self->dn, name, &val);
+	}
+
+	if (err != LDB_SUCCESS) {
+		PyErr_SetString(PyExc_TypeError, "Failed to set extended component");
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
 static PyObject *py_ldb_dn_repr(PyLdbDnObject *self)
 {
 	return PyString_FromFormat("Dn(%s)", PyObject_REPR(PyString_FromString(ldb_dn_get_linearized(self->dn))));
@@ -485,6 +543,9 @@ static PyMethodDef py_ldb_dn_methods[] = {
 	{ "canonical_ex_str", (PyCFunction)py_ldb_dn_canonical_ex_str, METH_NOARGS,
 		"S.canonical_ex_str() -> string\n"
 		"Canonical version of this DN (like a posix path, with terminating newline)." },
+	{ "extended_str", (PyCFunction)py_ldb_dn_extended_str, METH_VARARGS | METH_KEYWORDS,
+		"S.extended_str(mode=1) -> string\n"
+		"Extended version of this DN" },
 	{ "parent", (PyCFunction)py_ldb_dn_get_parent, METH_NOARGS,
    		"S.parent() -> dn\n"
 		"Get the parent for this DN." },
@@ -497,6 +558,12 @@ static PyMethodDef py_ldb_dn_methods[] = {
 	{ "check_special", (PyCFunction)py_ldb_dn_check_special, METH_VARARGS,
 		"S.check_special(name) -> bool\n\n"
 		"Check if name is a special DN name"},
+	{ "get_extended_component", (PyCFunction)py_ldb_dn_get_extended_component, METH_VARARGS,
+		"S.get_extended_component(name) -> string\n\n"
+		"returns a DN extended component as a binary string"},
+	{ "set_extended_component", (PyCFunction)py_ldb_dn_set_extended_component, METH_VARARGS,
+		"S.set_extended_component(name, value) -> string\n\n"
+		"set a DN extended component as a binary string"},
 	{ NULL }
 };
 
@@ -737,11 +804,11 @@ static int py_ldb_init(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 	char *url = NULL;
 	PyObject *py_options = Py_None;
 	const char **options;
-	int flags = 0;
+	unsigned int flags = 0;
 	int ret;
 	struct ldb_context *ldb;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ziO:Ldb.__init__",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|zIO:Ldb.__init__",
 					 discard_const_p(char *, kwnames),
 					 &url, &flags, &py_options))
 		return -1;
@@ -792,13 +859,13 @@ static PyObject *py_ldb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs
 static PyObject *py_ldb_connect(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	char *url;
-	int flags = 0;
+	unsigned int flags = 0;
 	PyObject *py_options = Py_None;
 	int ret;
 	const char **options;
 	const char * const kwnames[] = { "url", "flags", "options", NULL };
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ziO",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|zIO",
 					 discard_const_p(char *, kwnames),
 					 &url, &flags, &py_options))
 		return NULL;
@@ -819,7 +886,7 @@ static PyObject *py_ldb_connect(PyLdbObject *self, PyObject *args, PyObject *kwa
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_msg;
 	PyObject *py_controls = Py_None;
@@ -829,8 +896,12 @@ static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args)
 	struct ldb_message *msg;
 	int ret;
 	TALLOC_CTX *mem_ctx;
+	bool validate=true;
+	const char * const kwnames[] = { "message", "controls", "validate", NULL };
 
-	if (!PyArg_ParseTuple(args, "O|O", &py_msg, &py_controls))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Ob",
+					 discard_const_p(char *, kwnames),
+					 &py_msg, &py_controls, &validate))
 		return NULL;
 
 	mem_ctx = talloc_new(NULL);
@@ -855,11 +926,13 @@ static PyObject *py_ldb_modify(PyLdbObject *self, PyObject *args)
 	}
 	msg = PyLdbMessage_AsMessage(py_msg);
 
-	ret = ldb_msg_sanity_check(ldb_ctx, msg);
-	if (ret != LDB_SUCCESS) {
-		PyErr_LDB_ERROR_IS_ERR_RAISE(PyExc_LdbError, ret, ldb_ctx);
-		talloc_free(mem_ctx);
-		return NULL;
+	if (validate) {
+		ret = ldb_msg_sanity_check(ldb_ctx, msg);
+		if (ret != LDB_SUCCESS) {
+			PyErr_LDB_ERROR_IS_ERR_RAISE(PyExc_LdbError, ret, ldb_ctx);
+			talloc_free(mem_ctx);
+			return NULL;
+		}
 	}
 
 	ret = ldb_build_mod_req(&req, ldb_ctx, mem_ctx, msg, parsed_controls,
@@ -958,7 +1031,7 @@ static struct ldb_message *PyDict_AsMessage(TALLOC_CTX *mem_ctx,
 	return msg;
 }
 
-static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_obj;
 	int ret;
@@ -968,8 +1041,11 @@ static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args)
 	PyObject *py_controls = Py_None;
 	TALLOC_CTX *mem_ctx;
 	struct ldb_control **parsed_controls;
+	const char * const kwnames[] = { "message", "controls", NULL };
 
-	if (!PyArg_ParseTuple(args, "O|O", &py_obj, &py_controls ))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O",
+					 discard_const_p(char *, kwnames),
+					 &py_obj, &py_controls))
 		return NULL;
 
 	mem_ctx = talloc_new(NULL);
@@ -1047,7 +1123,7 @@ static PyObject *py_ldb_add(PyLdbObject *self, PyObject *args)
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_dn;
 	struct ldb_dn *dn;
@@ -1057,8 +1133,11 @@ static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args)
 	PyObject *py_controls = Py_None;
 	TALLOC_CTX *mem_ctx;
 	struct ldb_control **parsed_controls;
+	const char * const kwnames[] = { "dn", "controls", NULL };
 
-	if (!PyArg_ParseTuple(args, "O|O", &py_dn, &py_controls))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O",
+					 discard_const_p(char *, kwnames),
+					 &py_dn, &py_controls))
 		return NULL;
 
 	mem_ctx = talloc_new(NULL);
@@ -1119,7 +1198,7 @@ static PyObject *py_ldb_delete(PyLdbObject *self, PyObject *args)
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_ldb_rename(PyLdbObject *self, PyObject *args)
+static PyObject *py_ldb_rename(PyLdbObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *py_dn1, *py_dn2;
 	struct ldb_dn *dn1, *dn2;
@@ -1130,10 +1209,13 @@ static PyObject *py_ldb_rename(PyLdbObject *self, PyObject *args)
 	struct ldb_control **parsed_controls;
 	struct ldb_context *ldb_ctx;
 	struct ldb_request *req;
+	const char * const kwnames[] = { "dn1", "dn2", "controls", NULL };
 
 	ldb_ctx = PyLdb_AsLdbContext(self);
 
-	if (!PyArg_ParseTuple(args, "OO|O", &py_dn1, &py_dn2, &py_controls))
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|O",
+					 discard_const_p(char *, kwnames),
+					 &py_dn1, &py_dn2, &py_controls))
 		return NULL;
 
 
@@ -1586,17 +1668,17 @@ static PyMethodDef py_ldb_methods[] = {
 	{ "connect", (PyCFunction)py_ldb_connect, METH_VARARGS|METH_KEYWORDS, 
 		"S.connect(url, flags=0, options=None) -> None\n"
 		"Connect to a LDB URL." },
-	{ "modify", (PyCFunction)py_ldb_modify, METH_VARARGS, 
-		"S.modify(message) -> None\n"
+	{ "modify", (PyCFunction)py_ldb_modify, METH_VARARGS|METH_KEYWORDS,
+		"S.modify(message, controls=None, validate=False) -> None\n"
 		"Modify an entry." },
-	{ "add", (PyCFunction)py_ldb_add, METH_VARARGS, 
-		"S.add(message) -> None\n"
+	{ "add", (PyCFunction)py_ldb_add, METH_VARARGS|METH_KEYWORDS,
+		"S.add(message, controls=None) -> None\n"
 		"Add an entry." },
-	{ "delete", (PyCFunction)py_ldb_delete, METH_VARARGS,
-		"S.delete(dn) -> None\n"
+	{ "delete", (PyCFunction)py_ldb_delete, METH_VARARGS|METH_KEYWORDS,
+		"S.delete(dn, controls=None) -> None\n"
 		"Remove an entry." },
-	{ "rename", (PyCFunction)py_ldb_rename, METH_VARARGS,
-		"S.rename(old_dn, new_dn) -> None\n"
+	{ "rename", (PyCFunction)py_ldb_rename, METH_VARARGS|METH_KEYWORDS,
+		"S.rename(old_dn, new_dn, controls=None) -> None\n"
 		"Rename an entry." },
 	{ "search", (PyCFunction)py_ldb_search, METH_VARARGS|METH_KEYWORDS,
 		"S.search(base=None, scope=None, expression=None, attrs=None, controls=None) -> msgs\n"
@@ -2031,7 +2113,7 @@ static PyTypeObject PyLdbModule = {
 static struct ldb_message_element *PyObject_AsMessageElement(
 						      TALLOC_CTX *mem_ctx,
 						      PyObject *set_obj,
-						      int flags,
+						      unsigned int flags,
 						      const char *attr_name)
 {
 	struct ldb_message_element *me;
@@ -2122,9 +2204,9 @@ static PyObject *py_ldb_msg_element_flags(PyLdbMessageElementObject *self, PyObj
 
 static PyObject *py_ldb_msg_element_set_flags(PyLdbMessageElementObject *self, PyObject *args)
 {
-	int flags;
+	unsigned int flags;
 	struct ldb_message_element *el;
-	if (!PyArg_ParseTuple(args, "i", &flags))
+	if (!PyArg_ParseTuple(args, "I", &flags))
 		return NULL;
 
 	el = PyLdbMessageElement_AsMessageElement(self);
@@ -2192,13 +2274,13 @@ static PyObject *py_ldb_msg_element_new(PyTypeObject *type, PyObject *args, PyOb
 {
 	PyObject *py_elements = NULL;
 	struct ldb_message_element *el;
-	int flags = 0;
+	unsigned int flags = 0;
 	char *name = NULL;
 	const char * const kwnames[] = { "elements", "flags", "name", NULL };
 	PyLdbMessageElementObject *ret;
 	TALLOC_CTX *mem_ctx;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|Ois",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OIs",
 					 discard_const_p(char *, kwnames),
 					 &py_elements, &flags, &name))
 		return NULL;
@@ -2427,15 +2509,20 @@ static PyObject *py_ldb_msg_getitem(PyLdbMessageObject *self, PyObject *py_name)
 
 static PyObject *py_ldb_msg_get(PyLdbMessageObject *self, PyObject *args)
 {
-	PyObject *name, *ret;
-	if (!PyArg_ParseTuple(args, "O", &name))
+	PyObject *name, *ret, *retobj;
+	retobj = NULL;
+	if (!PyArg_ParseTuple(args, "O|O", &name, &retobj))
 		return NULL;
 
 	ret = py_ldb_msg_getitem_helper(self, name);
 	if (ret == NULL) {
 		if (PyErr_Occurred())
 			return NULL;
-		Py_RETURN_NONE;
+		if (retobj != NULL) {
+			return retobj;
+		} else {
+			Py_RETURN_NONE;
+		}
 	}
 	return ret;
 }
@@ -3202,4 +3289,14 @@ void initldb(void)
 	PyModule_AddObject(m, "Control", (PyObject *)&PyLdbControl);
 
 	PyModule_AddObject(m, "__version__", PyString_FromString(PACKAGE_VERSION));
+
+#define ADD_LDB_STRING(val)  PyModule_AddObject(m, #val, PyString_FromString(val))
+
+	ADD_LDB_STRING(LDB_SYNTAX_DN);
+	ADD_LDB_STRING(LDB_SYNTAX_DN);
+	ADD_LDB_STRING(LDB_SYNTAX_DIRECTORY_STRING);
+	ADD_LDB_STRING(LDB_SYNTAX_INTEGER);
+	ADD_LDB_STRING(LDB_SYNTAX_BOOLEAN);
+	ADD_LDB_STRING(LDB_SYNTAX_OCTET_STRING);
+	ADD_LDB_STRING(LDB_SYNTAX_UTC_TIME);
 }
diff --git a/source4/lib/ldb/tests/python/api.py b/source4/lib/ldb/tests/python/api.py
index a9f68cbd71..e7658d51ab 100755
--- a/source4/lib/ldb/tests/python/api.py
+++ b/source4/lib/ldb/tests/python/api.py
@@ -516,6 +516,11 @@ class LdbMsgTests(unittest.TestCase):
         self.msg["foo"] = ["bar"]
         self.assertEquals("bar", self.msg.get("foo")[0])
 
+    def test_get_default(self):
+        self.assertEquals(None, self.msg.get("tatayoyo"))
+
+        self.assertEquals("anniecordie", self.msg.get("tatayoyo", "anniecordie"))
+
     def test_get_unknown(self):
         self.assertEquals(None, self.msg.get("lalalala"))
 
diff --git a/source4/lib/ldb/tools/cmdline.c b/source4/lib/ldb/tools/cmdline.c
index b2be54ebf9..a06445fc0f 100644
--- a/source4/lib/ldb/tools/cmdline.c
+++ b/source4/lib/ldb/tools/cmdline.c
@@ -101,7 +101,7 @@ struct ldb_cmdline *ldb_cmdline_process(struct ldb_context *ldb,
 	poptContext pc;
 	int num_options = 0;
 	int opt;
-	int flags = 0;
+	unsigned int flags = 0;
 	int rc;
 	struct poptOption **popt_options;
 
@@ -297,7 +297,7 @@ struct ldb_cmdline *ldb_cmdline_process(struct ldb_context *ldb,
 	}
 
 	/* now connect to the ldb */
-	if (ldb_connect(ldb, ret->url, flags, ret->options) != 0) {
+	if (ldb_connect(ldb, ret->url, flags, ret->options) != LDB_SUCCESS) {
 		fprintf(stderr, "Failed to connect to %s - %s\n", 
 			ret->url, ldb_errstring(ldb));
 		goto failed;
diff --git a/source4/lib/ldb/tools/ldbtest.c b/source4/lib/ldb/tools/ldbtest.c
index a9d8fafe81..4e181af9d5 100644
--- a/source4/lib/ldb/tools/ldbtest.c
+++ b/source4/lib/ldb/tools/ldbtest.c
@@ -126,7 +126,7 @@ static void add_records(struct ldb_context *ldb,
 
 		ldb_delete(ldb, msg.dn);
 
-		if (ldb_add(ldb, &msg) != 0) {
+		if (ldb_add(ldb, &msg) != LDB_SUCCESS) {
 			printf("Add of %s failed - %s\n", name, ldb_errstring(ldb));
 			exit(LDB_ERR_OPERATIONS_ERROR);
 		}
@@ -183,7 +183,7 @@ static void modify_records(struct ldb_context *ldb,
 		vals[2].data = (uint8_t *)talloc_asprintf(tmp_ctx, "%s@other2.example.com", name);
 		vals[2].length = strlen((char *)vals[2].data);
 
-		if (ldb_modify(ldb, &msg) != 0) {
+		if (ldb_modify(ldb, &msg) != LDB_SUCCESS) {
 			printf("Modify of %s failed - %s\n", name, ldb_errstring(ldb));
 			exit(LDB_ERR_OPERATIONS_ERROR);
 		}
@@ -213,7 +213,7 @@ static void delete_records(struct ldb_context *ldb,
 		printf("Deleting uid Test%d\r", i);
 		fflush(stdout);
 
-		if (ldb_delete(ldb, dn) != 0) {
+		if (ldb_delete(ldb, dn) != LDB_SUCCESS) {
 			printf("Delete of %s failed - %s\n", ldb_dn_get_linearized(dn), ldb_errstring(ldb));
 			exit(LDB_ERR_OPERATIONS_ERROR);
 		}
@@ -304,7 +304,7 @@ static void start_test_index(struct ldb_context **ldb)
 	struct ldb_dn *indexlist;
 	struct ldb_dn *basedn;
 	int ret;
-	int flags = 0;
+	unsigned int flags = 0;
 	const char *specials;
 
 	specials = getenv("LDB_SPECIALS");
@@ -343,7 +343,7 @@ static void start_test_index(struct ldb_context **ldb)
 	ldb_msg_add_string(msg, "uid", strdup("test"));
 	ldb_msg_add_string(msg, "objectClass", strdup("OpenLDAPperson"));
 
-	if (ldb_add(*ldb, msg) != 0) {
+	if (ldb_add(*ldb, msg) != LDB_SUCCESS) {
 		printf("Add of %s failed - %s\n", ldb_dn_get_linearized(msg->dn), ldb_errstring(*ldb));
 		exit(LDB_ERR_OPERATIONS_ERROR);
 	}
@@ -356,7 +356,7 @@ static void start_test_index(struct ldb_context **ldb)
 	(*ldb) = ldb_init(options, NULL);
 
 	ret = ldb_connect(*ldb, options->url, flags, NULL);
-	if (ret != 0) {
+	if (ret != LDB_SUCCESS) {
 		printf("failed to connect to %s\n", options->url);
 		exit(LDB_ERR_OPERATIONS_ERROR);
 	}
diff --git a/source4/lib/ldb/wscript b/source4/lib/ldb/wscript
index b59e782e45..7de95494c7 100755
--- a/source4/lib/ldb/wscript
+++ b/source4/lib/ldb/wscript
@@ -16,29 +16,33 @@ sys.path.insert(0, srcdir + '/buildtools/wafsamba')
 import wafsamba, samba_dist, Options
 
 samba_dist.DIST_DIRS('''source4/lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
-                        lib/tdb:lib/tdb lib/tevent:lib/tevent lib/popt:lib/popt
+                        lib/tdb:lib/tdb lib/tdb2:lib/tdb2 lib/tdb_compat:lib/tdb_compat lib/ccan:lib/ccan lib/tevent:lib/tevent lib/popt:lib/popt
                         buildtools:buildtools''')
 
 
 def set_options(opt):
     opt.BUILTIN_DEFAULT('replace')
     opt.PRIVATE_EXTENSION_DEFAULT('ldb', noextension='ldb')
-    opt.RECURSE('lib/tdb')
+    opt.RECURSE('lib/tdb_compat')
     opt.RECURSE('lib/tevent')
     opt.RECURSE('lib/replace')
     opt.tool_options('python') # options for disabling pyc or pyo compilation
 
 def configure(conf):
-    conf.RECURSE('lib/tdb')
+    conf.RECURSE('lib/tdb_compat')
     conf.RECURSE('lib/tevent')
     conf.RECURSE('lib/popt')
     conf.RECURSE('lib/replace')
+    conf.RECURSE('lib/tdb_compat')
     conf.find_program('python', var='PYTHON')
     conf.find_program('xsltproc', var='XSLTPROC')
     conf.check_tool('python')
     conf.check_python_version((2,4,2))
     conf.SAMBA_CHECK_PYTHON_HEADERS(mandatory=True)
 
+    # This make #include <ccan/...> work.
+    conf.ADD_EXTRA_INCLUDES('''#lib''')
+
     # where does the default LIBDIR end up? in conf.env somewhere?
     #
     conf.CONFIG_PATH('LDB_MODULESDIR', conf.SUBST_ENV_VAR('MODULESDIR') + '/ldb')
@@ -72,10 +76,11 @@ def configure(conf):
     conf.SAMBA_CONFIG_H()
 
 def build(bld):
-    bld.RECURSE('lib/tdb')
+    bld.RECURSE('lib/tdb_compat')
     bld.RECURSE('lib/tevent')
     bld.RECURSE('lib/popt')
     bld.RECURSE('lib/replace')
+    bld.RECURSE('lib/tdb_compat')
 
     if bld.env.standalone_ldb:
         private_library = False
@@ -233,14 +238,14 @@ def build(bld):
                          init_function='ldb_tdb_init',
                          module_init_name='ldb_init_module',
                          internal_module=False,
-                         deps='tdb ldb',
+                         deps='tdb_compat ldb',
                          subsystem='ldb')
 
         # have a separate subsystem for common/ldb.c, so it can rebuild
         # for install with a different -DLDB_MODULESDIR=
         bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
                             'common/ldb.c',
-                            deps='tevent',
+                            deps='tevent tdb_compat',
                             includes='include',
                             cflags=['-DLDB_MODULESDIR=\"%s\"' % modules_dir])