r8037: a fairly major update to the internals of ldb. Changes are:

 - moved the knowledge of attribute types out of ldb_tdb and into the
   generic ldb code. This allows the ldb_match() message match logic
   to be generic, so it can be used by other backend

 - added the generic ability to load attribute handlers, for
   canonicalisation, compare, ldif read and ldif write. In the future
   this will be used by the schema module to allow us to correctly
   obey the attributetype schema elements

 - added attribute handlers for some of the core ldap attribute types,
   Integer, DirectoryString, DN, ObjectClass etc

 - added automatic registration of attribute handlers for well-known
   attribute names 'cn', 'dc', 'dn', 'ou' and 'objectClass'

 - converted the objectSid special handlers for Samba to the new system

 - added more correct handling of indexing in tdb backend based on the
   attribute canonicalisation function

 - added generic support for subclasses, moving it out of the tdb
   backend. This will be used in future by the schema module

 - fixed several bugs in the dn_explode code. It still needs more
   work, but doesn't corrupt ldb dbs any more.
(This used to be commit 944c5844ab441b96d8e5d7b2d151982139d1fab9)

6 files changed, 282 insertions, 543 deletions

diff --git a/source4/lib/ldb/ldb_tdb/ldb_cache.c b/source4/lib/ldb/ldb_tdb/ldb_cache.c
index 0bc2d7b123..0b7ddad5db 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_cache.c
@@ -37,6 +37,11 @@
 #include "ldb/include/ldb_private.h"
 #include "ldb/ldb_tdb/ldb_tdb.h"
 
+#define LTDB_FLAG_CASE_INSENSITIVE (1<<0)
+#define LTDB_FLAG_INTEGER          (1<<1)
+#define LTDB_FLAG_WILDCARD         (1<<2)
+#define LTDB_FLAG_HIDDEN           (1<<3)
+#define LTDB_FLAG_OBJECTCLASS      (1<<4)
 
 /* valid attribute flags */
 static const struct {
@@ -47,12 +52,182 @@ static const struct {
 	{ "INTEGER", LTDB_FLAG_INTEGER },
 	{ "WILDCARD", LTDB_FLAG_WILDCARD },
 	{ "HIDDEN", LTDB_FLAG_HIDDEN },
-	{ "NONE", LTDB_FLAG_NONE },
+	{ "NONE", 0 },
 	{ NULL, 0 }
 };
 
 
 /*
+  de-register any special handlers for @ATTRIBUTES
+*/
+static void ltdb_attributes_unload(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb = module->private_data;
+	struct ldb_message *msg;
+	int i;
+
+	if (ltdb->cache->attributes == NULL) {
+		/* no previously loaded attributes */
+		return;
+	}
+
+	msg = ltdb->cache->attributes;
+	for (i=0;i<msg->num_elements;i++) {
+		const struct ldb_attrib_handler *h;
+		/* this is rather ugly - a consequence of const handling */
+		h = ldb_attrib_handler(module->ldb, msg->elements[i].name);
+		ldb_remove_attrib_handler(module->ldb, msg->elements[i].name);
+		if (strcmp(h->attr, msg->elements[i].name) == 0) {
+			talloc_steal(msg, h->attr);
+		}
+	}
+
+	talloc_free(ltdb->cache->attributes);
+	ltdb->cache->attributes = NULL;
+}
+
+/*
+  add up the attrib flags for a @ATTRIBUTES element
+*/
+static int ltdb_attributes_flags(struct ldb_message_element *el, unsigned *v)
+{
+	int i;
+	unsigned value = 0;
+	for (i=0;i<el->num_values;i++) {
+		int j;
+		for (j=0;ltdb_valid_attr_flags[j].name;j++) {
+			if (strcmp(ltdb_valid_attr_flags[j].name, 
+				   el->values[i].data) == 0) {
+				value |= ltdb_valid_attr_flags[j].value;
+				break;
+			}
+		}
+		if (ltdb_valid_attr_flags[j].name == NULL) {
+			return -1;
+		}
+	}
+	*v = value;
+	return 0;
+}
+
+/*
+  register any special handlers from @ATTRIBUTES
+*/
+static int ltdb_attributes_load(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb = module->private_data;
+	struct ldb_message *msg = ltdb->cache->attributes;
+	int i;
+
+	if (ltdb_search_dn1(module, LTDB_ATTRIBUTES, msg) == -1) {
+		goto failed;
+	}
+	/* mapping these flags onto ldap 'syntaxes' isn't strictly correct,
+	   but its close enough for now */
+	for (i=0;i<msg->num_elements;i++) {
+		unsigned flags;
+		const char *syntax;
+		const struct ldb_attrib_handler *h;
+		struct ldb_attrib_handler h2;
+
+		if (ltdb_attributes_flags(&msg->elements[i], &flags) != 0) {
+			ldb_debug(module->ldb, LDB_DEBUG_ERROR, "Invalid @ATTRIBUTES element for '%s'\n", msg->elements[i].name);
+			goto failed;
+		}
+		switch (flags & ~LTDB_FLAG_HIDDEN) {
+		case 0:
+			syntax = LDB_SYNTAX_OCTET_STRING;
+			break;
+		case LTDB_FLAG_WILDCARD:
+		case LTDB_FLAG_WILDCARD | LTDB_FLAG_CASE_INSENSITIVE:
+			syntax = LDB_SYNTAX_WILDCARD;
+			break;
+		case LTDB_FLAG_CASE_INSENSITIVE:
+			syntax = LDB_SYNTAX_DIRECTORY_STRING;
+			break;
+		case LTDB_FLAG_INTEGER:
+			syntax = LDB_SYNTAX_INTEGER;
+			break;
+		default:
+			ldb_debug(module->ldb, LDB_DEBUG_ERROR, 
+				  "Invalid flag combination 0x%x for '%s' in @ATTRIBUTES\n",
+				  flags, msg->elements[i].name);
+			goto failed;
+		}
+
+		h = ldb_attrib_handler_syntax(module->ldb, syntax);
+		if (h == NULL) {
+			ldb_debug(module->ldb, LDB_DEBUG_ERROR, 
+				  "Invalid attribute syntax '%s' for '%s' in @ATTRIBUTES\n",
+				  syntax, msg->elements[i].name);
+			goto failed;
+		}
+		h2 = *h;
+		h2.attr = talloc_strdup(module, msg->elements[i].name);
+		if (ldb_set_attrib_handlers(module->ldb, &h2, 1) != 0) {
+			goto failed;
+		}
+	}
+
+	return 0;
+failed:
+	return -1;
+}
+
+
+/*
+  register any subclasses from @SUBCLASSES
+*/
+static int ltdb_subclasses_load(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb = module->private_data;
+	struct ldb_message *msg = ltdb->cache->subclasses;
+	int i, j;
+
+	if (ltdb_search_dn1(module, LTDB_SUBCLASSES, msg) == -1) {
+		goto failed;
+	}
+
+	for (i=0;i<msg->num_elements;i++) {
+		struct ldb_message_element *el = &msg->elements[i];
+		for (j=0;j<el->num_values;j++) {
+			if (ldb_subclass_add(module->ldb, el->name, el->values[j].data) != 0) {
+				goto failed;
+			}
+		}
+	}
+
+	return 0;
+failed:
+	return -1;
+}
+
+
+/*
+  de-register any @SUBCLASSES
+*/
+static void ltdb_subclasses_unload(struct ldb_module *module)
+{
+	struct ltdb_private *ltdb = module->private_data;
+	struct ldb_message *msg;
+	int i;
+
+	if (ltdb->cache->subclasses == NULL) {
+		/* no previously loaded subclasses */
+		return;
+	}
+
+	msg = ltdb->cache->subclasses;
+	for (i=0;i<msg->num_elements;i++) {
+		ldb_subclass_remove(module->ldb, msg->elements[i].name);
+	}
+
+	talloc_free(ltdb->cache->subclasses);
+	ltdb->cache->subclasses = NULL;
+}
+
+
+/*
   initialise the baseinfo record
 */
 static int ltdb_baseinfo_init(struct ldb_module *module)
@@ -122,6 +297,8 @@ static void ltdb_cache_free(struct ldb_module *module)
 */
 int ltdb_cache_reload(struct ldb_module *module)
 {
+	ltdb_attributes_unload(module);
+	ltdb_subclasses_unload(module);
 	ltdb_cache_free(module);
 	return ltdb_cache_load(module);
 }
@@ -176,9 +353,11 @@ int ltdb_cache_load(struct ldb_module *module)
 	talloc_free(ltdb->cache->last_attribute.name);
 	memset(&ltdb->cache->last_attribute, 0, sizeof(ltdb->cache->last_attribute));
 
+	ltdb_attributes_unload(module);
+	ltdb_subclasses_unload(module);
+
 	talloc_free(ltdb->cache->indexlist);
 	talloc_free(ltdb->cache->subclasses);
-	talloc_free(ltdb->cache->attributes);
 
 	ltdb->cache->indexlist = talloc_zero(ltdb->cache, struct ldb_message);
 	ltdb->cache->subclasses = talloc_zero(ltdb->cache, struct ldb_message);
@@ -192,10 +371,11 @@ int ltdb_cache_load(struct ldb_module *module)
 	if (ltdb_search_dn1(module, LTDB_INDEXLIST, ltdb->cache->indexlist) == -1) {
 		goto failed;
 	}
-	if (ltdb_search_dn1(module, LTDB_SUBCLASSES, ltdb->cache->subclasses) == -1) {
+
+	if (ltdb_attributes_load(module) == -1) {
 		goto failed;
 	}
-	if (ltdb_search_dn1(module, LTDB_ATTRIBUTES, ltdb->cache->attributes) == -1) {
+	if (ltdb_subclasses_load(module) == -1) {
 		goto failed;
 	}
 
diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c
index 00b124a9cf..4d8a14f7f0 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_index.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_index.c
@@ -101,17 +101,45 @@ static char *ldb_dn_key(struct ldb_context *ldb,
 			const char *attr, const struct ldb_val *value)
 {
 	char *ret = NULL;
+	struct ldb_val v;
+	const struct ldb_attrib_handler *h;
+	char *attr_folded;
 
-	if (ldb_should_b64_encode(value)) {
+	attr_folded = ldb_casefold(ldb, attr);
+	if (!attr_folded) {
+		return NULL;
+	}
+
+	h = ldb_attrib_handler(ldb, attr);
+	if (h->canonicalise_fn(ldb, value, &v) != 0) {
+		/* canonicalisation can be refused. For example, 
+		   a attribute that takes wildcards will refuse to canonicalise
+		   if the value contains a wildcard */
+		talloc_free(attr_folded);
+		return NULL;
+	}
+	
+	if (ldb_should_b64_encode(&v)) {
 		char *vstr = ldb_base64_encode(ldb, value->data, value->length);
 		if (!vstr) return NULL;
-		ret = talloc_asprintf(ldb, "%s:%s::%s", LTDB_INDEX, attr, vstr);
+		ret = talloc_asprintf(ldb, "%s:%s::%s", LTDB_INDEX, attr_folded, vstr);
 		talloc_free(vstr);
+		if (v.data != value->data) {
+			talloc_free(v.data);
+		}
+		talloc_free(attr_folded);
 		return ret;
 	}
 
-	return talloc_asprintf(ldb, "%s:%s:%.*s", 
-			       LTDB_INDEX, attr, value->length, (char *)value->data);
+	ret = talloc_asprintf(ldb, "%s:%s:%.*s", 
+			      LTDB_INDEX, attr_folded, v.length, (char *)v.data);
+
+	if (v.data != value->data) {
+		talloc_free(v.data);
+	}
+	talloc_free(attr_folded);
+
+	return ret;
 }
 
 /*
@@ -234,46 +262,50 @@ static int ltdb_index_dn_objectclass(struct ldb_module *module,
 				     struct dn_list *list)
 {
 	struct ldb_context *ldb = module->ldb;
-	struct ltdb_private *ltdb = module->private_data;
 	unsigned int i;
 	int ret;
 	const char *target = tree->u.simple.value.data;
+	const char **subclasses;
 
 	list->count = 0;
 	list->dn = NULL;
 
 	ret = ltdb_index_dn_simple(module, tree, index_list, list);
 
-	for (i=0;i<ltdb->cache->subclasses->num_elements;i++) {
-		struct ldb_message_element *el = &ltdb->cache->subclasses->elements[i];
-		if (ldb_attr_cmp(el->name, target) == 0) {
-			unsigned int j;
-			for (j=0;j<el->num_values;j++) {
-				struct ldb_parse_tree tree2;
-				struct dn_list *list2;
-				tree2.operation = LDB_OP_SIMPLE;
-				tree2.u.simple.attr = talloc_strdup(list, LTDB_OBJECTCLASS);
-				if (!tree2.u.simple.attr) {
-					return -1;
-				}
-				tree2.u.simple.value = el->values[j];
-				list2 = talloc(list, struct dn_list);
-				if (list2 == NULL) {
-					return -1;
-				}
-				if (ltdb_index_dn_objectclass(module, &tree2, 
-							      index_list, list2) == 1) {
-					if (list->count == 0) {
-						*list = *list2;
-						ret = 1;
-					} else {
-						list_union(ldb, list, list2);
-						talloc_free(list2);
-					}
-				}
-				talloc_free(tree2.u.simple.attr);
+	subclasses = ldb_subclass_list(module->ldb, target);
+
+	if (subclasses == NULL) {
+		return ret;
+	}
+
+	for (i=0;subclasses[i];i++) {
+		struct ldb_parse_tree tree2;
+		struct dn_list *list2;
+		tree2.operation = LDB_OP_SIMPLE;
+		tree2.u.simple.attr = talloc_strdup(list, LTDB_OBJECTCLASS);
+		if (!tree2.u.simple.attr) {
+			return -1;
+		}
+		tree2.u.simple.value.data = talloc_strdup(tree2.u.simple.attr, subclasses[i]);
+		if (tree2.u.simple.value.data == NULL) {
+			return -1;			
+		}
+		tree2.u.simple.value.length = strlen(subclasses[i]);
+		list2 = talloc(list, struct dn_list);
+		if (list2 == NULL) {
+			return -1;
+		}
+		if (ltdb_index_dn_objectclass(module, &tree2, 
+					      index_list, list2) == 1) {
+			if (list->count == 0) {
+				*list = *list2;
+				ret = 1;
+			} else {
+				list_union(ldb, list, list2);
+				talloc_free(list2);
 			}
 		}
+		talloc_free(tree2.u.simple.attr);
 	}
 
 	return ret;
@@ -607,7 +639,7 @@ static int ldb_index_filter(struct ldb_module *module, struct ldb_parse_tree *tr
 		}
 
 		ret = 0;
-		if (ltdb_message_match(module, msg, tree, base, scope) == 1) {
+		if (ldb_match_message(module->ldb, msg, tree, base, scope) == 1) {
 			ret = ltdb_add_attr_results(module, msg, attrs, &count, res);
 		}
 		talloc_free(msg);
@@ -799,6 +831,10 @@ int ltdb_index_add(struct ldb_module *module, const struct ldb_message *msg)
 	int ret;
 	unsigned int i, j;
 
+	if (msg->dn[0] == '@') {
+		return 0;
+	}
+
 	if (ltdb->cache->indexlist->num_elements == 0) {
 		/* no indexed fields */
 		return 0;
@@ -834,6 +870,10 @@ int ltdb_index_del_value(struct ldb_module *module, const char *dn,
 	int ret, i;
 	unsigned int j;
 
+	if (dn[0] == '@') {
+		return 0;
+	}
+
 	dn_key = ldb_dn_key(ldb, el->name, &el->values[v_idx]);
 	if (!dn_key) {
 		return -1;
@@ -895,6 +935,10 @@ int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg)
 	int ret;
 	unsigned int i, j;
 
+	if (msg->dn[0] == '@') {
+		return 0;
+	}
+
 	/* find the list of indexed fields */	
 	if (ltdb->cache->indexlist->num_elements == 0) {
 		/* no indexed fields */
diff --git a/source4/lib/ldb/ldb_tdb/ldb_match.c b/source4/lib/ldb/ldb_tdb/ldb_match.c
deleted file mode 100644
index b5b023bc09..0000000000
--- a/source4/lib/ldb/ldb_tdb/ldb_match.c
+++ /dev/null
@@ -1,426 +0,0 @@
-/* 
-   ldb database library
-
-   Copyright (C) Andrew Tridgell  2004
-
-     ** NOTE! The following LGPL license applies to the ldb
-     ** library. This does NOT imply that all of Samba is released
-     ** under the LGPL
-   
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
-
-/*
- *  Name: ldb
- *
- *  Component: ldb expression matching
- *
- *  Description: ldb expression matching for tdb backend
- *
- *  Author: Andrew Tridgell
- */
-
-#include "includes.h"
-#include "ldb/include/ldb.h"
-#include "ldb/include/ldb_private.h"
-#include "ldb/ldb_tdb/ldb_tdb.h"
-#include <fnmatch.h>
-
-/*
-  see if two ldb_val structures contain the same data as integers
-  return 1 for a match, 0 for a mis-match
-*/
-static int ltdb_val_equal_integer(const struct ldb_val *v1, const struct ldb_val *v2)
-{
-	uint64_t i1, i2;
-
-	i1 = strtoull(v1->data, NULL, 0);
-	i2 = strtoull(v2->data, NULL, 0);
-
-	return i1 == i2;
-}
-
-/*
-  see if two ldb_val structures contain the same data as case insensitive strings
-  return 1 for a match, 0 for a mis-match
-*/
-static int ltdb_val_equal_case_insensitive(const struct ldb_val *v1, 
-					  const struct ldb_val *v2)
-{
-	if (v1->length != v2->length) {
-		return 0;
-	}
-	if (strncasecmp(v1->data, v2->data, v1->length) == 0) {
-		return 1;
-	}
-	return 0;
-}
-
-/*
-  see if two ldb_val structures contain the same data with wildcards 
-  and case insensitive
-  return 1 for a match, 0 for a mis-match
-*/
-static int ltdb_val_equal_wildcard_ci(struct ldb_module *module,
-				     const struct ldb_val *v1, 
-				     const struct ldb_val *v2)
-{
-	struct ldb_context *ldb = module->ldb;
-	char *s1, *s2;
-	int ret;
-
-	if (!v1->data || !v2->data) {
-		return v1->data == v2->data;
-	}
-
-	s1 = ldb_casefold(ldb, v1->data);
-	if (!s1) {
-		return -1;
-	}
-
-	s2 = ldb_casefold(ldb, v2->data);
-	if (!s2) {
-		talloc_free(s1);
-		return -1;
-	}
-
-	ret = fnmatch(s2, s1, 0);
-
-	talloc_free(s1);
-	talloc_free(s2);
-
-	if (ret == 0) {
-		return 1;
-	}
-
-	return 0;
-}
-
-/*
-  see if two ldb_val structures contain the same data with wildcards
-  return 1 for a match, 0 for a mis-match
-*/
-static int ltdb_val_equal_wildcard(struct ldb_module *module,
-				  const struct ldb_val *v1, 
-				  const struct ldb_val *v2,
-				  int flags)
-{
-	if (flags & LTDB_FLAG_CASE_INSENSITIVE) {
-		return ltdb_val_equal_wildcard_ci(module, v1, v2);
-	}
-	if (!v1->data || !v2->data) {
-		return v1->data == v2->data;
-	}
-	if (fnmatch(v2->data, v1->data, 0) == 0) {
-		return 1;
-	}
-	return 0;
-}
-
-
-/*
-  see if two objectclasses are considered equal. This handles
-  the subclass attributes
-
-  v1 contains the in-database value, v2 contains the value
-  that the user gave
-
-  return 1 for a match, 0 for a mis-match
-*/
-static int ltdb_val_equal_objectclass(struct ldb_module *module, 
-				     const struct ldb_val *v1, const struct ldb_val *v2)
-{
-	struct ltdb_private *ltdb = module->private_data;
-	unsigned int i;
-
-	if (ltdb_val_equal_case_insensitive(v1, v2) == 1) {
-		return 1;
-	}
-
-	for (i=0;i<ltdb->cache->subclasses->num_elements;i++) {
-		struct ldb_message_element *el = &ltdb->cache->subclasses->elements[i];
-		if (ldb_attr_cmp(el->name, v2->data) == 0) {
-			unsigned int j;
-			for (j=0;j<el->num_values;j++) {
-				if (ltdb_val_equal_objectclass(module, v1, &el->values[j])) {
-					return 1;
-				}
-			}
-		}
-	}
-
-	return 0;
-}
-				     
-
-/*
-  see if two ldb_val structures contain the same data
-  
-  v1 contains the in-database value, v2 contains the value
-  that the user gave
-  
-  return 1 for a match, 0 for a mis-match
-*/
-int ltdb_val_equal(struct ldb_module *module,
-		  const char *attr_name,
-		  const struct ldb_val *v1, const struct ldb_val *v2)
-{
-	int flags = ltdb_attribute_flags(module, attr_name);
-
-	if (flags & LTDB_FLAG_OBJECTCLASS) {
-		return ltdb_val_equal_objectclass(module, v1, v2);
-	}
-
-	if (flags & LTDB_FLAG_INTEGER) {
-		return ltdb_val_equal_integer(v1, v2);
-	}
-
-	if (flags & LTDB_FLAG_WILDCARD) {
-		return ltdb_val_equal_wildcard(module, v1, v2, flags);
-	}
-
-	if (flags & LTDB_FLAG_CASE_INSENSITIVE) {
-		return ltdb_val_equal_case_insensitive(v1, v2);
-	}
-
-	if (v1->length != v2->length) return 0;
-
-	if (v1->length == 0) return 1;
-
-	if (memcmp(v1->data, v2->data, v1->length) == 0) {
-		return 1;
-	}
-
-	return 0;
-}
-
-/*
-  check if the scope matches in a search result
-*/
-static int scope_match(const char *dn, const char *base, enum ldb_scope scope)
-{
-	size_t dn_len, base_len;
-
-	if (base == NULL) {
-		return 1;
-	}
-
-	base_len = strlen(base);
-	dn_len = strlen(dn);
-
-	if (scope != LDB_SCOPE_ONELEVEL && ldb_dn_cmp(dn, base) == 0) {
-		return 1;
-	}
-
-	if (base_len+1 >= dn_len) {
-		return 0;
-	}
-
-	switch (scope) {
-	case LDB_SCOPE_BASE:
-		break;
-
-	case LDB_SCOPE_ONELEVEL:
-		if (ldb_dn_cmp(dn + (dn_len - base_len), base) == 0 &&
-		    dn[dn_len - base_len - 1] == ',' &&
-		    strchr(dn, ',') == &dn[dn_len - base_len - 1]) {
-			return 1;
-		}
-		break;
-		
-	case LDB_SCOPE_SUBTREE:
-	default:
-		if (ldb_dn_cmp(dn + (dn_len - base_len), base) == 0 &&
-		    dn[dn_len - base_len - 1] == ',') {
-			return 1;
-		}
-		break;
-	}
-
-	return 0;
-}
-
-
-/*
-  match a leaf node
-*/
-static int match_leaf(struct ldb_module *module, 
-		      struct ldb_message *msg,
-		      struct ldb_parse_tree *tree,
-		      const char *base,
-		      enum ldb_scope scope)
-{
-	unsigned int i;
-	struct ldb_message_element *el;
-
-	if (!scope_match(msg->dn, base, scope)) {
-		return 0;
-	}
-
-	if (ldb_attr_cmp(tree->u.simple.attr, "dn") == 0) {
-		if (strcmp(tree->u.simple.value.data, "*") == 0) {
-			return 1;
-		}
-		return ldb_dn_cmp(msg->dn, tree->u.simple.value.data) == 0;
-	}
-
-	el = ldb_msg_find_element(msg, tree->u.simple.attr);
-	if (el == NULL) {
-		return 0;
-	}
-
-	if (strcmp(tree->u.simple.value.data, "*") == 0) {
-		return 1;
-	}
-
-	for (i=0;i<el->num_values;i++) {
-		if (ltdb_val_equal(module, el->name, &el->values[i], 
-				   &tree->u.simple.value)) {
-			return 1;
-		}
-	}
-
-	return 0;
-}
-
-
-/*
-  bitwise-and comparator
-*/
-static int comparator_and(struct ldb_val *v1, struct ldb_val *v2)
-{
-	uint64_t i1, i2;
-	i1 = strtoull(v1->data, NULL, 0);
-	i2 = strtoull(v2->data, NULL, 0);
-	return ((i1 & i2) == i2);
-}
-
-/*
-  bitwise-or comparator
-*/
-static int comparator_or(struct ldb_val *v1, struct ldb_val *v2)
-{
-	uint64_t i1, i2;
-	i1 = strtoull(v1->data, NULL, 0);
-	i2 = strtoull(v2->data, NULL, 0);
-	return ((i1 & i2) != 0);
-}
-
-
-/*
-  extended match, handles things like bitops
-*/
-static int ltdb_extended_match(struct ldb_module *module, 
-			       struct ldb_message *msg,
-			       struct ldb_parse_tree *tree,
-			       const char *base,
-			       enum ldb_scope scope)
-{
-	int i;
-	const struct {
-		const char *oid;
-		int (*comparator)(struct ldb_val *, struct ldb_val *);
-	} rules[] = {
-		{ LDB_OID_COMPARATOR_AND, comparator_and},
-		{ LDB_OID_COMPARATOR_OR, comparator_or}
-	};
-	int (*comp)(struct ldb_val *, struct ldb_val *) = NULL;
-	struct ldb_message_element *el;
-
-	if (tree->u.extended.dnAttributes) {
-		ldb_debug(module->ldb, LDB_DEBUG_ERROR, "ldb: dnAttributes extended match not supported yet");
-		return -1;
-	}
-	if (tree->u.extended.rule_id == NULL) {
-		ldb_debug(module->ldb, LDB_DEBUG_ERROR, "ldb: no-rule extended matches not supported yet");
-		return -1;
-	}
-	if (tree->u.extended.attr == NULL) {
-		ldb_debug(module->ldb, LDB_DEBUG_ERROR, "ldb: no-attribute extended matches not supported yet");
-		return -1;
-	}
-
-	for (i=0;i<ARRAY_SIZE(rules);i++) {
-		if (strcmp(rules[i].oid, tree->u.extended.rule_id) == 0) {
-			comp = rules[i].comparator;
-			break;
-		}
-	}
-	if (comp == NULL) {
-		ldb_debug(module->ldb, LDB_DEBUG_ERROR, "ldb: unknown extended rule_id %s\n",
-			  tree->u.extended.rule_id);
-		return -1;
-	}
-
-	/* find the message element */
-	el = ldb_msg_find_element(msg, tree->u.extended.attr);
-	if (el == NULL) {
-		return 0;
-	}
-
-	for (i=0;i<el->num_values;i++) {
-		int ret = comp(&el->values[i], &tree->u.extended.value);
-		if (ret == -1 || ret == 1) return ret;
-	}
-
-	return 0;
-}
-
-/*
-  return 0 if the given parse tree matches the given message. Assumes
-  the message is in sorted order
-
-  return 1 if it matches, and 0 if it doesn't match
-
-  this is a recursive function, and does short-circuit evaluation
- */
-int ltdb_message_match(struct ldb_module *module, 
-		      struct ldb_message *msg,
-		      struct ldb_parse_tree *tree,
-		      const char *base,
-		      enum ldb_scope scope)
-{
-	unsigned int i;
-	int v;
-
-	switch (tree->operation) {
-	case LDB_OP_SIMPLE:
-		break;
-
-	case LDB_OP_EXTENDED:
-		return ltdb_extended_match(module, msg, tree, base, scope);
-
-	case LDB_OP_NOT:
-		return ! ltdb_message_match(module, msg, tree->u.not.child, base, scope);
-
-	case LDB_OP_AND:
-		for (i=0;i<tree->u.list.num_elements;i++) {
-			v = ltdb_message_match(module, msg, tree->u.list.elements[i],
-					      base, scope);
-			if (!v) return 0;
-		}
-		return 1;
-
-	case LDB_OP_OR:
-		for (i=0;i<tree->u.list.num_elements;i++) {
-			v = ltdb_message_match(module, msg, tree->u.list.elements[i],
-					      base, scope);
-			if (v) return 1;
-		}
-		return 0;
-	}
-
-	return match_leaf(module, msg, tree, base, scope);
-}
diff --git a/source4/lib/ldb/ldb_tdb/ldb_search.c b/source4/lib/ldb/ldb_tdb/ldb_search.c
index d6e7d66f68..e48043da88 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_search.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_search.c
@@ -92,8 +92,9 @@ static int msg_add_all_elements(struct ldb_module *module, struct ldb_message *r
 	unsigned int i;
 
 	for (i=0;i<msg->num_elements;i++) {
-		int flags = ltdb_attribute_flags(module, msg->elements[i].name);
-		if ((msg->dn[0] != '@') && (flags & LTDB_FLAG_HIDDEN)) {
+		const struct ldb_attrib_handler *h;
+		h = ldb_attrib_handler(ldb, msg->elements[i].name);
+		if ((msg->dn[0] != '@') && (h->flags & LDB_ATTR_FLAG_HIDDEN)) {
 			continue;
 		}
 		if (msg_add_element(ldb, ret, &msg->elements[i]) != 0) {
@@ -195,7 +196,7 @@ static struct ldb_message *ltdb_pull_attrs(struct ldb_module *module,
 int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name, 
 		      const struct ldb_val *val)
 {
-	int flags;
+	const struct ldb_attrib_handler *h;
 
 	/* all attribute types recognise the "*" wildcard */
 	if (val->length == 1 && strncmp((char *)val->data, "*", 1) == 0) {
@@ -206,8 +207,8 @@ int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name,
 		return 0;
 	}
 
-	flags = ltdb_attribute_flags(module, attr_name);
-	if (flags & LTDB_FLAG_WILDCARD) {
+	h = ldb_attrib_handler(module->ldb, attr_name);
+	if (h->flags & LDB_ATTR_FLAG_WILDCARD) {
 		return 1;
 	}
 
@@ -415,8 +416,8 @@ static int search_func(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, voi
 	}
 
 	/* see if it matches the given expression */
-	if (!ltdb_message_match(sinfo->module, msg, sinfo->tree, 
-				sinfo->base, sinfo->scope)) {
+	if (!ldb_match_message(sinfo->module->ldb, msg, sinfo->tree, 
+			       sinfo->base, sinfo->scope)) {
 		talloc_free(msg);
 		return 0;
 	}
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
index bc61378f18..22797b96d1 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -41,21 +41,10 @@
 #include "ldb/include/ldb_dn.h"
 #include "ldb/ldb_tdb/ldb_tdb.h"
 
-#define LDBLOCK	"INT_LDBLOCK"
+#define LDBLOCK	"@INT_LDBLOCK"
 
 
 /*
-  callback function used in call to ldb_dn_fold() for determining whether an
-  attribute type requires case folding.
-*/
-static int ltdb_case_fold_attr_required(void * user_data, char *attr)
-{
-	struct ldb_module *module = talloc_get_type(user_data, struct ldb_module);
-
-	return ltdb_attribute_flags(module, attr) & LTDB_FLAG_CASE_INSENSITIVE;
-}
-
-/*
   form a TDB_DATA for a record key
   caller frees
 
@@ -68,9 +57,6 @@ struct TDB_DATA ltdb_key(struct ldb_module *module, const char *dn)
 	TDB_DATA key;
 	char *key_str = NULL;
 	char *dn_folded = NULL;
-	const char *prefix = LTDB_INDEX ":";
-	const char *s;
-	int flags;
 
 	/*
 	  most DNs are case insensitive. The exception is index DNs for
@@ -78,52 +64,22 @@ struct TDB_DATA ltdb_key(struct ldb_module *module, const char *dn)
 
 	  there are 3 cases dealt with in this code:
 
-	  1) if the dn doesn't start with @INDEX: then uppercase the attribute
+	  1) if the dn doesn't start with @ then uppercase the attribute
              names and the attributes values of case insensitive attributes
-	  2) if the dn starts with @INDEX:attr and 'attr' is a case insensitive
-	     attribute then uppercase whole dn
-	  3) if the dn starts with @INDEX:attr and 'attr' is a case sensitive
-	     attribute then uppercase up to the value of the attribute, but 
-	     not the value itself
+	  2) if the dn starts with @ then leave it alone - the indexing code handles
+	     the rest
 	*/
-	if (strncmp(dn, prefix, strlen(prefix)) == 0 &&
-	    (s = strchr(dn+strlen(prefix), ':'))) {
-		char *attr_name, *attr_name_folded;
-		attr_name = talloc_strndup(ldb, dn+strlen(prefix), (s-(dn+strlen(prefix))));
-		if (!attr_name) {
-			goto failed;
-		}
-		flags = ltdb_attribute_flags(module, attr_name);
-		
-		if (flags & LTDB_FLAG_CASE_INSENSITIVE) {
-			dn_folded = ldb_casefold(ldb, dn);
-		} else {
-			attr_name_folded = ldb_casefold(ldb, attr_name);
-			if (!attr_name_folded) {
-				goto failed;
-			}
-			dn_folded = talloc_asprintf(ldb, "%s:%s:%s",
-						    prefix, attr_name_folded,
-						    s+1);
-			talloc_free(attr_name_folded);
-		}
-		talloc_free(attr_name);
-	}
-	/* special cases for tdb */
-	else if (*dn == '@' || strncmp(LDBLOCK, dn, strlen(LDBLOCK)) == 0) {
-
+	if (*dn == '@') {
 		dn_folded = talloc_strdup(ldb, dn);
-	}
-	else {
+	} else {
 		struct ldb_dn *edn, *cedn;
 
 		edn = ldb_dn_explode(ldb, dn);
 		if (!edn)
 			goto failed;
 
-		cedn = ldb_dn_casefold(ldb, edn, module,
-					ltdb_case_fold_attr_required);
-		if (!edn)
+		cedn = ldb_dn_casefold(ldb, edn);
+		if (!cedn)
 			goto failed;
 
                 dn_folded = ldb_dn_linearize(ldb, cedn);
@@ -563,6 +519,7 @@ static int msg_delete_element(struct ldb_module *module,
 	unsigned int i;
 	int found;
 	struct ldb_message_element *el;
+	const struct ldb_attrib_handler *h;
 
 	found = find_element(msg, name);
 	if (found == -1) {
@@ -571,8 +528,10 @@ static int msg_delete_element(struct ldb_module *module,
 
 	el = &msg->elements[found];
 
+	h = ldb_attrib_handler(ldb, el->name);
+
 	for (i=0;i<el->num_values;i++) {
-		if (ltdb_val_equal(module, msg->elements[i].name, &el->values[i], val)) {
+		if (h->comparison_fn(ldb, &el->values[i], val) == 0) {
 			if (i<el->num_values-1) {
 				memmove(&el->values[i], &el->values[i+1],
 					sizeof(el->values[i])*(el->num_values-(i+1)));
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
index 6377092a21..46c5843d60 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -45,20 +45,11 @@ struct ltdb_private {
 #define LTDB_SEQUENCE_NUMBER "sequenceNumber"
 #define LTDB_OBJECTCLASS "objectClass"
 
-/* well known attribute flags */
-#define LTDB_FLAG_CASE_INSENSITIVE (1<<0)
-#define LTDB_FLAG_INTEGER          (1<<1)
-#define LTDB_FLAG_WILDCARD         (1<<2)
-#define LTDB_FLAG_OBJECTCLASS      (1<<3)
-#define LTDB_FLAG_HIDDEN           (1<<4)
-#define LTDB_FLAG_NONE             0 
-
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_cache.c  */
 
 int ltdb_cache_reload(struct ldb_module *module);
 int ltdb_cache_load(struct ldb_module *module);
 int ltdb_increase_sequence_number(struct ldb_module *module);
-int ltdb_attribute_flags(struct ldb_module *module, const char *attr_name);
 int ltdb_check_at_attributes_values(const struct ldb_val *value);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_index.c  */
@@ -111,16 +102,6 @@ int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *ms
 int ltdb_lock_read(struct ldb_module *module);
 int ltdb_unlock_read(struct ldb_module *module);
 
-/* The following definitions come from lib/ldb/ldb_tdb/ldb_match.c  */
-int ltdb_val_equal(struct ldb_module *module,
-		  const char *attr_name,
-		  const struct ldb_val *v1, const struct ldb_val *v2);
-int ltdb_message_match(struct ldb_module *module, 
-		      struct ldb_message *msg,
-		      struct ldb_parse_tree *tree,
-		      const char *base,
-		      enum ldb_scope scope);
-
 int ltdb_index_del_value(struct ldb_module *module, const char *dn, 
 			 struct ldb_message_element *el, int v_idx);