Remove ad2oLschema, insted call it directly from provision-backend

This removes a level of indirection via external binaries in the
provision-backend code, and also makes better use of our internal code
for loading schema from an LDIF file.

Remaining to do: Sort the output again, as the load from LDIF is
unsorted (also needed because the normal LDB load from sorted input is too slow
anyway, and is only needed here).

Andrew Bartlett

7 files changed, 113 insertions, 225 deletions

diff --git a/source4/dsdb/config.mk b/source4/dsdb/config.mk
index 2ca4e4ca6d..6db2890738 100644
--- a/source4/dsdb/config.mk
+++ b/source4/dsdb/config.mk
@@ -37,7 +37,8 @@ SAMDB_SCHEMA_OBJ_FILES = $(addprefix $(dsdbsrcdir)/schema/, \
 		schema_set.o \
 		schema_query.o \
 		schema_syntax.o \
-		schema_description.o)
+		schema_description.o \
+		schema_convert_to_ol.o)
 
 $(eval $(call proto_header_template,$(dsdbsrcdir)/schema/proto.h,$(SAMDB_SCHEMA_OBJ_FILES:.o=.c)))
 # PUBLIC_HEADERS += dsdb/schema/schema.h
diff --git a/source4/utils/ad2oLschema.c b/source4/dsdb/schema/schema_convert_to_ol.c
index 236b1fa350..375e4e3810 100644
--- a/source4/utils/ad2oLschema.c
+++ b/source4/dsdb/schema/schema_convert_to_ol.c
@@ -1,124 +1,50 @@
 /* 
-   ldb database library
+   schema conversion routines
 
    Copyright (C) Andrew Bartlett 2006-2008
-
-     ** NOTE! The following LGPL license applies to the ldb
-     ** library. This does NOT imply that all of Samba is released
-     ** under the LGPL
+  
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
    
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+   
 */
 
-/*
- *  Name: ldb
- *
- *  Component: ad2oLschema
- *
- *  Description: utility to convert an AD schema into the format required by OpenLDAP
- *
- *  Author: Andrew Bartlett
- */
-
 #include "includes.h"
 #include "ldb.h"
-#include "system/locale.h"
-#include "lib/ldb/tools/cmdline.h"
-#include "param/param.h"
-#include "lib/cmdline/popt_common.h"
 #include "dsdb/samdb/samdb.h"
+#include "system/locale.h"
 
-struct schema_conv {
-	int count;
-	int skipped;
-	int failures;
-};
-	
+/* Routine to linearise our internal schema into the format that
+   OpenLDAP and Fedora DS use for their backend.  
 
-static void usage(void)
-{
-	printf("Usage: ad2oLschema <options>\n");
-	printf("\nConvert AD-like LDIF to OpenLDAP schema format\n\n");
-	printf("Options:\n");
-	printf("  -I inputfile     inputfile of mapped OIDs and skipped attributes/ObjectClasses");
-	printf("  -H url           LDB or LDAP server to read schmea from\n");
-	printf("  -O outputfile    outputfile otherwise STDOUT\n");
-	printf("  -o options       pass options like modules to activate\n");
-	printf("              e.g: -o modules:timestamps\n");
-	printf("\n");
-	printf("Converts records from an AD-like LDIF schema into an openLdap formatted schema\n\n");
-	exit(1);
-}
+   The 'mappings' are of a format like:
 
-static struct ldb_dn *find_schema_dn(struct ldb_context *ldb, TALLOC_CTX *mem_ctx) 
-{
-	const char *rootdse_attrs[] = {"schemaNamingContext", NULL};
-	struct ldb_dn *schemadn;
-	struct ldb_dn *basedn = ldb_dn_new(mem_ctx, ldb, NULL);
-	struct ldb_result *rootdse_res;
-	struct ldb_result *schema_res;
-	int ldb_ret;
-	
-	if (!basedn) {
-		return NULL;
-	}
-	
-	/* Search for rootdse */
-	ldb_ret = ldb_search(ldb, mem_ctx, &rootdse_res,
-			     basedn, LDB_SCOPE_BASE, rootdse_attrs, NULL);
-	if (ldb_ret != LDB_SUCCESS) {
-		ldb_ret = ldb_search(ldb, mem_ctx, &schema_res, basedn, LDB_SCOPE_SUBTREE,
-				     NULL, "(&(objectClass=dMD)(cn=Schema))");
-		if (ldb_ret) {
-			printf("cn=Schema Search failed: %s\n", ldb_errstring(ldb));
-			return NULL;
-		}
+#Standard OpenLDAP attributes
+labeledURI
+#The memberOf plugin provides this attribute
+memberOf
+#These conflict with OpenLDAP builtins
+attributeTypes:samba4AttributeTypes
+2.5.21.5:1.3.6.1.4.1.7165.4.255.7
 
-		if (schema_res->count != 1) {
-			talloc_free(schema_res);
-			printf("Failed to find rootDSE");
-			return NULL;
-		}
-		
-		schemadn = talloc_steal(mem_ctx, schema_res->msgs[0]->dn);
-		talloc_free(schema_res);
-		return schemadn;
-	}
-	
-	if (rootdse_res->count != 1) {
-		printf("Failed to find rootDSE");
-		talloc_free(rootdse_res);
-		return NULL;
-	}
-	
-	/* Locate schema */
-	schemadn = ldb_msg_find_attr_as_dn(ldb, mem_ctx, rootdse_res->msgs[0], "schemaNamingContext");
-	talloc_free(rootdse_res);
-
-	if (!schemadn) {
-		return NULL;
-	}
-
-	return schemadn;
-}
+*/
 
 
-static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_schema_convert_target target, FILE *in, FILE *out) 
+char *dsdb_convert_schema_to_openldap(struct ldb_context *ldb, char *target_str, const char *mappings) 
 {
 	/* Read list of attributes to skip, OIDs to map */
 	TALLOC_CTX *mem_ctx = talloc_new(ldb);
 	char *line;
+	char *out;
 	const char **attrs_skip = NULL;
 	int num_skip = 0;
 	struct oid_map {
@@ -133,19 +59,31 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 	int num_attr_maps = 0;	
 	struct dsdb_class *objectclass;
 	struct dsdb_attribute *attribute;
-	struct ldb_dn *schemadn;
-	struct schema_conv ret;
 	struct dsdb_schema *schema;
 	const char *seperator;
-	char *error_string;
+	enum dsdb_schema_convert_target target;
+
+	char *next_line = talloc_strdup(mem_ctx, mappings);
 
-	int ldb_ret;
+	if (!target_str || strcasecmp(target_str, "openldap") == 0) {
+		target = TARGET_OPENLDAP;
+	} else if (strcasecmp(target_str, "fedora-ds") == 0) {
+		target = TARGET_FEDORA_DS;
+	} else {
+		DEBUG(0, ("Invalid target type for schema conversion %s\n", target_str));
+		return NULL;
+	}
 
-	ret.count = 0;
-	ret.skipped = 0;
-	ret.failures = 0;
+	/* The mappings are line-seperated, and specify details such as OIDs to skip etc */
+	while (1) {
+		line = next_line;
+		next_line = strchr(line, '\n');
+		if (!next_line) {
+			break;
+		}
+		next_line[0] = '\0';
+		next_line++;
 
-	while ((line = afdgets(fileno(in), mem_ctx, 0))) {
 		/* Blank Line */
 		if (line[0] == '\0') {
 			continue;
@@ -154,17 +92,18 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 		if (line[0] == '#') {
 			continue;
 		}
+
 		if (isdigit(line[0])) {
 			char *p = strchr(line, ':');
 			if (!p) {
-				ret.failures++;
-				return ret;
+				DEBUG(0, ("schema mapping file line has OID but no OID to map to: %s\n", line));
+				return NULL;
 			}
 			p[0] = '\0';
 			p++;
 			oid_map = talloc_realloc(mem_ctx, oid_map, struct oid_map, num_oid_maps + 2);
 			trim_string(line, " ", " ");
-			oid_map[num_oid_maps].old_oid = talloc_move(oid_map, &line);
+			oid_map[num_oid_maps].old_oid = talloc_strdup(oid_map, line);
 			trim_string(p, " ", " ");
 			oid_map[num_oid_maps].new_oid = p;
 			num_oid_maps++;
@@ -177,7 +116,7 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 				p++;
 				attr_map = talloc_realloc(mem_ctx, attr_map, struct attr_map, num_attr_maps + 2);
 				trim_string(line, " ", " ");
-				attr_map[num_attr_maps].old_attr = talloc_move(attr_map, &line);
+				attr_map[num_attr_maps].old_attr = talloc_strdup(attr_map, line);
 				trim_string(p, " ", " ");
 				attr_map[num_attr_maps].new_attr = p;
 				num_attr_maps++;
@@ -186,36 +125,26 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 				/* skip attribute/objectClass */
 				attrs_skip = talloc_realloc(mem_ctx, attrs_skip, const char *, num_skip + 2);
 				trim_string(line, " ", " ");
-				attrs_skip[num_skip] = talloc_move(attrs_skip, &line);
+				attrs_skip[num_skip] = talloc_strdup(attrs_skip, line);
 				num_skip++;
 				attrs_skip[num_skip] = NULL;
 			}
 		}
 	}
 
-	schemadn = find_schema_dn(ldb, mem_ctx);
-	if (!schemadn) {
-		printf("Failed to find schema DN: %s\n", ldb_errstring(ldb));
-		ret.failures = 1;
-		return ret;
-	}
-	
-	ldb_ret = dsdb_schema_from_schema_dn(mem_ctx, ldb,
-					     lp_iconv_convenience(cmdline_lp_ctx),
-					     schemadn, &schema, &error_string);
-	if (ldb_ret != LDB_SUCCESS) {
-		printf("Failed to load schema: %s\n", error_string);
-		ret.failures = 1;
-		return ret;
+	schema = dsdb_get_schema(ldb);
+	if (!schema) {
+		DEBUG(0, ("No schema on ldb to convert!\n"));
+		return NULL;
 	}
-
 	switch (target) {
 	case TARGET_OPENLDAP:
 		seperator = "\n  ";
+		out = talloc_strdup(mem_ctx, "");
 		break;
 	case TARGET_FEDORA_DS:
 		seperator = "\n  ";
-		fprintf(out, "dn: cn=schema\n");
+		out = talloc_strdup(mem_ctx, "dn: cn=schema\n");
 		break;
 	}
 
@@ -231,7 +160,6 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 
 		/* We have been asked to skip some attributes/objectClasses */
 		if (attrs_skip && str_list_check_ci(attrs_skip, name)) {
-			ret.skipped++;
 			continue;
 		}
 
@@ -283,19 +211,18 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 							    false, false);
 
 		if (schema_entry == NULL) {
-			ret.failures++;
-			return ret;
+			DEBUG(0, ("failed to generate attribute description for %s\n", name));
+			return NULL;
 		}
 
 		switch (target) {
 		case TARGET_OPENLDAP:
-			fprintf(out, "attributetype %s\n\n", schema_entry);
+			out = talloc_asprintf_append(out, "attributetype %s\n\n", schema_entry);
 			break;
 		case TARGET_FEDORA_DS:
-			fprintf(out, "attributeTypes: %s\n", schema_entry);
+			out = talloc_asprintf_append(out, "attributeTypes: %s\n", schema_entry);
 			break;
 		}
-		ret.count++;
 	}
 
 	/* This is already sorted to have 'top' and similar classes first */
@@ -316,7 +243,6 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 		
 		/* We have been asked to skip some attributes/objectClasses */
 		if (attrs_skip && str_list_check_ci(attrs_skip, name)) {
-			ret.skipped++;
 			continue;
 		}
 
@@ -371,72 +297,20 @@ static struct schema_conv process_convert(struct ldb_context *ldb, enum dsdb_sch
 							may,
 							NULL);
 		if (schema_entry == NULL) {
-			ret.failures++;
-			return ret;
+			DEBUG(0, ("failed to generate schema description for %s\n", name));
+			return NULL;
 		}
 
 		switch (target) {
 		case TARGET_OPENLDAP:
-			fprintf(out, "objectclass %s\n\n", schema_entry);
+			out = talloc_asprintf_append(out, "objectclass %s\n\n", schema_entry);
 			break;
 		case TARGET_FEDORA_DS:
-			fprintf(out, "objectClasses: %s\n", schema_entry);
+			out = talloc_asprintf_append(out, "objectClasses: %s\n", schema_entry);
 			break;
 		}
-		ret.count++;
 	}
 
-	return ret;
+	return out;
 }
 
- int main(int argc, const char **argv)
-{
-	TALLOC_CTX *ctx;
-	struct ldb_cmdline *options;
-	FILE *in = stdin;
-	FILE *out = stdout;
-	struct ldb_context *ldb;
-	struct schema_conv ret;
-	const char *target_str;
-	enum dsdb_schema_convert_target target;
-
-	ctx = talloc_new(NULL);
-	ldb = ldb_init(ctx, NULL);
-
-	options = ldb_cmdline_process(ldb, argc, argv, usage);
-
-	if (options->input) {
-		in = fopen(options->input, "r");
-		if (!in) {
-			perror(options->input);
-			exit(1);
-		}
-	}
-	if (options->output) {
-		out = fopen(options->output, "w");
-		if (!out) {
-			perror(options->output);
-			exit(1);
-		}
-	}
-
-	target_str = lp_parm_string(cmdline_lp_ctx, NULL, "convert", "target");
-
-	if (!target_str || strcasecmp(target_str, "openldap") == 0) {
-		target = TARGET_OPENLDAP;
-	} else if (strcasecmp(target_str, "fedora-ds") == 0) {
-		target = TARGET_FEDORA_DS;
-	} else {
-		printf("Unsupported target: %s\n", target_str);
-		exit(1);
-	}
-
-	ret = process_convert(ldb, target, in, out);
-
-	fclose(in);
-	fclose(out);
-
-	printf("Converted %d records (skipped %d) with %d failures\n", ret.count, ret.skipped, ret.failures);
-
-	return 0;
-}
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index 6abd8a8f88..d52976958d 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -277,7 +277,7 @@ void dsdb_make_schema_global(struct ldb_context *ldb)
  * schema itself to the directory.
  */
 
-WERROR dsdb_attach_schema_from_ldif_file(struct ldb_context *ldb, const char *pf, const char *df)
+WERROR dsdb_attach_schema_from_ldif(struct ldb_context *ldb, const char *pf, const char *df)
 {
 	struct ldb_ldif *ldif;
 	struct ldb_message *msg;
diff --git a/source4/scripting/python/pyglue.c b/source4/scripting/python/pyglue.c
index a2c4790611..1480e54403 100644
--- a/source4/scripting/python/pyglue.c
+++ b/source4/scripting/python/pyglue.c
@@ -212,7 +212,7 @@ static PyObject *py_dsdb_set_global_schema(PyObject *self, PyObject *args)
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_dsdb_attach_schema_from_ldif_file(PyObject *self, PyObject *args)
+static PyObject *py_dsdb_attach_schema_from_ldif(PyObject *self, PyObject *args)
 {
 	WERROR result;
 	char *pf, *df;
@@ -224,12 +224,35 @@ static PyObject *py_dsdb_attach_schema_from_ldif_file(PyObject *self, PyObject *
 
 	PyErr_LDB_OR_RAISE(py_ldb, ldb);
 
-	result = dsdb_attach_schema_from_ldif_file(ldb, pf, df);
+	result = dsdb_attach_schema_from_ldif(ldb, pf, df);
 	PyErr_WERROR_IS_ERR_RAISE(result);
 
 	Py_RETURN_NONE;
 }
 
+static PyObject *py_dsdb_convert_schema_to_openldap(PyObject *self, PyObject *args)
+{
+	char *target_str, *mapping;
+	PyObject *py_ldb;
+	struct ldb_context *ldb;
+	PyObject *ret;
+	char *retstr;
+
+	if (!PyArg_ParseTuple(args, "Oss", &py_ldb, &target_str, &mapping))
+		return NULL;
+
+	PyErr_LDB_OR_RAISE(py_ldb, ldb);
+
+	retstr = dsdb_convert_schema_to_openldap(ldb, target_str, mapping);
+	if (!retstr) {
+		PyErr_SetString(PyExc_RuntimeError, "dsdb_convert_schema_to_openldap failed");
+		return NULL;
+	} 
+	ret = PyString_FromString(retstr);
+	talloc_free(retstr);
+	return ret;
+}
+
 static PyMethodDef py_misc_methods[] = {
 	{ "generate_random_str", (PyCFunction)py_generate_random_str, METH_VARARGS,
 		"random_password(len) -> string\n"
@@ -255,7 +278,9 @@ static PyMethodDef py_misc_methods[] = {
 		NULL },
 	{ "dsdb_set_global_schema", (PyCFunction)py_dsdb_set_global_schema, METH_VARARGS,
 		NULL },
-	{ "dsdb_attach_schema_from_ldif_file", (PyCFunction)py_dsdb_attach_schema_from_ldif_file, METH_VARARGS,
+	{ "dsdb_attach_schema_from_ldif", (PyCFunction)py_dsdb_attach_schema_from_ldif, METH_VARARGS,
+		NULL },
+	{ "dsdb_convert_schema_to_openldap", (PyCFunction)py_dsdb_convert_schema_to_openldap, METH_VARARGS,
 		NULL },
 	{ NULL }
 };
diff --git a/source4/scripting/python/samba/provision.py b/source4/scripting/python/samba/provision.py
index 8029565399..ed6548b13b 100644
--- a/source4/scripting/python/samba/provision.py
+++ b/source4/scripting/python/samba/provision.py
@@ -1245,7 +1245,7 @@ def provision_backend(setup_dir=None, message=None,
     except OSError:
         pass
 
-    schemadb = Ldb(schemadb_path, lp=lp)
+    schemadb = SamDB(schemadb_path, lp=lp)
  
     prefixmap = open(setup_path("prefixMap.txt"), 'r').read()
 
@@ -1263,10 +1263,8 @@ def provision_backend(setup_dir=None, message=None,
                            "PREFIXMAP_B64": b64encode(prefixmap)
                            })
     
-    setup_add_ldif(schemadb, setup_path("schema_samba4.ldif"), 
-                   {"SCHEMADN": names.schemadn })
-
-    data = get_schema_data(setup_path, {"SCHEMADN": names.schemadn})
+    data = load_schema(setup_path, schemadb, names.schemadn, names.netbiosname, 
+                       names.configdn, DEFAULTSITE, names.serverdn)
     schemadb.add_ldif(data)
 
     if ldap_backend_type == "fedora-ds":
@@ -1480,10 +1478,10 @@ def provision_backend(setup_dir=None, message=None,
 
         ldapuser = "--username=samba-admin"
 
-            
-    schema_command = "bin/ad2oLschema --option=convert:target=" + ldap_backend_type + " -I " + setup_path(mapping) + " -H tdb://" + schemadb_path + " -O " + os.path.join(paths.ldapdir, backend_schema)
-            
-    os.system(schema_command)
+
+    backend_schema_data = schemadb.convert_schema_to_openldap(ldap_backend_type, open(setup_path(mapping), 'r').read())
+    assert backend_schema_data is not None
+    open(os.path.join(paths.ldapdir, backend_schema), 'w').write(backend_schema_data)
 
     message("Your %s Backend for Samba4 is now configured, and is ready to be started" % ldap_backend_type)
     message("Server Role:         %s" % serverrole)
@@ -1646,7 +1644,7 @@ def create_krb5_conf(path, setup_path, dnsdomain, hostname, realm):
 
 
 def load_schema(setup_path, samdb, schemadn, netbiosname, configdn, sitename,
-                serverdn, servername):
+                serverdn):
     """Load schema for the SamDB.
     
     :param samdb: Load a schema into a SamDB.
@@ -1655,7 +1653,6 @@ def load_schema(setup_path, samdb, schemadn, netbiosname, configdn, sitename,
     :param netbiosname: NetBIOS name of the host.
     :param configdn: DN of the configuration
     :param serverdn: DN of the server
-    :param servername: Host name of the server
 
     Returns the schema data loaded, to avoid double-parsing when then needing to add it to the db
     """
@@ -1674,7 +1671,6 @@ def load_schema(setup_path, samdb, schemadn, netbiosname, configdn, sitename,
                     "DEFAULTSITE": sitename,
                     "PREFIXMAP_B64": prefixmap,
                     "SERVERDN": serverdn,
-                    "SERVERNAME": servername,
     })
     check_all_substituted(head_data)
     samdb.attach_schema_from_ldif(head_data, schema_data)
@@ -1685,6 +1681,8 @@ def get_schema_data(setup_path, subst_vars = None):
 
     :param setup_path: Setup path function.
     :param subst_vars: Optional variables to substitute in the file.
+
+    Returns the schema data after substitution
     """ 
 
     # this data used to be read from schema.ldif
diff --git a/source4/scripting/python/samba/samdb.py b/source4/scripting/python/samba/samdb.py
index 614970d3ec..881f5912fb 100644
--- a/source4/scripting/python/samba/samdb.py
+++ b/source4/scripting/python/samba/samdb.py
@@ -198,7 +198,10 @@ userAccountControl: %u
         glue.samdb_set_domain_sid(self, sid)
 
     def attach_schema_from_ldif(self, pf, df):
-        glue.dsdb_attach_schema_from_ldif_file(self, pf, df)
+        glue.dsdb_attach_schema_from_ldif(self, pf, df)
+
+    def convert_schema_to_openldap(self, target, mapping):
+        return glue.dsdb_convert_schema_to_openldap(self, target, mapping)
 
     def set_invocation_id(self, invocation_id):
         """Set the invocation id for this SamDB handle.
diff --git a/source4/utils/config.mk b/source4/utils/config.mk
index ea8e00ff8a..5fa7e200f0 100644
--- a/source4/utils/config.mk
+++ b/source4/utils/config.mk
@@ -92,16 +92,3 @@ oLschema2ldif_OBJ_FILES = $(addprefix $(utilssrcdir)/, oLschema2ldif.o)
 
 MANPAGES += $(utilssrcdir)/man/oLschema2ldif.1
 
-################################################
-# Start BINARY  ad2oLschema
-[BINARY::ad2oLschema]
-INSTALLDIR = BINDIR
-PRIVATE_DEPENDENCIES = \
-		LIBLDB_CMDLINE SAMDB 
-# End BINARY ad2oLschema
-################################################
-
-ad2oLschema_OBJ_FILES = $(addprefix $(utilssrcdir)/, ad2oLschema.o)
-
-MANPAGES += $(utilssrcdir)/man/ad2oLschema.1
-