Merge branch 'wspp-schema'

4 files changed, 364 insertions, 42 deletions

diff --git a/source4/scripting/python/pyglue.c b/source4/scripting/python/pyglue.c
index a2c4790611..1480e54403 100644
--- a/source4/scripting/python/pyglue.c
+++ b/source4/scripting/python/pyglue.c
@@ -212,7 +212,7 @@ static PyObject *py_dsdb_set_global_schema(PyObject *self, PyObject *args)
 	Py_RETURN_NONE;
 }
 
-static PyObject *py_dsdb_attach_schema_from_ldif_file(PyObject *self, PyObject *args)
+static PyObject *py_dsdb_attach_schema_from_ldif(PyObject *self, PyObject *args)
 {
 	WERROR result;
 	char *pf, *df;
@@ -224,12 +224,35 @@ static PyObject *py_dsdb_attach_schema_from_ldif_file(PyObject *self, PyObject *
 
 	PyErr_LDB_OR_RAISE(py_ldb, ldb);
 
-	result = dsdb_attach_schema_from_ldif_file(ldb, pf, df);
+	result = dsdb_attach_schema_from_ldif(ldb, pf, df);
 	PyErr_WERROR_IS_ERR_RAISE(result);
 
 	Py_RETURN_NONE;
 }
 
+static PyObject *py_dsdb_convert_schema_to_openldap(PyObject *self, PyObject *args)
+{
+	char *target_str, *mapping;
+	PyObject *py_ldb;
+	struct ldb_context *ldb;
+	PyObject *ret;
+	char *retstr;
+
+	if (!PyArg_ParseTuple(args, "Oss", &py_ldb, &target_str, &mapping))
+		return NULL;
+
+	PyErr_LDB_OR_RAISE(py_ldb, ldb);
+
+	retstr = dsdb_convert_schema_to_openldap(ldb, target_str, mapping);
+	if (!retstr) {
+		PyErr_SetString(PyExc_RuntimeError, "dsdb_convert_schema_to_openldap failed");
+		return NULL;
+	} 
+	ret = PyString_FromString(retstr);
+	talloc_free(retstr);
+	return ret;
+}
+
 static PyMethodDef py_misc_methods[] = {
 	{ "generate_random_str", (PyCFunction)py_generate_random_str, METH_VARARGS,
 		"random_password(len) -> string\n"
@@ -255,7 +278,9 @@ static PyMethodDef py_misc_methods[] = {
 		NULL },
 	{ "dsdb_set_global_schema", (PyCFunction)py_dsdb_set_global_schema, METH_VARARGS,
 		NULL },
-	{ "dsdb_attach_schema_from_ldif_file", (PyCFunction)py_dsdb_attach_schema_from_ldif_file, METH_VARARGS,
+	{ "dsdb_attach_schema_from_ldif", (PyCFunction)py_dsdb_attach_schema_from_ldif, METH_VARARGS,
+		NULL },
+	{ "dsdb_convert_schema_to_openldap", (PyCFunction)py_dsdb_convert_schema_to_openldap, METH_VARARGS,
 		NULL },
 	{ NULL }
 };
diff --git a/source4/scripting/python/samba/ms_schema.py b/source4/scripting/python/samba/ms_schema.py
new file mode 100644
index 0000000000..2e8050e503
--- /dev/null
+++ b/source4/scripting/python/samba/ms_schema.py
@@ -0,0 +1,275 @@
+#!/usr/bin/env python
+#
+# create schema.ldif (as a string) from WSPP documentation
+#
+# based on minschema.py and minschema_wspp
+#
+
+import re
+import base64
+
+bitFields = {}
+
+# ADTS: 2.2.9
+# bit positions as labeled in the docs
+bitFields["searchflags"] = {
+    'fATTINDEX': 31,         # IX
+    'fPDNTATTINDEX': 30,     # PI
+    'fANR': 29, #AR
+    'fPRESERVEONDELETE': 28,         # PR
+    'fCOPY': 27,     # CP
+    'fTUPLEINDEX': 26,       # TP
+    'fSUBTREEATTINDEX': 25,  # ST
+    'fCONFIDENTIAL': 24,     # CF
+    'fNEVERVALUEAUDIT': 23,  # NV
+    'fRODCAttribute': 22,    # RO
+
+
+    # missing in ADTS but required by LDIF
+    'fRODCFilteredAttribute': 22,    # RO ?
+    'fCONFIDENTAIL': 24, # typo
+    'fRODCFILTEREDATTRIBUTE': 22 # case
+    }
+
+# ADTS: 2.2.10
+bitFields["systemflags"] = {
+    'FLAG_ATTR_NOT_REPLICATED': 31, 'FLAG_CR_NTDS_NC': 31, 	# NR
+    'FLAG_ATTR_REQ_PARTIAL_SET_MEMBER': 30, 'FLAG_CR_NTDS_DOMAIN': 30, 	# PS
+    'FLAG_ATTR_IS_CONSTRUCTED': 29, 'FLAG_CR_NTDS_NOT_GC_REPLICATED': 29, 	# CS
+    'FLAG_ATTR_IS_OPERATIONAL': 28, 	# OP
+    'FLAG_SCHEMA_BASE_OBJECT': 27, 	# BS
+    'FLAG_ATTR_IS_RDN': 26, 	# RD
+    'FLAG_DISALLOW_MOVE_ON_DELETE': 6, 	# DE
+    'FLAG_DOMAIN_DISALLOW_MOVE': 5, 	# DM
+    'FLAG_DOMAIN_DISALLOW_RENAME': 4, 	# DR
+    'FLAG_CONFIG_ALLOW_LIMITED_MOVE': 3, 	# AL
+    'FLAG_CONFIG_ALLOW_MOVE': 2, 	# AM
+    'FLAG_CONFIG_ALLOW_RENAME': 1, 	# AR
+    'FLAG_DISALLOW_DELETE': 0 	# DD
+    }
+
+# ADTS: 2.2.11
+bitFields["schemaflagsex"] = {
+    'FLAG_ATTR_IS_CRITICAL': 31
+    }
+
+# ADTS: 3.1.1.2.2.2
+oMObjectClassBER = {
+    '1.3.12.2.1011.28.0.702' : base64.b64encode('\x2B\x0C\x02\x87\x73\x1C\x00\x85\x3E'),
+    '1.2.840.113556.1.1.1.12': base64.b64encode('\x2A\x86\x48\x86\xF7\x14\x01\x01\x01\x0C'),
+    '2.6.6.1.2.5.11.29'      : base64.b64encode('\x56\x06\x01\x02\x05\x0B\x1D'),
+    '1.2.840.113556.1.1.1.11': base64.b64encode('\x2A\x86\x48\x86\xF7\x14\x01\x01\x01\x0B'),
+    '1.3.12.2.1011.28.0.714' : base64.b64encode('\x2B\x0C\x02\x87\x73\x1C\x00\x85\x4A'),
+    '1.3.12.2.1011.28.0.732' : base64.b64encode('\x2B\x0C\x02\x87\x73\x1C\x00\x85\x5C'),
+    '1.2.840.113556.1.1.1.6' : base64.b64encode('\x2A\x86\x48\x86\xF7\x14\x01\x01\x01\x06')
+}
+
+# separated by commas in docs, and must be broken up
+multivalued_attrs = set(["auxiliaryclass","maycontain","mustcontain","posssuperiors",
+                         "systemauxiliaryclass","systemmaycontain","systemmustcontain",
+                         "systemposssuperiors"])
+
+def __read_folded_line(f, buffer):
+    """ reads a line from an LDIF file, unfolding it"""
+    line = buffer
+    
+    while True:
+        l = f.readline()
+
+        if l[:1] == " ":
+            # continued line
+
+            # cannot fold an empty line
+            assert(line != "" and line != "\n")
+
+            # preserves '\n '
+            line = line + l
+        else:
+            # non-continued line            
+            if line == "":
+                line = l
+
+                if l == "":
+                    # eof, definitely won't be folded
+                    break
+            else:
+                # marks end of a folded line
+                # line contains the now unfolded line
+                # buffer contains the start of the next possibly folded line
+                buffer = l
+                break
+        
+    return (line, buffer)
+
+
+def __read_raw_entries(f):
+    """reads an LDIF entry, only unfolding lines"""
+
+    # will not match options after the attribute type
+    attr_type_re = re.compile("^([A-Za-z]+[A-Za-z0-9-]*):")
+
+    buffer = ""
+    
+    while True:
+        entry = []
+        
+        while True:
+            (l, buffer) = __read_folded_line(f, buffer)
+                        
+            if l[:1] == "#":
+                continue
+
+            if l == "\n" or l == "":
+                break
+
+            m = attr_type_re.match(l)
+
+            if m:
+                if l[-1:] == "\n":
+                    l = l[:-1]
+                    
+                entry.append(l)
+            else:
+                print >>sys.stderr, "Invalid line: %s" % l,
+                sys.exit(1)
+
+        if len(entry):
+            yield entry
+
+        if l == "":
+            break
+
+
+def fix_dn(dn):
+    """fix a string DN to use ${SCHEMADN}"""
+
+    # folding?
+    if dn.find("<RootDomainDN>") != -1:
+        dn = dn.replace("\n ", "")
+        dn = dn.replace(" ", "")
+        return dn.replace("CN=Schema,CN=Configuration,<RootDomainDN>", "${SCHEMADN}")
+    else:
+        return dn
+
+def __convert_bitfield(key, value):
+    """Evaluate the OR expression in 'value'"""
+    assert(isinstance(value, str))
+
+    value = value.replace("\n ", "")
+    value = value.replace(" ", "")
+    
+    try:
+        # some attributes already have numeric values
+        o = int(value)
+    except ValueError:
+        o = 0
+        flags = value.split("|")
+        for f in flags:
+            bitpos = bitFields[key][f]
+            o = o | (1 << (31 - bitpos))
+
+    return str(o)
+
+def __write_ldif_one(entry):
+    """Write out entry as LDIF"""
+    out = []
+    
+    for l in entry:
+        if isinstance(l[1], str):
+            vl = [l[1]]
+        else:
+            vl = l[1]
+
+        if l[0].lower() == 'omobjectclass':
+            out.append("%s:: %s" % (l[0], l[1]))
+            continue
+            
+        for v in vl:
+            out.append("%s: %s" % (l[0], v))
+
+
+    return "\n".join(out)
+    
+def __transform_entry(entry, objectClass):
+    """Perform transformations required to convert the LDIF-like schema
+       file entries to LDIF, including Samba-specific stuff."""
+    
+    entry = [l.split(":", 1) for l in entry]
+
+    cn = ""
+    
+    for l in entry:
+        key = l[0].lower()
+        l[1] = l[1].lstrip()
+        l[1] = l[1].rstrip()
+
+        if not cn and key == "cn":
+            cn = l[1]
+        
+        if key in multivalued_attrs:
+            # unlike LDIF, these are comma-separated
+            l[1] = l[1].replace("\n ", "")
+            l[1] = l[1].replace(" ", "")
+
+            l[1] = l[1].split(",")
+            
+        if key in bitFields:
+            l[1] = __convert_bitfield(key, l[1])
+
+        if key == "omobjectclass":
+            l[1] = oMObjectClassBER[l[1].strip()]
+
+        if isinstance(l[1], str):
+            l[1] = fix_dn(l[1])
+
+
+    assert(cn)
+    entry.insert(0, ["dn", "CN=%s,${SCHEMADN}" % cn])
+    entry.insert(1, ["objectClass", ["top", objectClass]])
+    
+    for l in entry:
+        key = l[0].lower()
+
+        if key == "cn":
+            entry.remove(l)
+
+    return entry
+
+def __parse_schema_file(filename, objectClass):
+    """Load and transform a schema file."""
+
+    out = []
+    
+    f = open(filename, "rU")
+    for entry in __read_raw_entries(f):
+        out.append(__write_ldif_one(__transform_entry(entry, objectClass)))
+
+    return "\n\n".join(out)
+
+
+def read_ms_schema(attr_file, classes_file, dump_attributes = True, dump_classes = True, debug = False):
+    """Read WSPP documentation-derived schema files."""
+
+    attr_ldif = ""
+    classes_ldif = ""
+    
+    if dump_attributes:
+        attr_ldif =  __parse_schema_file(attr_file, "attributeSchema")
+    if dump_classes:
+        classes_ldif = __parse_schema_file(classes_file, "classSchema")
+
+    return attr_ldif + "\n\n" + classes_ldif + "\n\n"
+
+if __name__ == '__main__':
+    import sys
+
+    try:
+        attr_file = sys.argv[1]
+        classes_file = sys.argv[2]
+    except IndexError:
+        print >>sys.stderr, "Usage: %s attr-file.txt classes-file.txt" % (sys.argv[0])
+        sys.exit(1)
+        
+    print read_ms_schema(attr_file, classes_file)
+
+        
diff --git a/source4/scripting/python/samba/provision.py b/source4/scripting/python/samba/provision.py
index 896f237bd7..189c93a1fc 100644
--- a/source4/scripting/python/samba/provision.py
+++ b/source4/scripting/python/samba/provision.py
@@ -44,6 +44,7 @@ from samba.dcerpc import security
 import urllib
 from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE, LdbError, \
         timestring, CHANGETYPE_MODIFY, CHANGETYPE_NONE
+from ms_schema import read_ms_schema
 
 __docformat__ = "restructuredText"
 
@@ -783,10 +784,8 @@ def setup_samdb(path, setup_path, session_info, credentials, lp,
     if serverrole == "domain controller":
         samdb.set_invocation_id(invocationid)
 
-    load_schema(setup_path, samdb, names.schemadn, names.netbiosname, 
-                names.configdn, names.sitename, names.serverdn,
-                names.hostname)
-
+    schema_data = load_schema(setup_path, samdb, names.schemadn, names.netbiosname, 
+                              names.configdn, names.sitename, names.serverdn)
     samdb.transaction_start()
         
     try:
@@ -851,12 +850,8 @@ def setup_samdb(path, setup_path, session_info, credentials, lp,
             "PREFIXMAP_B64": b64encode(prefixmap)
             })
 
-        message("Setting up sam.ldb Samba4 schema")
-        setup_add_ldif(samdb, setup_path("schema_samba4.ldif"), 
-                       {"SCHEMADN": names.schemadn })
-        message("Setting up sam.ldb AD schema")
-        setup_add_ldif(samdb, setup_path("schema.ldif"), 
-                       {"SCHEMADN": names.schemadn})
+        message("Setting up sam.ldb schema")
+        samdb.add_ldif(schema_data)
         setup_add_ldif(samdb, setup_path("aggregate_schema.ldif"), 
                        {"SCHEMADN": names.schemadn})
 
@@ -1249,28 +1244,33 @@ def provision_backend(setup_dir=None, message=None,
     except OSError:
         pass
 
-    schemadb = Ldb(schemadb_path, lp=lp)
+    schemadb = SamDB(schemadb_path, lp=lp)
+    schemadb.transaction_start()
+    try:
  
-    prefixmap = open(setup_path("prefixMap.txt"), 'r').read()
+        prefixmap = open(setup_path("prefixMap.txt"), 'r').read()
 
-    setup_add_ldif(schemadb, setup_path("provision_schema_basedn.ldif"), 
-                   {"SCHEMADN": names.schemadn,
-                    "ACI": "#",
-                    })
-    setup_modify_ldif(schemadb, 
-                      setup_path("provision_schema_basedn_modify.ldif"), \
-                          {"SCHEMADN": names.schemadn,
-                           "NETBIOSNAME": names.netbiosname,
-                           "DEFAULTSITE": DEFAULTSITE,
-                           "CONFIGDN": names.configdn,
-                           "SERVERDN": names.serverdn,
-                           "PREFIXMAP_B64": b64encode(prefixmap)
-                           })
-    
-    setup_add_ldif(schemadb, setup_path("schema_samba4.ldif"), 
-                   {"SCHEMADN": names.schemadn })
-    setup_add_ldif(schemadb, setup_path("schema.ldif"), 
-                   {"SCHEMADN": names.schemadn})
+        setup_add_ldif(schemadb, setup_path("provision_schema_basedn.ldif"), 
+                       {"SCHEMADN": names.schemadn,
+                        "ACI": "#",
+                        })
+        setup_modify_ldif(schemadb, 
+                          setup_path("provision_schema_basedn_modify.ldif"), \
+                              {"SCHEMADN": names.schemadn,
+                               "NETBIOSNAME": names.netbiosname,
+                               "DEFAULTSITE": DEFAULTSITE,
+                               "CONFIGDN": names.configdn,
+                               "SERVERDN": names.serverdn,
+                               "PREFIXMAP_B64": b64encode(prefixmap)
+                               })
+        
+        data = load_schema(setup_path, schemadb, names.schemadn, names.netbiosname, 
+                           names.configdn, DEFAULTSITE, names.serverdn)
+        schemadb.add_ldif(data)
+    except:
+        schemadb.transaction_cancel()
+        raise
+    schemadb.transaction_commit()
 
     if ldap_backend_type == "fedora-ds":
         if ldap_backend_port is not None:
@@ -1483,10 +1483,10 @@ def provision_backend(setup_dir=None, message=None,
 
         ldapuser = "--username=samba-admin"
 
-            
-    schema_command = "bin/ad2oLschema --option=convert:target=" + ldap_backend_type + " -I " + setup_path(mapping) + " -H tdb://" + schemadb_path + " -O " + os.path.join(paths.ldapdir, backend_schema)
-            
-    os.system(schema_command)
+
+    backend_schema_data = schemadb.convert_schema_to_openldap(ldap_backend_type, open(setup_path(mapping), 'r').read())
+    assert backend_schema_data is not None
+    open(os.path.join(paths.ldapdir, backend_schema), 'w').write(backend_schema_data)
 
     message("Your %s Backend for Samba4 is now configured, and is ready to be started" % ldap_backend_type)
     message("Server Role:         %s" % serverrole)
@@ -1649,7 +1649,7 @@ def create_krb5_conf(path, setup_path, dnsdomain, hostname, realm):
 
 
 def load_schema(setup_path, samdb, schemadn, netbiosname, configdn, sitename,
-                serverdn, servername):
+                serverdn):
     """Load schema for the SamDB.
     
     :param samdb: Load a schema into a SamDB.
@@ -1658,9 +1658,10 @@ def load_schema(setup_path, samdb, schemadn, netbiosname, configdn, sitename,
     :param netbiosname: NetBIOS name of the host.
     :param configdn: DN of the configuration
     :param serverdn: DN of the server
-    :param servername: Host name of the server
+
+    Returns the schema data loaded, to avoid double-parsing when then needing to add it to the db
     """
-    schema_data = open(setup_path("schema.ldif"), 'r').read()
+    schema_data = get_schema_data(setup_path, {"SCHEMADN": schemadn})
     schema_data += open(setup_path("schema_samba4.ldif"), 'r').read()
     schema_data = substitute_var(schema_data, {"SCHEMADN": schemadn})
     check_all_substituted(schema_data)
@@ -1675,8 +1676,26 @@ def load_schema(setup_path, samdb, schemadn, netbiosname, configdn, sitename,
                     "DEFAULTSITE": sitename,
                     "PREFIXMAP_B64": prefixmap,
                     "SERVERDN": serverdn,
-                    "SERVERNAME": servername,
     })
     check_all_substituted(head_data)
     samdb.attach_schema_from_ldif(head_data, schema_data)
+    return schema_data;
+
+def get_schema_data(setup_path, subst_vars = None):
+    """Get schema data from the AD schema files instead of schema.ldif.
+
+    :param setup_path: Setup path function.
+    :param subst_vars: Optional variables to substitute in the file.
+
+    Returns the schema data after substitution
+    """ 
 
+    # this data used to be read from schema.ldif
+    
+    data = read_ms_schema(setup_path('ad-schema/MS-AD_Schema_2K8_Attributes.txt'),
+                          setup_path('ad-schema/MS-AD_Schema_2K8_Classes.txt'))
+
+    if subst_vars is not None:
+        data = substitute_var(data, subst_vars)
+    check_all_substituted(data)
+    return data    
diff --git a/source4/scripting/python/samba/samdb.py b/source4/scripting/python/samba/samdb.py
index b92a91e2ef..17b5450a3e 100644
--- a/source4/scripting/python/samba/samdb.py
+++ b/source4/scripting/python/samba/samdb.py
@@ -199,7 +199,10 @@ userPassword:: %s
         glue.samdb_set_domain_sid(self, sid)
 
     def attach_schema_from_ldif(self, pf, df):
-        glue.dsdb_attach_schema_from_ldif_file(self, pf, df)
+        glue.dsdb_attach_schema_from_ldif(self, pf, df)
+
+    def convert_schema_to_openldap(self, target, mapping):
+        return glue.dsdb_convert_schema_to_openldap(self, target, mapping)
 
     def set_invocation_id(self, invocation_id):
         """Set the invocation id for this SamDB handle.