Added mmr and olc to the OpenLDAP backend provisioning-scripts

These extensions add mmr (multi-master-replication) and olc
(openldap-online-configuration) capabilities to the
provisioning-scripts (provision-backend and provision.py), for use
with the openldap-backend (only versions >=2.4.15!).

Changes / additions made to the provision-backend  -script:
added new command-line-options:
--ol-mmr-urls=<list of whitespace separated ldap-urls> for use with mmr
(can be combined with --ol-olc=yes),
--ol-olc=[yes/no] (activate automatic conversion from static slapd.conf
to olc),
--ol-slaptest=<path to slaptest binary> (needed in conjunction with
--ol-olc=yes)

Changes / additions made to the provision.py -script: added
extensions, that will automatically generate the chosen mmr and/or olc
setup for the openldap backend, according to the to chosen parameters
set in the provision-backend script

Signed-off-by: Andrew Bartlett <abartlet@samba.org>

13 files changed, 224 insertions, 63 deletions

diff --git a/source4/scripting/python/samba/provision.py b/source4/scripting/python/samba/provision.py
index a5b3e8322f..8442c24d71 100644
--- a/source4/scripting/python/samba/provision.py
+++ b/source4/scripting/python/samba/provision.py
@@ -26,6 +26,7 @@
 
 from base64 import b64encode
 import os
+import sys
 import pwd
 import grp
 import time
@@ -93,9 +94,14 @@ class ProvisionPaths(object):
         self.memberofconf = None
         self.fedoradsinf = None
         self.fedoradspartitions = None
-	self.olmmron = None
-	self.olmmrserveridsconf = None
-	self.olmmrsyncreplconf = None
+        self.olmmron = None
+        self.olmmrserveridsconf = None
+        self.olmmrsyncreplconf = None
+        self.olcdir = None
+        self.olslaptest = None
+        self.olcseedldif = None
+        self.olcsyncprovdir = None
+        self.olcsyncprovfile = None
 
 
 class ProvisionNames(object):
@@ -268,6 +274,14 @@ def provision_paths_from_lp(lp, dnsdomain):
                                             "mmr_serverids.conf")
     paths.olmmrsyncreplconf = os.path.join(paths.ldapdir, 
                                            "mmr_syncrepl.conf")
+    paths.olcdir = os.path.join(paths.ldapdir, 
+                                 "slapd.d")
+    paths.olcseedldif = os.path.join(paths.ldapdir, 
+                                 "olc_seed.ldif")
+    paths.olcsyncprovdir = os.path.join(paths.olcdir, 
+                                 "cn=config/olcDatabase={0}config")
+    paths.olcsyncprovfile = os.path.join(paths.olcsyncprovdir, 
+                                 "olcOverlay={0}syncprov.ldif")
     paths.hklm = "hklm.ldb"
     paths.hkcr = "hkcr.ldb"
     paths.hkcu = "hkcu.ldb"
@@ -1178,7 +1192,7 @@ def provision_backend(setup_dir=None, message=None,
                       rootdn=None, domaindn=None, schemadn=None, configdn=None,
                       domain=None, hostname=None, adminpass=None, root=None, serverrole=None, 
                       ldap_backend_type=None, ldap_backend_port=None,
-		      ol_mmr_urls=None):
+                      ol_mmr_urls=None,ol_olc=None,ol_slaptest=None):
 
     def setup_path(file):
         return os.path.join(setup_dir, file)
@@ -1205,6 +1219,19 @@ def provision_backend(setup_dir=None, message=None,
         make_smbconf(smbconf, setup_path, hostname, domain, realm, serverrole, 
                      targetdir)
 
+    # openldap-online-configuration: validation of olc and slaptest
+    if ol_olc == "yes" and ol_slaptest is None: 
+        sys.exit("Warning: OpenLDAP-Online-Configuration cant be setup without path to slaptest-Binary!")
+
+    if ol_olc == "yes" and ol_slaptest is not None:
+        ol_slaptest = ol_slaptest + "/slaptest"
+        if not os.path.exists(ol_slaptest):
+            message (ol_slaptest)
+            sys.exit("Warning: Given Path to slaptest-Binary does not exist!")
+    ###
+
+
+
     lp = param.LoadParm()
     lp.load(smbconf)
 
@@ -1300,52 +1327,95 @@ def provision_backend(setup_dir=None, message=None,
                                             { "LINK_ATTRS" : refint_attributes})
 
 # generate serverids, ldap-urls and syncrepl-blocks for mmr hosts
-	mmr_on_config = ""
-	mmr_replicator_acl = ""
-	mmr_serverids_config = ""
+        mmr_on_config = ""
+        mmr_replicator_acl = ""
+        mmr_serverids_config = ""
         mmr_syncrepl_schema_config = "" 
-	mmr_syncrepl_config_config = "" 
-	mmr_syncrepl_user_config = "" 
-	
-	if ol_mmr_urls is not None:
+        mmr_syncrepl_config_config = "" 
+        mmr_syncrepl_user_config = "" 
+       
+ 
+        if ol_mmr_urls is not None:
                 # For now, make these equal
                 mmr_pass = adminpass
 
- 		url_list=filter(None,ol_mmr_urls.split(' ')) 
+                url_list=filter(None,ol_mmr_urls.split(' ')) 
                 if (len(url_list) == 1):
                     url_list=filter(None,ol_mmr_urls.split(',')) 
                      
 
-		mmr_on_config = "MirrorMode On"
-		mmr_replicator_acl = "  by dn=cn=replicator,cn=samba read"
- 		serverid=0
-		for url in url_list:
-			serverid=serverid+1
-			mmr_serverids_config += read_and_sub_file(setup_path("mmr_serverids.conf"),
-								     { "SERVERID" : str(serverid),
-        		                                               "LDAPSERVER" : url })
+                mmr_on_config = "MirrorMode On"
+                mmr_replicator_acl = "  by dn=cn=replicator,cn=samba read"
+                serverid=0
+                for url in url_list:
+                        serverid=serverid+1
+                        mmr_serverids_config += read_and_sub_file(setup_path("mmr_serverids.conf"),
+                                                                     { "SERVERID" : str(serverid),
+                                                                       "LDAPSERVER" : url })
                         rid=serverid*10
-			rid=rid+1
-			mmr_syncrepl_schema_config += read_and_sub_file(setup_path("mmr_syncrepl.conf"),
-								     { 	"RID" : str(rid),
-                    							"MMRDN": names.schemadn,
-        		                                               	"LDAPSERVER" : url,
+                        rid=rid+1
+                        mmr_syncrepl_schema_config += read_and_sub_file(setup_path("mmr_syncrepl.conf"),
+                                                                     {  "RID" : str(rid),
+                                                                        "MMRDN": names.schemadn,
+                                                                        "LDAPSERVER" : url,
                                                                         "MMR_PASSWORD": mmr_pass})
 
-			rid=rid+1
-			mmr_syncrepl_config_config += read_and_sub_file(setup_path("mmr_syncrepl.conf"),
-								     { 	"RID" : str(rid),
-                    							"MMRDN": names.configdn,
-        		                                               	"LDAPSERVER" : url,
+                        rid=rid+1
+                        mmr_syncrepl_config_config += read_and_sub_file(setup_path("mmr_syncrepl.conf"),
+                                                                     {  "RID" : str(rid),
+                                                                        "MMRDN": names.configdn,
+                                                                        "LDAPSERVER" : url,
                                                                         "MMR_PASSWORD": mmr_pass})
 
-			rid=rid+1
-			mmr_syncrepl_user_config += read_and_sub_file(setup_path("mmr_syncrepl.conf"),
-								     { 	"RID" : str(rid),
-                    							"MMRDN": names.domaindn,
-        		                                               	"LDAPSERVER" : url,
+                        rid=rid+1
+                        mmr_syncrepl_user_config += read_and_sub_file(setup_path("mmr_syncrepl.conf"),
+                                                                     {  "RID" : str(rid),
+                                                                        "MMRDN": names.domaindn,
+                                                                        "LDAPSERVER" : url,
                                                                         "MMR_PASSWORD": mmr_pass })
+	# olc = yes?
+        olc_config_pass = ""
+        olc_config_acl = ""
+        olc_syncrepl_config = ""
+        olc_mmr_config = "" 
+        if ol_olc == "yes":
+                olc_config_pass += read_and_sub_file(setup_path("olc_pass.conf"),
+                                                                { "OLC_PW": adminpass })
+                olc_config_acl += read_and_sub_file(setup_path("olc_acl.conf"),{})
+                
+            # if olc = yes + mmr = yes, generate cn=config-replication directives
+            # and  olc_seed.lif for the other mmr-servers
+                if ol_olc == "yes" and ol_mmr_urls is not None:
+                        serverid=0
+                        olc_serverids_config = ""
+                        olc_syncrepl_config = ""
+                        olc_syncrepl_seed_config = ""
+                        olc_mmr_config = "" 
+                        olc_mmr_config += read_and_sub_file(setup_path("olc_mmr.conf"),{})
+                        rid=1000
+                        for url in url_list:
+                                serverid=serverid+1
+                                olc_serverids_config += read_and_sub_file(setup_path("olc_serverid.conf"),
+                                                                     { "SERVERID" : str(serverid),
+                                                                       "LDAPSERVER" : url })
+                        
+                                rid=rid+1
+                                olc_syncrepl_config += read_and_sub_file(setup_path("olc_syncrepl.conf"),
+                                                                     {  "RID" : str(rid),
+                                                                        "LDAPSERVER" : url,
+                                                                        "MMR_PASSWORD": adminpass})
+
+                                olc_syncrepl_seed_config += read_and_sub_file(setup_path("olc_syncrepl_seed.conf"),
+                                                                     {  "RID" : str(rid),
+                                                                        "LDAPSERVER" : url})
+
+                                setup_file(setup_path("olc_seed.ldif"), paths.olcseedldif,
+                                                                     {"OLC_SERVER_ID_CONF": olc_serverids_config,
+                                                                      "OLC_PW": adminpass,
+                                                                      "OLC_SYNCREPL_CONF": olc_syncrepl_seed_config})
+        
 
+                # end olc
 
         setup_file(setup_path("slapd.conf"), paths.slapdconf,
                    {"DNSDOMAIN": names.dnsdomain,
@@ -1360,8 +1430,12 @@ def provision_backend(setup_dir=None, message=None,
                     "MMR_SYNCREPL_SCHEMA_CONFIG": mmr_syncrepl_schema_config,
                     "MMR_SYNCREPL_CONFIG_CONFIG": mmr_syncrepl_config_config,
                     "MMR_SYNCREPL_USER_CONFIG": mmr_syncrepl_user_config,
+                    "OLC_CONFIG_PASS": olc_config_pass,
+                    "OLC_SYNCREPL_CONFIG": olc_syncrepl_config,
+                    "OLC_CONFIG_ACL": olc_config_acl,
+                    "OLC_MMR_CONFIG": olc_mmr_config,
                     "REFINT_CONFIG": refint_config})
-	setup_file(setup_path("modules.conf"), paths.modulesconf,
+        setup_file(setup_path("modules.conf"), paths.modulesconf,
                    {"REALM": names.realm})
         
         setup_db_config(setup_path, os.path.join(paths.ldapdir, "db", "user"))
@@ -1380,16 +1454,15 @@ def provision_backend(setup_dir=None, message=None,
                               {"LDAPADMINPASS_B64": b64encode(adminpass),
                                "UUID": str(uuid.uuid4()), 
                                "LDAPTIME": timestring(int(time.time()))} )
-	
-	if ol_mmr_urls is not None:
- 	   setup_file(setup_path("cn=replicator.ldif"),
+        
+        if ol_mmr_urls is not None:
+           setup_file(setup_path("cn=replicator.ldif"),
                               os.path.join(paths.ldapdir, "db", "samba",  "cn=samba", "cn=replicator.ldif"),
                               {"MMR_PASSWORD_B64": b64encode(mmr_pass),
                                "UUID": str(uuid.uuid4()),
                                "LDAPTIME": timestring(int(time.time()))} )
 
 
-
         mapping = "schema-map-openldap-2.3"
         backend_schema = "backend-schema.schema"
 
@@ -1399,7 +1472,18 @@ def provision_backend(setup_dir=None, message=None,
         else:
             server_port_string = ""
 
-        slapdcommand="Start slapd with:    slapd -f " + paths.ldapdir + "/slapd.conf -h " + ldapi_uri + server_port_string
+        if ol_olc != "yes" and ol_mmr_urls is None:
+          slapdcommand="Start slapd with:    slapd -f " + paths.ldapdir + "/slapd.conf -h " + ldapi_uri + server_port_string
+
+        if ol_olc == "yes" and ol_mmr_urls is None:
+          slapdcommand="Start slapd with:    slapd -F " + paths.olcdir + " -h \"" + ldapi_uri + " ldap://<FQHN>:<PORT>\"" 
+
+        if ol_olc != "yes" and ol_mmr_urls is not None:
+          slapdcommand="Start slapd with:    slapd -F " + paths.ldapdir + "/slapd.conf -h \"" + ldapi_uri + " ldap://<FQHN>:<PORT>\""
+
+        if ol_olc == "yes" and ol_mmr_urls is not None:
+          slapdcommand="Start slapd with:    slapd -F " + paths.olcdir + " -h \"" + ldapi_uri + " ldap://<FQHN>:<PORT>\""
+
 
         ldapuser = "--username=samba-admin"
 
@@ -1437,6 +1521,27 @@ def provision_backend(setup_dir=None, message=None,
     message("Run provision with: " + " ".join(args))
 
 
+    # if --ol-olc=yes, generate online-configuration in ../private/ldap/slapd.d 
+    if ol_olc == "yes":
+          if not os.path.isdir(paths.olcdir):
+             os.makedirs(paths.olcdir, 0770)
+          paths.olslaptest = str(ol_slaptest)
+          olc_command = paths.olslaptest + " -f" + paths.slapdconf + " -F" +  paths.olcdir + " >/dev/null 2>&1"
+          os.system(olc_command)
+          #os.remove(paths.slapdconf)        
+          # use line below for debugging during olc-conversion with slaptest 
+          #olc_command = paths.olslaptest + " -f" + paths.slapdconf + " -F" +  paths.olcdir"
+
+    # workaround, if overlay syncprov is was not created properly during conversion to cn=config.
+    # otherwise, cn=config won't be replicated
+    if ol_olc == "yes" and ol_mmr_urls is not None:
+        if not os.path.exists(paths.olcsyncprovdir):
+            os.makedirs(paths.olcsyncprovdir, 0770)
+            setup_file(setup_path("olcOverlay={0}syncprov.ldif"),
+                   os.path.join(paths.olcsyncprovdir, "olcOverlay={0}syncprov.ldif"), {})
+
+
+
 def create_phpldapadmin_config(path, setup_path, ldapi_uri):
     """Create a PHP LDAP admin configuration file.
 
diff --git a/source4/setup/DB_CONFIG b/source4/setup/DB_CONFIG
index b4d2bfa868..74bb09d800 100644
--- a/source4/setup/DB_CONFIG
+++ b/source4/setup/DB_CONFIG
@@ -1,17 +1,6 @@
-#
-# Set the database in memory cache size.
-#
 set_cachesize   0       524288        0
-
-#
-# Set log values.
-#
 set_lg_regionmax        104857
 set_lg_max              1048576
 set_lg_bsize            209715
 set_lg_dir              ${LDAPDBDIR}/bdb-logs
-
-#
-# Set temporary file creation directory.
-#			
 set_tmp_dir             ${LDAPDBDIR}/tmp
diff --git a/source4/setup/mmr_serverids.conf b/source4/setup/mmr_serverids.conf
index c6d14010b4..e4daf2028a 100644
--- a/source4/setup/mmr_serverids.conf
+++ b/source4/setup/mmr_serverids.conf
@@ -1,2 +1 @@
-# Generated from template mmr_serverids.conf
 ServerID ${SERVERID} "${LDAPSERVER}"
diff --git a/source4/setup/olcOverlay={0}syncprov.ldif b/source4/setup/olcOverlay={0}syncprov.ldif
new file mode 100644
index 0000000000..4f5b513c67
--- /dev/null
+++ b/source4/setup/olcOverlay={0}syncprov.ldif
@@ -0,0 +1,11 @@
+dn: olcOverlay={0}syncprov
+objectClass: olcOverlayConfig
+objectClass: olcSyncProvConfig
+olcOverlay: {0}syncprov
+structuralObjectClass: olcSyncProvConfig
+entryUUID: 41df5aca-785a-102d-9077-999999999999
+creatorsName: cn=config
+createTimestamp: 20090116201111Z
+entryCSN: 20090116201111.111111Z#000000#000#000000
+modifiersName: cn=config
+modifyTimestamp: 20090116201111Z
diff --git a/source4/setup/olc_acl.conf b/source4/setup/olc_acl.conf
new file mode 100644
index 0000000000..c248b30fb5
--- /dev/null
+++ b/source4/setup/olc_acl.conf
@@ -0,0 +1,4 @@
+access to dn.sub="cn=config"
+	by dn="cn=samba-admin,cn=samba" write
+	by dn="cn=replicator,cn=samba" read
+
diff --git a/source4/setup/olc_mmr.conf b/source4/setup/olc_mmr.conf
new file mode 100644
index 0000000000..2f60df1421
--- /dev/null
+++ b/source4/setup/olc_mmr.conf
@@ -0,0 +1,3 @@
+overlay syncprov
+MirrorMode on
+ 
diff --git a/source4/setup/olc_pass.conf b/source4/setup/olc_pass.conf
new file mode 100644
index 0000000000..4c66c1c43f
--- /dev/null
+++ b/source4/setup/olc_pass.conf
@@ -0,0 +1,3 @@
+database	config
+rootdn		cn=config
+
diff --git a/source4/setup/olc_seed.ldif b/source4/setup/olc_seed.ldif
new file mode 100644
index 0000000000..afc3abe5a0
--- /dev/null
+++ b/source4/setup/olc_seed.ldif
@@ -0,0 +1,16 @@
+dn: cn=config
+objectClass: olcGlobal
+cn: config
+${OLC_SERVER_ID_CONF}
+
+dn: olcDatabase={0}config,cn=config
+objectClass: olcDatabaseConfig
+olcDatabase: {0}config
+olcRootDN: cn=config
+olcRootPW: ${OLC_PW}
+${OLC_SYNCREPL_CONF}olcMirrorMode: TRUE
+
+dn: olcOverlay=syncprov,olcDatabase={0}config,cn=config
+objectClass: olcSyncProvConfig
+olcOverlay: syncprov
+ 
diff --git a/source4/setup/olc_serverid.conf b/source4/setup/olc_serverid.conf
new file mode 100644
index 0000000000..3d28acbfb4
--- /dev/null
+++ b/source4/setup/olc_serverid.conf
@@ -0,0 +1 @@
+olcServerID: ${SERVERID} "${LDAPSERVER}"
diff --git a/source4/setup/olc_syncrepl.conf b/source4/setup/olc_syncrepl.conf
new file mode 100644
index 0000000000..fd7a58d03b
--- /dev/null
+++ b/source4/setup/olc_syncrepl.conf
@@ -0,0 +1,13 @@
+# Generated from template olc_syncrepl.conf 
+
+syncrepl rid=${RID} 
+	provider="${LDAPSERVER}"
+	searchbase="cn=config"
+	filter="(!(olcDatabase={0}config))"
+	type=refreshAndPersist
+	retry="10 +"
+	bindmethod=sasl
+	saslmech=DIGEST-MD5
+	authcid="replicator"
+	credentials="${MMR_PASSWORD}"
+
diff --git a/source4/setup/olc_syncrepl_seed.conf b/source4/setup/olc_syncrepl_seed.conf
new file mode 100644
index 0000000000..1833fb9228
--- /dev/null
+++ b/source4/setup/olc_syncrepl_seed.conf
@@ -0,0 +1,5 @@
+olcSyncRepl: rid=${RID} provider="${LDAPSERVER}"
+  binddn="cn=config" bindmethod=sasl saslmech=DIGEST-MD5
+  authcid="replicator" credentials="linux"
+  searchbase="cn=config" filter="(!(olcDatabase={0}config))"
+  type=refreshAndPersist retry="10 +"
diff --git a/source4/setup/provision-backend b/source4/setup/provision-backend
index eca209cb18..20e4420414 100755
--- a/source4/setup/provision-backend
+++ b/source4/setup/provision-backend
@@ -65,8 +65,12 @@ parser.add_option("--server-role", type="choice", metavar="ROLE",
 parser.add_option("--targetdir", type="string", metavar="DIR", 
 		          help="Set target directory")
 parser.add_option("--ol-mmr-urls", type="string", metavar="LDAPSERVER",
-                help="List of LDAP-URLS [ ldap://<FQDN>:port/  (where port != 389) ] separated with whitespaces for use with OpenLDAP-MMR")
-
+                help="List of LDAP-URLS [ ldap://<FQDN>:port/  (where port != 389) ] separated with whitespaces for use with OpenLDAP-MMR (Multi-Master-Replication)")
+parser.add_option("--ol-olc", type="choice", metavar="OPENLDAP-OLC", 
+		help="To setup OpenLDAP-Backend with Online-Configuration [slapd.d] choose 'yes'",
+		choices=["yes", "no"])
+parser.add_option("--ol-slaptest", type="string", metavar="SLAPTEST-PATH", 
+		help="Path to slaptest-binary [e.g.:'/usr/local/sbin']. Only for use with --ol-olc='yes'")
 
 opts = parser.parse_args()[0]
 
@@ -103,5 +107,7 @@ provision_backend(setup_dir=setup_dir, message=message, smbconf=smbconf, targetd
 		  root=opts.root, serverrole=server_role, 
 		  ldap_backend_type=opts.ldap_backend_type,
 		  ldap_backend_port=opts.ldap_backend_port,
-		  ol_mmr_urls=opts.ol_mmr_urls)
+		  ol_mmr_urls=opts.ol_mmr_urls,
+		  ol_olc=opts.ol_olc,
+		  ol_slaptest=opts.ol_slaptest)
 
diff --git a/source4/setup/slapd.conf b/source4/setup/slapd.conf
index 506dc504b4..09dffbbfa3 100644
--- a/source4/setup/slapd.conf
+++ b/source4/setup/slapd.conf
@@ -7,7 +7,6 @@ sizelimit unlimited
 
 ${MMR_SERVERIDS_CONFIG}
 
-
 include ${LDAPDIR}/backend-schema.schema
 
 pidfile		${LDAPDIR}/slapd.pid
@@ -63,6 +62,13 @@ directory       ${LDAPDIR}/db/samba
 rootdn          cn=Manager,cn=Samba
 
 ########################################
+## olc - configuration ###
+${OLC_CONFIG_PASS}
+${OLC_SYNCREPL_CONFIG}
+${OLC_MMR_CONFIG}
+${OLC_CONFIG_ACL}
+
+########################################
 ### cn=schema ###
 database        hdb
 suffix		${SCHEMADN}
@@ -78,10 +84,10 @@ index cn eq
 index entryUUID,entryCSN eq
 
 #syncprov is stable in OpenLDAP 2.3, and available in 2.2.  
-#We only need this for the contextCSN attribute anyway....
+#We need this for the contextCSN attribute and mmr.
 overlay syncprov
 syncprov-sessionlog 100
-# syncprov-checkpoint 100 10
+syncprov-checkpoint 100 10
 
 
 ### Multimaster-Replication of cn=schema Subcontext ###
@@ -107,10 +113,10 @@ index cn eq
 index entryUUID,entryCSN eq
 
 #syncprov is stable in OpenLDAP 2.3, and available in 2.2.  
-#We only need this for the contextCSN attribute anyway....
+#We need this for the contextCSN attribute and mmr.
 overlay syncprov
 syncprov-sessionlog 100
-# syncprov-checkpoint 100 10
+syncprov-checkpoint 100 10
 
 ### Multimaster-Replication of cn=config Subcontext ###
 ${MMR_SYNCREPL_CONFIG_CONFIG}
@@ -139,10 +145,10 @@ index cn eq
 index entryUUID,entryCSN eq
 
 #syncprov is stable in OpenLDAP 2.3, and available in 2.2.  
-#We only need this for the contextCSN attribute anyway....
+#We need this for the contextCSN attribute and mmr.
 overlay syncprov
 syncprov-sessionlog 100
-# syncprov-checkpoint 100 10
+syncprov-checkpoint 100 10
 
 ### Multimaster-Replication of cn=user/base-dn context ###
 ${MMR_SYNCREPL_USER_CONFIG}