upgradeprovision: split the big script to put reusable functions appart

Signed-off-by: Jelmer Vernooij <jelmer@samba.org>

1 files changed, 48 insertions, 231 deletions

diff --git a/source4/scripting/bin/upgradeprovision b/source4/scripting/bin/upgradeprovision
index 1d58f5e2c9..ca5613fcc2 100755
--- a/source4/scripting/bin/upgradeprovision
+++ b/source4/scripting/bin/upgradeprovision
@@ -29,35 +29,33 @@ import sys
 import random
 import string
 import re
-import base64
 import tempfile
-# Find right directory when running from source tree
+# Allow to run from s4 source directory (without installing samba)
 sys.path.insert(0, "bin/python")
 
-from base64 import b64encode
 
 import samba
+import samba.getopt as options
 from samba.credentials import DONT_USE_KERBEROS
 from samba.auth import system_session, admin_session
-from samba import Ldb, DS_DOMAIN_FUNCTION_2000, DS_DOMAIN_FUNCTION_2003, DS_DOMAIN_FUNCTION_2008, DS_DC_FUNCTION_2008_R2
-from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE, LdbError
-import ldb
-import samba.getopt as options
+from samba import Ldb
+from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE, LdbError, FLAG_MOD_REPLACE, FLAG_MOD_ADD, FLAG_MOD_DELETE, MessageElement, Message, Dn
 from samba.samdb import SamDB
 from samba import param
 from samba import glue
 from samba.misc import messageEltFlagToString
-from samba.provision import  ProvisionNames,provision_paths_from_lp,find_setup_dir,FILL_FULL,provision, get_domain_descriptor, get_config_descriptor, secretsdb_self_join
+from samba.provision import  find_setup_dir, get_domain_descriptor, get_config_descriptor, secretsdb_self_join
 from samba.provisionexceptions import ProvisioningError
 from samba.schema import get_dnsyntax_attributes, get_linked_attributes, Schema, get_schema_descriptor
 from samba.dcerpc import misc, security
 from samba.ndr import ndr_pack, ndr_unpack
 from samba.dcerpc.misc import SEC_CHAN_BDC
+from samba.upgradehelpers import dn_sort, get_paths, newprovision, find_provision_key_parameters, rmall
 
 never=0
-replace=2^ldb.FLAG_MOD_REPLACE
-add=2^ldb.FLAG_MOD_ADD
-delete=2^ldb.FLAG_MOD_DELETE
+replace=2^FLAG_MOD_REPLACE
+add=2^FLAG_MOD_ADD
+delete=2^FLAG_MOD_DELETE
 
 #Errors are always logged
 ERROR = 	-1
@@ -152,42 +150,23 @@ session = system_session()
 # simple helper to allow back and forth rename
 def identic_rename(ldbobj,dn):
 	(before,sep,after)=str(dn).partition('=')
-	ldbobj.rename(dn,ldb.Dn(ldbobj,"%s=foo%s"%(before,after)))
-	ldbobj.rename(ldb.Dn(ldbobj,"%s=foo%s"%(before,after)),dn)
+	ldbobj.rename(dn,Dn(ldbobj,"%s=foo%s"%(before,after)))
+	ldbobj.rename(Dn(ldbobj,"%s=foo%s"%(before,after)),dn)
 
 # Create an array of backlinked attributes
 def populate_backlink(newpaths,creds,session,schemadn):
 	newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
-	linkedAttHash = get_linked_attributes(ldb.Dn(newsam_ldb,str(schemadn)),newsam_ldb)
+	linkedAttHash = get_linked_attributes(Dn(newsam_ldb,str(schemadn)),newsam_ldb)
 	backlinked.extend(linkedAttHash.values())
 
 # Create an array of  attributes with a dn synthax (2.5.5.1)
 def populate_dnsyntax(newpaths,creds,session,schemadn):
 	newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
-	res = newsam_ldb.search(expression="(attributeSyntax=2.5.5.1)",base=ldb.Dn(newsam_ldb,str(schemadn)), scope=SCOPE_SUBTREE, attrs=["lDAPDisplayName"])
+	res = newsam_ldb.search(expression="(attributeSyntax=2.5.5.1)",base=Dn(newsam_ldb,str(schemadn)),
+							scope=SCOPE_SUBTREE, attrs=["lDAPDisplayName"])
 	for elem in res:
 		dn_syntax_att.append(elem["lDAPDisplayName"])
 
-# Get Paths for important objects (ldb, keytabs ...)
-def get_paths(targetdir=None,smbconf=None):
-	if targetdir is not None:
-		if (not os.path.exists(os.path.join(targetdir, "etc"))):
-			os.makedirs(os.path.join(targetdir, "etc"))
-		smbconf = os.path.join(targetdir, "etc", "smb.conf")
-	if smbconf is None:
-			smbconf = param.default_path()
-
-	if not os.path.exists(smbconf):
-		message(ERROR,"Unable to find smb.conf ..")
-		parser.print_usage()
-		sys.exit(1)
-
-	lp = param.LoadParm()
-	lp.load(smbconf)
-# Normally we need the domain name for this function but for our needs it's
-# pointless
-	paths = provision_paths_from_lp(lp,"foo")
-	return paths
 
 
 def sanitychecks(credentials,session_info,names,paths):
@@ -206,93 +185,8 @@ domain with more than one DC, please demote the other DC before upgrading"%len(r
 		return 1
 
 
-# This function guesses (fetches) informations needed to make a fresh provision
-# from the current provision
-# It includes: realm, workgroup, partitions, netbiosname, domain guid, ...
-def guess_names_from_current_provision(credentials,session_info,paths):
-	lp = param.LoadParm()
-	lp.load(paths.smbconf)
-	names = ProvisionNames()
-	# NT domain, kerberos realm, root dn, domain dn, domain dns name
-	names.domain = string.upper(lp.get("workgroup"))
-	names.realm = lp.get("realm")
-	basedn = "DC=" + names.realm.replace(".",",DC=")
-	names.dnsdomain = names.realm
-	names.realm = string.upper(names.realm)
-	# netbiosname
-	secrets_ldb = Ldb(paths.secrets, session_info=session_info, credentials=credentials,lp=lp, options=["modules:samba_secrets"])
-	# Get the netbiosname first (could be obtained from smb.conf in theory)
-	attrs = ["sAMAccountName"]
-	res = secrets_ldb.search(expression="(flatname=%s)"%names.domain,base="CN=Primary Domains", scope=SCOPE_SUBTREE, attrs=attrs)
-	names.netbiosname = str(res[0]["sAMAccountName"]).replace("$","")
-
-	names.smbconf = smbconf
-	# It's important here to let ldb load with the old module or it's quite
-	# certain that the LDB won't load ...
-	samdb = Ldb(paths.samdb, session_info=session_info,
-		    credentials=credentials, lp=lp, options=["modules:samba_dsdb"])
-
-	# That's a bit simplistic but it's ok as long as we have only 3
-	# partitions
-	attrs2 = ["defaultNamingContext", "schemaNamingContext","configurationNamingContext","rootDomainNamingContext"]
-	current = samdb.search(expression="(objectClass=*)",base="", scope=SCOPE_BASE, attrs=attrs2)
-
-	names.configdn = current[0]["configurationNamingContext"]
-	configdn = str(names.configdn)
-	names.schemadn = current[0]["schemaNamingContext"]
-	if not (ldb.Dn(samdb, basedn) == (ldb.Dn(samdb, current[0]["defaultNamingContext"][0]))):
-		raise ProvisioningError(("basedn in %s (%s) and from %s (%s) is not the same ..." % (paths.samdb, str(current[0]["defaultNamingContext"][0]), paths.smbconf, basedn)))
-
-	names.domaindn=current[0]["defaultNamingContext"]
-	names.rootdn=current[0]["rootDomainNamingContext"]
-	# default site name
-	attrs3 = ["cn"]
-	res3= samdb.search(expression="(objectClass=*)",base="CN=Sites,"+configdn, scope=SCOPE_ONELEVEL, attrs=attrs3)
-	names.sitename = str(res3[0]["cn"])
-
-	# dns hostname and server dn
-	attrs4 = ["dNSHostName"]
-	res4= samdb.search(expression="(CN=%s)"%names.netbiosname,base="OU=Domain Controllers,"+basedn, \
-						scope=SCOPE_ONELEVEL, attrs=attrs4)
-	names.hostname = str(res4[0]["dNSHostName"]).replace("."+names.dnsdomain,"")
-
-	server_res = samdb.search(expression="serverReference=%s"%res4[0].dn, attrs=[], base=configdn)
-	names.serverdn = server_res[0].dn
-
-	# invocation id/objectguid
-	res5 = samdb.search(expression="(objectClass=*)",base="CN=NTDS Settings,%s" % str(names.serverdn), scope=SCOPE_BASE, attrs=["invocationID","objectGUID"])
-	names.invocation = str(ndr_unpack( misc.GUID,res5[0]["invocationId"][0]))
-	names.ntdsguid = str(ndr_unpack( misc.GUID,res5[0]["objectGUID"][0]))
-
-	# domain guid/sid
-	attrs6 = ["objectGUID", "objectSid","msDS-Behavior-Version" ]
-	res6 = samdb.search(expression="(objectClass=*)",base=basedn, scope=SCOPE_BASE, attrs=attrs6)
-	names.domainguid = str(ndr_unpack( misc.GUID,res6[0]["objectGUID"][0]))
-	names.domainsid = ndr_unpack( security.dom_sid,res6[0]["objectSid"][0])
-	if res6[0].get("msDS-Behavior-Version") == None or int(res6[0]["msDS-Behavior-Version"][0]) < DS_DOMAIN_FUNCTION_2000:
-		names.domainlevel = DS_DOMAIN_FUNCTION_2000
-	else:
-		names.domainlevel = int(res6[0]["msDS-Behavior-Version"][0])
-
-	# policy guid
-	attrs7 = ["cn","displayName"]
-	res7 = samdb.search(expression="(displayName=Default Domain Policy)",base="CN=Policies,CN=System,"+basedn, \
-							scope=SCOPE_ONELEVEL, attrs=attrs7)
-	names.policyid = str(res7[0]["cn"]).replace("{","").replace("}","")
-	# dc policy guid
-	attrs8 = ["cn","displayName"]
-	res8 = samdb.search(expression="(displayName=Default Domain Controllers Policy)",base="CN=Policies,CN=System,"+basedn, \
-							scope=SCOPE_ONELEVEL, attrs=attrs7)
-	if len(res8) == 1:
-		names.policyid_dc = str(res8[0]["cn"]).replace("{","").replace("}","")
-	else:
-		names.policyid_dc = None
-
-
-	return names
-
 # Debug a little bit
-def print_names(names):
+def print_provision_key_parameters(names):
 	message(GUESS, "rootdn      :"+str(names.rootdn))
 	message(GUESS, "configdn    :"+str(names.configdn))
 	message(GUESS, "schemadn    :"+str(names.schemadn))
@@ -311,77 +205,6 @@ def print_names(names):
 	message(GUESS, "ntdsguid    :"+names.ntdsguid)
 	message(GUESS, "domainlevel :"+str(names.domainlevel))
 
-# Create a fresh new reference provision
-# This provision will be the reference for knowing what has changed in the
-# since the latest upgrade in the current provision
-def newprovision(names,setup_dir,creds,session,smbconf):
-	message(SIMPLE, "Creating a reference provision")
-	provdir=tempfile.mkdtemp(dir=paths.private_dir, prefix="referenceprovision")
-	if os.path.isdir(provdir):
-		rmall(provdir)
-	logstd=os.path.join(provdir,"log.std")
-	os.chdir(os.path.join(setup_dir,".."))
-	os.mkdir(provdir)
-	os.close(2)
-	sys.stderr = open("%s/provision.log"%provdir, "w")
-	message(PROVISION, "Reference provision stored in %s"%provdir)
-	message(PROVISION, "STDERR message of provision will be logged in %s/provision.log"%provdir)
-	sys.stderr = open("/dev/stdout", "w")
-	provision(setup_dir, messageprovision,
-		session, creds, smbconf=smbconf, targetdir=provdir,
-		samdb_fill=FILL_FULL, realm=names.realm, domain=names.domain,
-		domainguid=names.domainguid, domainsid=str(names.domainsid),ntdsguid=names.ntdsguid,
-		policyguid=names.policyid,policyguid_dc=names.policyid_dc,hostname=names.netbiosname,
-		hostip=None, hostip6=None,
-		invocationid=names.invocation, adminpass=None,
-		krbtgtpass=None, machinepass=None,
-		dnspass=None, root=None, nobody=None,
-		wheel=None, users=None,
-		serverrole="domain controller",
-		ldap_backend_extra_port=None,
-		backend_type=None,
-		ldapadminpass=None,
-		ol_mmr_urls=None,
-		slapd_path=None,
-		setup_ds_path=None,
-		nosync=None,
-		dom_for_fun_level=names.domainlevel,
-		ldap_dryrun_mode=None,useeadb=True)
-	return provdir
-
-# This function sorts two DNs in the lexicographical order and put higher level
-# DN before.
-# So given the dns cn=bar,cn=foo and cn=foo the later will be return as smaller
-# (-1) as it has less level
-def dn_sort(x,y):
-	p = re.compile(r'(?<!\\),')
-	tab1 = p.split(str(x))
-	tab2 = p.split(str(y))
-	min = 0
-	if (len(tab1) > len(tab2)):
-		min = len(tab2)
-	elif (len(tab1) < len(tab2)):
-		min = len(tab1)
-	else:
-		min = len(tab1)
-	len1=len(tab1)-1
-	len2=len(tab2)-1
-	space = " "
-	# Note: python range go up to upper limit but do not include it
-	for i in range(0,min):
-		ret=cmp(tab1[len1-i],tab2[len2-i])
-		if(ret != 0):
-			return ret
-		else:
-			if(i==min-1):
-				if(len1==len2):
-					message(ERROR,"PB PB PB"+space.join(tab1)+" / "+space.join(tab2))
-				if(len1>len2):
-					return 1
-				else:
-					return -1
-	return ret
-
 # Check for security descriptors modifications return 1 if it is and 0 otherwise
 # it also populate hash structure for later use in the upgrade process
 def handle_security_desc(ischema,att,msgElt,hashallSD,old,new):
@@ -406,7 +229,7 @@ def handle_security_desc(ischema,att,msgElt,hashallSD,old,new):
 # It can be also if we want to do a merge of value instead of a simple replace
 def handle_special_case(att,delta,new,old,ischema):
 	flag = delta.get(att).flags()
-	if (att == "gPLink" or att == "gPCFileSysPath") and flag ==  ldb.FLAG_MOD_REPLACE and str(new[0].dn).lower() == str(old[0].dn).lower():
+	if (att == "gPLink" or att == "gPCFileSysPath") and flag ==  FLAG_MOD_REPLACE and str(new[0].dn).lower() == str(old[0].dn).lower():
 		delta.remove(att)
 		return 1
 	if att == "forceLogoff":
@@ -417,13 +240,12 @@ def handle_special_case(att,delta,new,old,ischema):
 		return 1
 	if (att == "adminDisplayName" or att == "adminDescription") and ischema:
 		return 1
-	if (str(old[0].dn) == "CN=Samba4-Local-Domain,%s"%(str(names.schemadn)) and att == "defaultObjectCategory" and flag  == ldb.FLAG_MOD_REPLACE):
+	if (str(old[0].dn) == "CN=Samba4-Local-Domain,%s"%(str(names.schemadn)) and att == "defaultObjectCategory" and flag  == FLAG_MOD_REPLACE):
 		return 1
-#	if (str(old[0].dn) == "CN=S-1-5-11,CN=ForeignSecurityPrincipals,%s"%(str(names.rootdn)) and att == "description" and flag  == ldb.FLAG_MOD_DELETE):
-#		return 1
-	if (str(old[0].dn) == "CN=Title,%s"%(str(names.schemadn)) and att == "rangeUpper" and flag  == ldb.FLAG_MOD_REPLACE):
+
+	if (str(old[0].dn) == "CN=Title,%s"%(str(names.schemadn)) and att == "rangeUpper" and flag  == FLAG_MOD_REPLACE):
 		return 1
-	if ( (att == "member" or att == "servicePrincipalName") and flag  == ldb.FLAG_MOD_REPLACE):
+	if ( (att == "member" or att == "servicePrincipalName") and flag  == FLAG_MOD_REPLACE):
 
 		hash = {}
 		newval = []
@@ -437,11 +259,11 @@ def handle_special_case(att,delta,new,old,ischema):
 				changeDelta=1
 				newval.append(str(elem))
 		if changeDelta == 1:
-			delta[att] = ldb.MessageElement(newval, ldb.FLAG_MOD_REPLACE, att)
+			delta[att] = MessageElement(newval, FLAG_MOD_REPLACE, att)
 		else:
 			delta.remove(att)
 		return 1
-	if (str(old[0].dn) == "%s"%(str(names.rootdn)) and att == "subRefs" and flag  == ldb.FLAG_MOD_REPLACE):
+	if (str(old[0].dn) == "%s"%(str(names.rootdn)) and att == "subRefs" and flag  == FLAG_MOD_REPLACE):
 		return 1
 	if str(delta.dn).endswith("CN=DisplaySpecifiers,%s"%names.configdn):
 		return 1
@@ -466,7 +288,7 @@ def update_secrets(newpaths,paths,creds,session):
 	listMissing = []
 	listPresent = []
 
-	empty = ldb.Message()
+	empty = Message()
 	for i in range(0,len(reference)):
 		hash_new[str(reference[i]["dn"]).lower()] = reference[i]["dn"]
 
@@ -581,7 +403,7 @@ def add_missing_object(newsam_ldb,sam_ldb,dn,names,basedn,hash,index):
 	handle_special_add(sam_ldb,dn,names)
 	reference = newsam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn,
 					scope=SCOPE_SUBTREE,controls=["search_options:1:2"])
-	empty = ldb.Message()
+	empty = Message()
 	delta = sam_ldb.msg_diff(empty,reference[0])
 	for att in hashAttrNotCopied.keys():
 		delta.remove(att)
@@ -726,7 +548,7 @@ def check_diff_name(newpaths,paths,creds,session,basedn,names,ischema):
 				if  hashOverwrittenAtt.has_key(att) and hashOverwrittenAtt.get(att)==never:
 					delta.remove(att)
 					continue
-				if  handle_special_case(att,delta,reference,current,ischema)==0 and msgElt.flags()!=ldb.FLAG_MOD_ADD:
+				if  handle_special_case(att,delta,reference,current,ischema)==0 and msgElt.flags()!=FLAG_MOD_ADD:
 					i = 0
 					if opts.debugchange or opts.debugall:
 						try:
@@ -772,24 +594,24 @@ def update_sd(paths,creds,session,names):
 	# First update the SD for the rootdn
 	sam_ldb.set_session_info(session)
 	res = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
-	delta = ldb.Message()
-	delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
+	delta = Message()
+	delta.dn = Dn(sam_ldb,str(res[0]["dn"]))
 	descr = get_domain_descriptor(names.domainsid)
-	delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
+	delta["nTSecurityDescriptor"] = MessageElement( descr,FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
 	sam_ldb.modify(delta,["recalculate_sd:0"])
 	# Then the config dn
 	res = sam_ldb.search(expression="objectClass=*",base=str(names.configdn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
-	delta = ldb.Message()
-	delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
+	delta = Message()
+	delta.dn = Dn(sam_ldb,str(res[0]["dn"]))
 	descr = get_config_descriptor(names.domainsid)
-	delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
+	delta["nTSecurityDescriptor"] = MessageElement( descr,FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
 	sam_ldb.modify(delta,["recalculate_sd:0"])
 	# Then the schema dn
 	res = sam_ldb.search(expression="objectClass=*",base=str(names.schemadn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
-	delta = ldb.Message()
-	delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
+	delta = Message()
+	delta.dn = Dn(sam_ldb,str(res[0]["dn"]))
 	descr = get_schema_descriptor(names.domainsid)
-	delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
+	delta["nTSecurityDescriptor"] = MessageElement( descr,FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
 	sam_ldb.modify(delta,["recalculate_sd:0"])
 
 	# Then the rest
@@ -806,9 +628,9 @@ def update_sd(paths,creds,session,names):
 
 	for key in listkeys:
 		try:
-			delta = ldb.Message()
-			delta.dn = ldb.Dn(sam_ldb,key)
-			delta["whenCreated"] = ldb.MessageElement( hash[key],ldb.FLAG_MOD_REPLACE,"whenCreated" )
+			delta = Message()
+			delta.dn = Dn(sam_ldb,key)
+			delta["whenCreated"] = MessageElement( hash[key],FLAG_MOD_REPLACE,"whenCreated" )
 			sam_ldb.modify(delta,["recalculate_sd:0"])
 		except:
 			sam_ldb.transaction_cancel()
@@ -817,14 +639,6 @@ def update_sd(paths,creds,session,names):
 			return
 	sam_ldb.transaction_commit()
 
-def rmall(topdir):
-	for root, dirs, files in os.walk(topdir, topdown=False):
-		for name in files:
-			os.remove(os.path.join(root, name))
-		for name in dirs:
-			os.rmdir(os.path.join(root, name))
-	os.rmdir(topdir)
-
 
 def update_basesamdb(newpaths,paths,names):
 	message(SIMPLE,"Copy samdb")
@@ -899,24 +713,27 @@ def update_machine_account_password(paths,creds,session,names):
 	else:
 		secrets_ldb.transaction_cancel()
 
+def setup_path(file):
+	return os.path.join(setup_dir, file)
 # From here start the big steps of the program
 # First get files paths
-paths=get_paths(smbconf=smbconf)
+paths=get_paths(param,smbconf=smbconf)
 paths.setup = setup_dir
-def setup_path(file):
-	return os.path.join(setup_dir, file)
 # Guess all the needed names (variables in fact) from the current
 # provision.
-names = guess_names_from_current_provision(creds,session,paths)
+
+names = find_provision_key_parameters(param,creds,session,paths,smbconf)
 if not sanitychecks(creds,session,names,paths):
-	print "Sanity checks for the upgrade fails, checks messages and correct it before rerunning upgradeprovision"
+	message(SIMPLE,"Sanity checks for the upgrade fails, checks messages and correct it before rerunning upgradeprovision")
 	sys.exit(1)
 # Let's see them
-print_names(names)
+print_provision_key_parameters(names)
 # With all this information let's create a fresh new provision used as reference
-provisiondir = newprovision(names,setup_dir,creds,session,smbconf)
+message(SIMPLE,"Creating a reference provision")
+provisiondir = tempfile.mkdtemp(dir=paths.private_dir, prefix="referenceprovision")
+newprovision(names,setup_dir,creds,session,smbconf,provisiondir,messageprovision)
 # Get file paths of this new provision
-newpaths = get_paths(targetdir=provisiondir)
+newpaths = get_paths(param,targetdir=provisiondir)
 populate_backlink(newpaths,creds,session,names.schemadn)
 populate_dnsyntax(newpaths,creds,session,names.schemadn)
 # Check the difference