path: root/source4/scripting/bin/upgradeprovision

#!/usr/bin/python
#
# Copyright (C) Matthieu Patou <mat@matws.net> 2009
#
# Based on provision a Samba4 server by
# Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2007-2008
# Copyright (C) Andrew Bartlett <abartlet@samba.org> 2008
#
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


import getopt
import shutil
import optparse
import os
import sys
import random
import string
import re
import base64
import tempfile
# Find right directory when running from source tree
sys.path.insert(0, "bin/python")

from base64 import b64encode

import samba
from samba.credentials import DONT_USE_KERBEROS
from samba.auth import system_session, admin_session
from samba import Ldb, DS_DOMAIN_FUNCTION_2000, DS_DOMAIN_FUNCTION_2003, DS_DOMAIN_FUNCTION_2008, DS_DC_FUNCTION_2008_R2
from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE, LdbError
import ldb
import samba.getopt as options
from samba.samdb import SamDB
from samba import param
from samba import glue
from samba.misc import messageEltFlagToString
from samba.provision import  ProvisionNames,provision_paths_from_lp,find_setup_dir,FILL_FULL,provision, get_domain_descriptor, get_config_descriptor, secretsdb_self_join
from samba.provisionexceptions import ProvisioningError
from samba.schema import get_dnsyntax_attributes, get_linked_attributes, Schema, get_schema_descriptor
from samba.dcerpc import misc, security
from samba.ndr import ndr_pack, ndr_unpack
from samba.dcerpc.misc import SEC_CHAN_BDC

never=0
replace=2^ldb.FLAG_MOD_REPLACE
add=2^ldb.FLAG_MOD_ADD
delete=2^ldb.FLAG_MOD_DELETE

#Errors are always logged
ERROR = 	-1
SIMPLE = 	0x00
CHANGE = 	0x01
CHANGESD = 	0x02
GUESS = 	0x04
PROVISION =	0x08
CHANGEALL =	0xff

# Attributes that are never copied from the reference provision (even if they
# do not exist in the destination object).
# This is most probably because they are populated automatcally when object is
# created
# This also apply to imported object from reference provision
hashAttrNotCopied = { 	"dn": 1,"whenCreated": 1,"whenChanged": 1,"objectGUID": 1,"replPropertyMetaData": 1,"uSNChanged": 1,
						"uSNCreated": 1,"parentGUID": 1,"objectCategory": 1,"distinguishedName": 1,
						"showInAdvancedViewOnly": 1,"instanceType": 1, "cn": 1, "msDS-Behavior-Version":1, "nextRid":1,
						"nTMixedDomain": 1,"versionNumber":1, "lmPwdHistory":1, "pwdLastSet": 1, "ntPwdHistory":1, "unicodePwd":1,
						"dBCSPwd":1,"supplementalCredentials":1,"gPCUserExtensionNames":1, "gPCMachineExtensionNames":1,
						"maxPwdAge":1, "mail":1, "secret":1,"possibleInferiors":1, "sAMAccountType":1}

# Usually for an object that already exists we do not overwrite attributes as
# they might have been changed for good reasons. Anyway for a few of them it's
# mandatory to replace them otherwise the provision will be broken somehow.
hashOverwrittenAtt = {	"prefixMap": replace, "systemMayContain": replace,"systemOnly":replace, "searchFlags":replace,
						"mayContain":replace,  "systemFlags":replace,"description":replace,
						"oEMInformation":replace, "operatingSystemVersion":replace, "adminPropertyPages":replace,
						"defaultSecurityDescriptor": replace,"wellKnownObjects":replace,"privilege":delete,"groupType":replace,
						"rIDAvailablePool": never}


backlinked = []
dn_syntax_att = []
def define_what_to_log(opts):
	what = 0
	if opts.debugchange:
		what = what | CHANGE
	if opts.debugchangesd:
		what = what | CHANGESD
	if opts.debugguess:
		what = what | GUESS
	if opts.debugprovision:
		what = what | PROVISION
	if opts.debugall:
		what = what | CHANGEALL
	return what


parser = optparse.OptionParser("provision [options]")
sambaopts = options.SambaOptions(parser)
parser.add_option_group(sambaopts)
parser.add_option_group(options.VersionOptions(parser))
credopts = options.CredentialsOptions(parser)
parser.add_option_group(credopts)
parser.add_option("--setupdir", type="string", metavar="DIR",
					help="directory with setup files")
parser.add_option("--debugprovision", help="Debug provision", action="store_true")
parser.add_option("--debugguess", help="Print information on what is different but won't be changed", action="store_true")
parser.add_option("--debugchange", help="Print information on what is different but won't be changed", action="store_true")
parser.add_option("--debugchangesd", help="Print information security descriptors differences", action="store_true")
parser.add_option("--debugall", help="Print all available information (very verbose)", action="store_true")
parser.add_option("--full", help="Perform full upgrade of the samdb (schema, configuration, new objects, ...", action="store_true")

opts = parser.parse_args()[0]

whatToLog = define_what_to_log(opts)

def messageprovision(text):
	"""print a message if quiet is not set."""
	if opts.debugprovision or opts.debugall:
		print text

def message(what,text):
	"""print a message if quiet is not set."""
	if (whatToLog & what) or (what <= 0 ):
		print text

if len(sys.argv) == 1:
	opts.interactive = True
lp = sambaopts.get_loadparm()
smbconf = lp.configfile

creds = credopts.get_credentials(lp)
creds.set_kerberos_state(DONT_USE_KERBEROS)
setup_dir = opts.setupdir
if setup_dir is None:
    setup_dir = find_setup_dir()

session = system_session()

# simple helper to allow back and forth rename
def identic_rename(ldbobj,dn):
	(before,sep,after)=str(dn).partition('=')
	ldbobj.rename(dn,ldb.Dn(ldbobj,"%s=foo%s"%(before,after)))
	ldbobj.rename(ldb.Dn(ldbobj,"%s=foo%s"%(before,after)),dn)

# Create an array of backlinked attributes
def populate_backlink(newpaths,creds,session,schemadn):
	newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
	linkedAttHash = get_linked_attributes(ldb.Dn(newsam_ldb,str(schemadn)),newsam_ldb)
	backlinked.extend(linkedAttHash.values())

# Create an array of  attributes with a dn synthax (2.5.5.1)
def populate_dnsyntax(newpaths,creds,session,schemadn):
	newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
	res = newsam_ldb.search(expression="(attributeSyntax=2.5.5.1)",base=ldb.Dn(newsam_ldb,str(schemadn)), scope=SCOPE_SUBTREE, attrs=["lDAPDisplayName"])
	for elem in res:
		dn_syntax_att.append(elem["lDAPDisplayName"])

# Get Paths for important objects (ldb, keytabs ...)
def get_paths(targetdir=None,smbconf=None):
	if targetdir is not None:
		if (not os.path.exists(os.path.join(targetdir, "etc"))):
			os.makedirs(os.path.join(targetdir, "etc"))
		smbconf = os.path.join(targetdir, "etc", "smb.conf")
	if smbconf is None:
			smbconf = param.default_path()

	if not os.path.exists(smbconf):
		message(ERROR,"Unable to find smb.conf ..")
		parser.print_usage()
		sys.exit(1)

	lp = param.LoadParm()
	lp.load(smbconf)
# Normally we need the domain name for this function but for our needs it's
# pointless
	paths = provision_paths_from_lp(lp,"foo")
	return paths


def sanitychecks(credentials,session_info,names,paths):
	sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp,options=["modules:samba_dsdb"])
	# First update the SD for the rootdn
	sam_ldb.set_session_info(session)
	res = sam_ldb.search(expression="objectClass=ntdsdsa",base=str(names.configdn), scope=SCOPE_SUBTREE,attrs=["dn"],controls=["search_options:1:2"])
	if len(res) == 0:
		print "No DC found, your provision is most probalby hardly broken !"
		return 0
	elif len(res) != 1:
		print "Found %d domain controllers, for the moment upgradeprovision is not able to handle upgrade on \
domain with more than one DC, please demote the other DC before upgrading"%len(res)
		return 0
	else:
		return 1


# This function guesses (fetches) informations needed to make a fresh provision
# from the current provision
# It includes: realm, workgroup, partitions, netbiosname, domain guid, ...
def guess_names_from_current_provision(credentials,session_info,paths):
	lp = param.LoadParm()
	lp.load(paths.smbconf)
	names = ProvisionNames()
	# NT domain, kerberos realm, root dn, domain dn, domain dns name
	names.domain = string.upper(lp.get("workgroup"))
	names.realm = lp.get("realm")
	basedn = "DC=" + names.realm.replace(".",",DC=")
	names.dnsdomain = names.realm
	names.realm = string.upper(names.realm)
	# netbiosname
	secrets_ldb = Ldb(paths.secrets, session_info=session_info, credentials=credentials,lp=lp, options=["modules:samba_secrets"])
	# Get the netbiosname first (could be obtained from smb.conf in theory)
	attrs = ["sAMAccountName"]
	res = secrets_ldb.search(expression="(flatname=%s)"%names.domain,base="CN=Primary Domains", scope=SCOPE_SUBTREE, attrs=attrs)
	names.netbiosname = str(res[0]["sAMAccountName"]).replace("$","")

	names.smbconf = smbconf
	# It's important here to let ldb load with the old module or it's quite
	# certain that the LDB won't load ...
	samdb = Ldb(paths.samdb, session_info=session_info,
		    credentials=credentials, lp=lp, options=["modules:samba_dsdb"])

	# That's a bit simplistic but it's ok as long as we have only 3
	# partitions
	attrs2 = ["defaultNamingContext", "schemaNamingContext","configurationNamingContext","rootDomainNamingContext"]
	current = samdb.search(expression="(objectClass=*)",base="", scope=SCOPE_BASE, attrs=attrs2)

	names.configdn = current[0]["configurationNamingContext"]
	configdn = str(names.configdn)
	names.schemadn = current[0]["schemaNamingContext"]
	if not (ldb.Dn(samdb, basedn) == (ldb.Dn(samdb, current[0]["defaultNamingContext"][0]))):
		raise ProvisioningError(("basedn in %s (%s) and from %s (%s) is not the same ..." % (paths.samdb, str(current[0]["defaultNamingContext"][0]), paths.smbconf, basedn)))

	names.domaindn=current[0]["defaultNamingContext"]
	names.rootdn=current[0]["rootDomainNamingContext"]
	# default site name
	attrs3 = ["cn"]
	res3= samdb.search(expression="(objectClass=*)",base="CN=Sites,"+configdn, scope=SCOPE_ONELEVEL, attrs=attrs3)
	names.sitename = str(res3[0]["cn"])

	# dns hostname and server dn
	attrs4 = ["dNSHostName"]
	res4= samdb.search(expression="(CN=%s)"%names.netbiosname,base="OU=Domain Controllers,"+basedn, \
						scope=SCOPE_ONELEVEL, attrs=attrs4)
	names.hostname = str(res4[0]["dNSHostName"]).replace("."+names.dnsdomain,"")

	server_res = samdb.search(expression="serverReference=%s"%res4[0].dn, attrs=[], base=configdn)
	names.serverdn = server_res[0].dn

	# invocation id/objectguid
	res5 = samdb.search(expression="(objectClass=*)",base="CN=NTDS Settings,%s" % str(names.serverdn), scope=SCOPE_BASE, attrs=["invocationID","objectGUID"])
	names.invocation = str(ndr_unpack( misc.GUID,res5[0]["invocationId"][0]))
	names.ntdsguid = str(ndr_unpack( misc.GUID,res5[0]["objectGUID"][0]))

	# domain guid/sid
	attrs6 = ["objectGUID", "objectSid","msDS-Behavior-Version" ]
	res6 = samdb.search(expression="(objectClass=*)",base=basedn, scope=SCOPE_BASE, attrs=attrs6)
	names.domainguid = str(ndr_unpack( misc.GUID,res6[0]["objectGUID"][0]))
	names.domainsid = ndr_unpack( security.dom_sid,res6[0]["objectSid"][0])
	if res6[0].get("msDS-Behavior-Version") == None or int(res6[0]["msDS-Behavior-Version"][0]) < DS_DOMAIN_FUNCTION_2000:
		names.domainlevel = DS_DOMAIN_FUNCTION_2000
	else:
		names.domainlevel = int(res6[0]["msDS-Behavior-Version"][0])

	# policy guid
	attrs7 = ["cn","displayName"]
	res7 = samdb.search(expression="(displayName=Default Domain Policy)",base="CN=Policies,CN=System,"+basedn, \
							scope=SCOPE_ONELEVEL, attrs=attrs7)
	names.policyid = str(res7[0]["cn"]).replace("{","").replace("}","")
	# dc policy guid
	attrs8 = ["cn","displayName"]
	res8 = samdb.search(expression="(displayName=Default Domain Controllers Policy)",base="CN=Policies,CN=System,"+basedn, \
							scope=SCOPE_ONELEVEL, attrs=attrs7)
	if len(res8) == 1:
		names.policyid_dc = str(res8[0]["cn"]).replace("{","").replace("}","")
	else:
		names.policyid_dc = None


	return names

# Debug a little bit
def print_names(names):
	message(GUESS, "rootdn      :"+str(names.rootdn))
	message(GUESS, "configdn    :"+str(names.configdn))
	message(GUESS, "schemadn    :"+str(names.schemadn))
	message(GUESS, "serverdn    :"+str(names.serverdn))
	message(GUESS, "netbiosname :"+names.netbiosname)
	message(GUESS, "defaultsite :"+names.sitename)
	message(GUESS, "dnsdomain   :"+names.dnsdomain)
	message(GUESS, "hostname    :"+names.hostname)
	message(GUESS, "domain      :"+names.domain)
	message(GUESS, "realm       :"+names.realm)
	message(GUESS, "invocationid:"+names.invocation)
	message(GUESS, "policyguid  :"+names.policyid)
	message(GUESS, "policyguiddc:"+str(names.policyid_dc))
	message(GUESS, "domainsid   :"+str(names.domainsid))
	message(GUESS, "domainguid  :"+names.domainguid)
	message(GUESS, "ntdsguid    :"+names.ntdsguid)
	message(GUESS, "domainlevel :"+str(names.domainlevel))

# Create a fresh new reference provision
# This provision will be the reference for knowing what has changed in the
# since the latest upgrade in the current provision
def newprovision(names,setup_dir,creds,session,smbconf):
	message(SIMPLE, "Creating a reference provision")
	provdir=tempfile.mkdtemp(dir=paths.private_dir, prefix="referenceprovision")
	if os.path.isdir(provdir):
		rmall(provdir)
	logstd=os.path.join(provdir,"log.std")
	os.chdir(os.path.join(setup_dir,".."))
	os.mkdir(provdir)
	os.close(2)
	sys.stderr = open("%s/provision.log"%provdir, "w")
	message(PROVISION, "Reference provision stored in %s"%provdir)
	message(PROVISION, "STDERR message of provision will be logged in %s/provision.log"%provdir)
	sys.stderr = open("/dev/stdout", "w")
	provision(setup_dir, messageprovision,
		session, creds, smbconf=smbconf, targetdir=provdir,
		samdb_fill=FILL_FULL, realm=names.realm, domain=names.domain,
		domainguid=names.domainguid, domainsid=str(names.domainsid),ntdsguid=names.ntdsguid,
		policyguid=names.policyid,policyguid_dc=names.policyid_dc,hostname=names.netbiosname,
		hostip=None, hostip6=None,
		invocationid=names.invocation, adminpass=None,
		krbtgtpass=None, machinepass=None,
		dnspass=None, root=None, nobody=None,
		wheel=None, users=None,
		serverrole="domain controller",
		ldap_backend_extra_port=None,
		backend_type=None,
		ldapadminpass=None,
		ol_mmr_urls=None,
		slapd_path=None,
		setup_ds_path=None,
		nosync=None,
		dom_for_fun_level=names.domainlevel,
		ldap_dryrun_mode=None,useeadb=True)
	return provdir

# This function sorts two DNs in the lexicographical order and put higher level
# DN before.
# So given the dns cn=bar,cn=foo and cn=foo the later will be return as smaller
# (-1) as it has less level
def dn_sort(x,y):
	p = re.compile(r'(?<!\\),')
	tab1 = p.split(str(x))
	tab2 = p.split(str(y))
	min = 0
	if (len(tab1) > len(tab2)):
		min = len(tab2)
	elif (len(tab1) < len(tab2)):
		min = len(tab1)
	else:
		min = len(tab1)
	len1=len(tab1)-1
	len2=len(tab2)-1
	space = " "
	# Note: python range go up to upper limit but do not include it
	for i in range(0,min):
		ret=cmp(tab1[len1-i],tab2[len2-i])
		if(ret != 0):
			return ret
		else:
			if(i==min-1):
				if(len1==len2):
					message(ERROR,"PB PB PB"+space.join(tab1)+" / "+space.join(tab2))
				if(len1>len2):
					return 1
				else:
					return -1
	return ret

# Check for security descriptors modifications return 1 if it is and 0 otherwise
# it also populate hash structure for later use in the upgrade process
def handle_security_desc(ischema,att,msgElt,hashallSD,old,new):
	if ischema == 1 and att == "defaultSecurityDescriptor"  and msgElt.flags() == ldb.FLAG_MOD_REPLACE:
		hashSD = {}
		hashSD["oldSD"] = old[0][att]
		hashSD["newSD"] = new[0][att]
		hashallSD[str(old[0].dn)] = hashSD
		return 0
	if att == "nTSecurityDescriptor"  and msgElt.flags() == ldb.FLAG_MOD_REPLACE:
		if ischema == 0:
			hashSD = {}
			hashSD["oldSD"] =  ndr_unpack(security.descriptor,str(old[0][att]))
			hashSD["newSD"] =  ndr_unpack(security.descriptor,str(new[0][att]))
			hashallSD[str(old[0].dn)] = hashSD
		return 1
	return 0

# Handle special cases ... That's when we want to update a particular attribute
# only, e.g. if it has a certain value or if it's for a certain object or
# a class of object.
# It can be also if we want to do a merge of value instead of a simple replace
def handle_special_case(att,delta,new,old,ischema):
	flag = delta.get(att).flags()
	if (att == "gPLink" or att == "gPCFileSysPath") and flag ==  ldb.FLAG_MOD_REPLACE and str(new[0].dn).lower() == str(old[0].dn).lower():
		delta.remove(att)
		return 1
	if att == "forceLogoff":
		ref=0x8000000000000000
		oldval=int(old[0][att][0])
		newval=int(new[0][att][0])
		ref == old and ref == abs(new)
		return 1
	if (att == "adminDisplayName" or att == "adminDescription") and ischema:
		return 1
	if (str(old[0].dn) == "CN=Samba4-Local-Domain,%s"%(str(names.schemadn)) and att == "defaultObjectCategory" and flag  == ldb.FLAG_MOD_REPLACE):
		return 1
#	if (str(old[0].dn) == "CN=S-1-5-11,CN=ForeignSecurityPrincipals,%s"%(str(names.rootdn)) and att == "description" and flag  == ldb.FLAG_MOD_DELETE):
#		return 1
	if (str(old[0].dn) == "CN=Title,%s"%(str(names.schemadn)) and att == "rangeUpper" and flag  == ldb.FLAG_MOD_REPLACE):
		return 1
	if ( (att == "member" or att == "servicePrincipalName") and flag  == ldb.FLAG_MOD_REPLACE):

		hash = {}
		newval = []
		changeDelta=0
		for elem in old[0][att]:
			hash[str(elem)]=1
			newval.append(str(elem))

		for elem in new[0][att]:
			if not hash.has_key(str(elem)):
				changeDelta=1
				newval.append(str(elem))
		if changeDelta == 1:
			delta[att] = ldb.MessageElement(newval, ldb.FLAG_MOD_REPLACE, att)
		else:
			delta.remove(att)
		return 1
	if (str(old[0].dn) == "%s"%(str(names.rootdn)) and att == "subRefs" and flag  == ldb.FLAG_MOD_REPLACE):
		return 1
	if str(delta.dn).endswith("CN=DisplaySpecifiers,%s"%names.configdn):
		return 1
	return 0

def update_secrets(newpaths,paths,creds,session):
	message(SIMPLE,"update secrets.ldb")
	newsecrets_ldb = Ldb(newpaths.secrets, session_info=session, credentials=creds,lp=lp)
	secrets_ldb = Ldb(paths.secrets, session_info=session, credentials=creds,lp=lp, options=["modules:samba_secrets"])
	reference = newsecrets_ldb.search(expression="dn=@MODULES",base="", scope=SCOPE_SUBTREE)
	current = secrets_ldb.search(expression="dn=@MODULES",base="", scope=SCOPE_SUBTREE)
	delta = secrets_ldb.msg_diff(current[0],reference[0])
	delta.dn = current[0].dn
	secrets_ldb.modify(delta)

	newsecrets_ldb = Ldb(newpaths.secrets, session_info=session, credentials=creds,lp=lp)
	secrets_ldb = Ldb(paths.secrets, session_info=session, credentials=creds,lp=lp)
	reference = newsecrets_ldb.search(expression="objectClass=top",base="", scope=SCOPE_SUBTREE,attrs=["dn"])
	current = secrets_ldb.search(expression="objectClass=top",base="", scope=SCOPE_SUBTREE,attrs=["dn"])
	hash_new = {}
	hash = {}
	listMissing = []
	listPresent = []

	empty = ldb.Message()
	for i in range(0,len(reference)):
		hash_new[str(reference[i]["dn"]).lower()] = reference[i]["dn"]

	# Create a hash for speeding the search of existing object in the
	# current provision
	for i in range(0,len(current)):
		hash[str(current[i]["dn"]).lower()] = current[i]["dn"]

	for k in hash_new.keys():
		if not hash.has_key(k):
			listMissing.append(hash_new[k])
		else:
			listPresent.append(hash_new[k])
	for entry in listMissing:
		reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
		current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
		delta = secrets_ldb.msg_diff(empty,reference[0])
		for att in hashAttrNotCopied.keys():
			delta.remove(att)
		message(CHANGE,"Entry %s is missing from secrets.ldb"%reference[0].dn)
		for att in delta:
			message(CHANGE," Adding attribute %s"%att)
		delta.dn = reference[0].dn
		secrets_ldb.add(delta)

	for entry in listPresent:
		reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
		current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
		delta = secrets_ldb.msg_diff(current[0],reference[0])
		i=0
		for att in hashAttrNotCopied.keys():
			delta.remove(att)
		for att in delta:
			i = i + 1

			if att == "name":
				message(CHANGE,"Found attribute name on  %s, must rename the DN "%(current[0].dn))
				identic_rename(secrets_ldb,reference[0].dn)
			else:
				delta.remove(att)


	for entry in listPresent:
		reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
		current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
		delta = secrets_ldb.msg_diff(current[0],reference[0])
		i=0
		for att in hashAttrNotCopied.keys():
			delta.remove(att)
		for att in delta:
			i = i + 1
			if att != "dn":
				message(CHANGE," Adding/Changing attribute %s to %s"%(att,current[0].dn))

		delta.dn = current[0].dn
		secrets_ldb.modify(delta)

def dump_denied_change(dn,att,flagtxt,current,reference):
	message(CHANGE, "dn= "+str(dn)+" "+att+" with flag "+flagtxt+" is not allowed to be changed/removed, I discard this change ...")
	if att != "objectSid" :
		i = 0
		for e in range(0,len(current)):
			message(CHANGE,"old %d : %s"%(i,str(current[e])))
			i=i+1
		if reference != None:
			i = 0
			for e in range(0,len(reference)):
					message(CHANGE,"new %d : %s"%(i,str(reference[e])))
					i=i+1
	else:
		message(CHANGE,"old : %s"%str(ndr_unpack( security.dom_sid,current[0])))
		message(CHANGE,"new : %s"%str(ndr_unpack( security.dom_sid,reference[0])))

#This function is for doing case by case treatment on special object

def handle_special_add(sam_ldb,dn,names):
	dntoremove=None
	if str(dn).lower() == ("CN=Certificate Service DCOM Access,CN=Builtin,%s"%names.rootdn).lower():
		#This entry was misplaced lets remove it if it exists
		dntoremove="CN=Certificate Service DCOM Access,CN=Users,%s"%names.rootdn

	if str(dn).lower() == ("CN=Cryptographic Operators,CN=Builtin,%s"%names.rootdn).lower():
		#This entry was misplaced lets remove it if it exists
		dntoremove="CN=Cryptographic Operators,CN=Users,%s"%names.rootdn

	if str(dn).lower() == ("CN=Event Log Readers,CN=Builtin,%s"%names.rootdn).lower():
		#This entry was misplaced lets remove it if it exists
		dntoremove="CN=Event Log Readers,CN=Users,%s"%names.rootdn

	if dntoremove != None:
		res = sam_ldb.search(expression="objectClass=*",base=dntoremove, scope=SCOPE_BASE,attrs=["dn"],controls=["search_options:1:2"])
		if len(res) > 0:
			message(CHANGE,"Existing object %s must be replaced by %s, removing old object"%(dntoremove,str(dn)))
			sam_ldb.delete(res[0]["dn"])

#Check if the one of the dn in the listdn will be created after the current dn
#hash is indexed by dn to be created, with each key is associated the creation order
#First dn to be created has the creation order 0, second has 1, ...
#Index contain the current creation order
def check_dn_nottobecreated(hash,index,listdn):
	if listdn == None:
		return None
	for dn in listdn:
		key = str(dn).lower()
		if hash.has_key(key) and hash[key] > index:
			return str(dn)
	return None

#This function tries to add the missing object "dn" if this object depends on some others
# the function returns 0, if the object was created 1 is returned
def add_missing_object(newsam_ldb,sam_ldb,dn,names,basedn,hash,index):
	handle_special_add(sam_ldb,dn,names)
	reference = newsam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn,
					scope=SCOPE_SUBTREE,controls=["search_options:1:2"])
	empty = ldb.Message()
	delta = sam_ldb.msg_diff(empty,reference[0])
	for att in hashAttrNotCopied.keys():
		delta.remove(att)
	for att in backlinked:
		delta.remove(att)
	depend_on_yettobecreated = None
	for att in dn_syntax_att:
		depend_on_yet_tobecreated = check_dn_nottobecreated(hash,index,delta.get(str(att)))
		if depend_on_yet_tobecreated != None:
			message(CHANGE,"Object %s depends on %s in attribute %s, delaying the creation"
							%(str(dn),depend_on_yet_tobecreated,str(att)))
			return 0
	delta.dn = dn
	message(CHANGE,"Object %s will be added"%dn)
	sam_ldb.add(delta,["relax:0"])
	return 1

def gen_dn_index_hash(listMissing):
	hash = {}
	for i in range(0,len(listMissing)):
		hash[str(listMissing[i]).lower()] = i
	return hash

def add_missing_entries(newsam_ldb,sam_ldb,names,basedn,list):
	listMissing = []
	listDefered = list

	while(len(listDefered) != len(listMissing) and len(listDefered) > 0):
		index = 0
		listMissing = listDefered
		listDefered = []
		hashMissing = gen_dn_index_hash(listMissing)
		for dn in listMissing:
			ret =  add_missing_object(newsam_ldb,sam_ldb,dn,names,basedn,hashMissing,index)
			index = index + 1
			if ret == 0:
				#DN can't be created because it depends on some other DN in the list
				listDefered.append(dn)
	if len(listDefered) != 0:
		raise ProvisioningError("Unable to insert missing elements: circular references")




# Check difference between the current provision and the reference provision.
# It looks for all objects which base DN is name. If ischema is "false" then
# the scan is done in cross partition mode.
# If "ischema" is true, then special handling is done for dealing with schema
def check_diff_name(newpaths,paths,creds,session,basedn,names,ischema):
	hash_new = {}
	hash = {}
	hashallSD = {}
	listMissing = []
	listPresent = []
	reference = []
	current = []
	# Connect to the reference provision and get all the attribute in the
	# partition referred by name
	newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
	sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp, options=["modules:samba_dsdb"])
	sam_ldb.transaction_start()
	if ischema:
		reference = newsam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"])
		current = sam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"])
	else:
		reference = newsam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"],controls=["search_options:1:2"])
		current = sam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"],controls=["search_options:1:2"])

	sam_ldb.transaction_commit()
	# Create a hash for speeding the search of new object
	for i in range(0,len(reference)):
		hash_new[str(reference[i]["dn"]).lower()] = reference[i]["dn"]

	# Create a hash for speeding the search of existing object in the
	# current provision
	for i in range(0,len(current)):
		hash[str(current[i]["dn"]).lower()] = current[i]["dn"]

	for k in hash_new.keys():
		if not hash.has_key(k):
			print hash_new[k]
			listMissing.append(hash_new[k])
		else:
			listPresent.append(hash_new[k])

	# Sort the missing object in order to have object of the lowest level
	# first (which can be containers for higher level objects)
	listMissing.sort(dn_sort)
	listPresent.sort(dn_sort)

	if ischema:
		# The following lines (up to the for loop) is to load the up to
		# date schema into our current LDB
		# a complete schema is needed as the insertion of attributes
		# and class is done against it
		# and the schema is self validated
		# The double ldb open and schema validation is taken from the
		# initial provision script
		# it's not certain that it is really needed ....
		sam_ldb = Ldb(session_info=session, credentials=creds, lp=lp)
		schema = Schema(setup_path, names.domainsid, schemadn=basedn, serverdn=str(names.serverdn))
		# Load the schema from the one we computed earlier
		sam_ldb.set_schema_from_ldb(schema.ldb)
		# And now we can connect to the DB - the schema won't be loaded
		# from the DB
		sam_ldb.connect(paths.samdb)
	else:
		sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp, options=["modules:samba_dsdb"])

	sam_ldb.transaction_start()

	message(SIMPLE,"There are %d missing objects"%(len(listMissing)))
	add_missing_entries(newsam_ldb,sam_ldb,names,basedn,listMissing)
	changed = 0
	for dn in listPresent:
		reference = newsam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn, scope=SCOPE_SUBTREE,controls=["search_options:1:2"])
		current = sam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn, scope=SCOPE_SUBTREE,controls=["search_options:1:2"])
		if ((str(current[0].dn) != str(reference[0].dn)) and (str(current[0].dn).upper() == str(reference[0].dn).upper())):
			message(CHANGE,"Name are the same but case change, let's rename %s to %s"%(str(current[0].dn),str(reference[0].dn)))
			identic_rename(sam_ldb,reference[0].dn)
			current = sam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn, scope=SCOPE_SUBTREE,controls=["search_options:1:2"])

		delta = sam_ldb.msg_diff(current[0],reference[0])
		for att in hashAttrNotCopied.keys():
			delta.remove(att)
		for att in backlinked:
			delta.remove(att)
		delta.remove("parentGUID")
		nb = 0
		
		for att in delta:
			msgElt = delta.get(att)
			if att == "dn":
				continue
			if att == "name":
				delta.remove(att)
				continue
			if handle_security_desc(ischema,att,msgElt,hashallSD,current,reference):
				delta.remove(att)
				continue
			if (not hashOverwrittenAtt.has_key(att) or not (hashOverwrittenAtt.get(att)&2^msgElt.flags())):
				if  hashOverwrittenAtt.has_key(att) and hashOverwrittenAtt.get(att)==never:
					delta.remove(att)
					continue
				if  handle_special_case(att,delta,reference,current,ischema)==0 and msgElt.flags()!=ldb.FLAG_MOD_ADD:
					i = 0
					if opts.debugchange or opts.debugall:
						try:
							dump_denied_change(dn,att,messageEltFlagToString(msgElt.flags()),current[0][att],reference[0][att])
						except:
							dump_denied_change(dn,att,messageEltFlagToString(msgElt.flags()),current[0][att],None)
					delta.remove(att)
		delta.dn = dn
		if len(delta.items()) >1:
			attributes=",".join(delta.keys())
			message(CHANGE,"%s is different from the reference one, changed attributes: %s"%(dn,attributes))
			changed = changed + 1
			sam_ldb.modify(delta)

	sam_ldb.transaction_commit()
	message(SIMPLE,"There are %d changed objects"%(changed))
	return hashallSD

# Check that SD are correct
def check_updated_sd(newpaths,paths,creds,session,names):
	newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
	sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp)
	reference = newsam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","nTSecurityDescriptor"],controls=["search_options:1:2"])
	current = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","nTSecurityDescriptor"],controls=["search_options:1:2"])
	hash_new = {}
	for i in range(0,len(reference)):
		hash_new[str(reference[i]["dn"]).lower()] = ndr_unpack(security.descriptor,str(reference[i]["nTSecurityDescriptor"])).as_sddl(names.domainsid)

	for i in range(0,len(current)):
		key = str(current[i]["dn"]).lower()
		if hash_new.has_key(key):
			sddl = ndr_unpack(security.descriptor,str(current[i]["nTSecurityDescriptor"])).as_sddl(names.domainsid)
			if sddl != hash_new[key]:
				print "%s new sddl/sddl in ref"%key
				print "%s\n%s"%(sddl,hash_new[key])

# Simple update method for updating the SD that rely on the fact that nobody
# should have modified the SD
# This assumption is safe right now (alpha9) but should be removed asap
def update_sd(paths,creds,session,names):
	sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp,options=["modules:samba_dsdb"])
	sam_ldb.transaction_start()
	# First update the SD for the rootdn
	sam_ldb.set_session_info(session)
	res = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
	delta = ldb.Message()
	delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
	descr = get_domain_descriptor(names.domainsid)
	delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
	sam_ldb.modify(delta,["recalculate_sd:0"])
	# Then the config dn
	res = sam_ldb.search(expression="objectClass=*",base=str(names.configdn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
	delta = ldb.Message()
	delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
	descr = get_config_descriptor(names.domainsid)
	delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
	sam_ldb.modify(delta,["recalculate_sd:0"])
	# Then the schema dn
	res = sam_ldb.search(expression="objectClass=*",base=str(names.schemadn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
	delta = ldb.Message()
	delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
	descr = get_schema_descriptor(names.domainsid)
	delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
	sam_ldb.modify(delta,["recalculate_sd:0"])

	# Then the rest
	hash = {}
	res = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
	for obj in res:
		if not (str(obj["dn"]) == str(names.rootdn) or
			str(obj["dn"]) == str(names.configdn) or \
			str(obj["dn"]) == str(names.schemadn)):
			hash[str(obj["dn"])] = obj["whenCreated"]

	listkeys = hash.keys()
	listkeys.sort(dn_sort)

	for key in listkeys:
		try:
			delta = ldb.Message()
			delta.dn = ldb.Dn(sam_ldb,key)
			delta["whenCreated"] = ldb.MessageElement( hash[key],ldb.FLAG_MOD_REPLACE,"whenCreated" )
			sam_ldb.modify(delta,["recalculate_sd:0"])
		except:
			sam_ldb.transaction_cancel()
			res = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","nTSecurityDescriptor"],controls=["search_options:1:2"])
			print "bad stuff" +ndr_unpack(security.descriptor,str(res[0]["nTSecurityDescriptor"])).as_sddl(names.domainsid)
			return
	sam_ldb.transaction_commit()

def rmall(topdir):
	for root, dirs, files in os.walk(topdir, topdown=False):
		for name in files:
			os.remove(os.path.join(root, name))
		for name in dirs:
			os.rmdir(os.path.join(root, name))
	os.rmdir(topdir)


def update_basesamdb(newpaths,paths,names):
	message(SIMPLE,"Copy samdb")
	shutil.copy(newpaths.samdb,paths.samdb)

	message(SIMPLE,"Update partitions filename if needed")
	schemaldb=os.path.join(paths.private_dir,"schema.ldb")
	configldb=os.path.join(paths.private_dir,"configuration.ldb")
	usersldb=os.path.join(paths.private_dir,"users.ldb")
	samldbdir=os.path.join(paths.private_dir,"sam.ldb.d")

	if not os.path.isdir(samldbdir):
		os.mkdir(samldbdir)
		os.chmod(samldbdir,0700)
	if os.path.isfile(schemaldb):
		shutil.copy(schemaldb,os.path.join(samldbdir,"%s.ldb"%str(names.schemadn).upper()))
		os.remove(schemaldb)
	if os.path.isfile(usersldb):
		shutil.copy(usersldb,os.path.join(samldbdir,"%s.ldb"%str(names.rootdn).upper()))
		os.remove(usersldb)
	if os.path.isfile(configldb):
		shutil.copy(configldb,os.path.join(samldbdir,"%s.ldb"%str(names.configdn).upper()))
		os.remove(configldb)

def update_privilege(newpaths,paths):
	message(SIMPLE,"Copy privilege")
	shutil.copy(os.path.join(newpaths.private_dir,"privilege.ldb"),os.path.join(paths.private_dir,"privilege.ldb"))

# For each partition check the differences
def update_samdb(newpaths,paths,creds,session,names):

	message(SIMPLE, "Doing schema update")
	hashdef = check_diff_name(newpaths,paths,creds,session,str(names.schemadn),names,1)
	message(SIMPLE,"Done with schema update")
	message(SIMPLE,"Scanning whole provision for updates and additions")
	hashSD = check_diff_name(newpaths,paths,creds,session,str(names.rootdn),names,0)
	message(SIMPLE,"Done with scanning")

def update_machine_account_password(paths,creds,session,names):

	secrets_ldb = Ldb(paths.secrets, session_info=session, credentials=creds,lp=lp)
	secrets_ldb.transaction_start()
	secrets_msg = secrets_ldb.search(expression=("samAccountName=%s$" % names.netbiosname), attrs=["secureChannelType"])
	sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp)
	sam_ldb.transaction_start()
	if int(secrets_msg[0]["secureChannelType"][0]) == SEC_CHAN_BDC:
		res = sam_ldb.search(expression=("samAccountName=%s$" % names.netbiosname), attrs=[])
		assert(len(res) == 1)

		msg = ldb.Message(res[0].dn)
		machinepass = glue.generate_random_password(128, 255)
		msg["userPassword"] = ldb.MessageElement(machinepass, ldb.FLAG_MOD_REPLACE, "userPassword")
		sam_ldb.modify(msg)

		res = sam_ldb.search(expression=("samAccountName=%s$" % names.netbiosname),
				     attrs=["msDs-keyVersionNumber"])
		assert(len(res) == 1)
		kvno = int(str(res[0]["msDs-keyVersionNumber"]))

		secretsdb_self_join(secrets_ldb, domain=names.domain,
				    realm=names.realm,
				    domainsid=names.domainsid,
				    dnsdomain=names.dnsdomain,
				    netbiosname=names.netbiosname,
				    machinepass=machinepass,
				    key_version_number=kvno,
				    secure_channel_type=int(secrets_msg[0]["secureChannelType"][0]))
		sam_ldb.transaction_prepare_commit()
		secrets_ldb.transaction_prepare_commit()
		sam_ldb.transaction_commit()
		secrets_ldb.transaction_commit()
	else:
		secrets_ldb.transaction_cancel()

# From here start the big steps of the program
# First get files paths
paths=get_paths(smbconf=smbconf)
paths.setup = setup_dir
def setup_path(file):
	return os.path.join(setup_dir, file)
# Guess all the needed names (variables in fact) from the current
# provision.
names = guess_names_from_current_provision(creds,session,paths)
if not sanitychecks(creds,session,names,paths):
	print "Sanity checks for the upgrade fails, checks messages and correct it before rerunning upgradeprovision"
	sys.exit(1)
# Let's see them
print_names(names)
# With all this information let's create a fresh new provision used as reference
provisiondir = newprovision(names,setup_dir,creds,session,smbconf)
# Get file paths of this new provision
newpaths = get_paths(targetdir=provisiondir)
populate_backlink(newpaths,creds,session,names.schemadn)
populate_dnsyntax(newpaths,creds,session,names.schemadn)
# Check the difference
update_basesamdb(newpaths,paths,names)

if opts.full:
	update_samdb(newpaths,paths,creds,session,names)
update_secrets(newpaths,paths,creds,session)
update_privilege(newpaths,paths)
update_machine_account_password(paths,creds,session,names)
# SD should be created with admin but as some previous acl were so wrong that admin can't modify them we have first
# to recreate them with the good form but with system account and then give the ownership to admin ...
admin_session_info = admin_session(lp, str(names.domainsid))
message(SIMPLE,"Updating SD")
update_sd(paths,creds,session,names)
update_sd(paths,creds,admin_session_info,names)
check_updated_sd(newpaths,paths,creds,session,names)
message(SIMPLE,"Upgrade finished !")
# remove reference provision now that everything is done !
rmall(provisiondir)