r26593: - More work on the python versions of samba3dump and the samba3sam tests.

- Initial work converting the upgrade code to Python.
- Removed the old EJS upgrade code because it has been broken for a long time.
(This used to be commit 150cf39fbd4fe088546870fb0d8f20c0d9eb4aca)

10 files changed, 132 insertions, 968 deletions

diff --git a/source4/dsdb/samdb/ldb_modules/tests/samba3sam.py b/source4/dsdb/samdb/ldb_modules/tests/samba3sam.py
index 6a4935bf4d..8ca92e152e 100644
--- a/source4/dsdb/samdb/ldb_modules/tests/samba3sam.py
+++ b/source4/dsdb/samdb/ldb_modules/tests/samba3sam.py
@@ -27,35 +27,33 @@ import ldb
 from samba import Ldb, substitute_var
 from samba.tests import LdbTestCase, TestCaseInTempDir
 
-datadir = sys.argv[2]
+datadir = os.path.join(os.path.dirname(__file__), "../../../../../testdata/samba3")
 
 class Samba3SamTestCase(TestCaseInTempDir):
     def setup_data(self, obj, ldif):
         self.assertTrue(ldif is not None)
         obj.db.add_ldif(substitute_var(ldif, obj.substvars))
 
-    def setup_modules(self, ldb, s3, s4, ldif):
-        self.assertTrue(ldif is not None)
-        ldb.add_ldif(substitute_var(ldif, s4.substvars))
+    def setup_modules(self, ldb, s3, s4):
 
         ldif = """
 dn: @MAP=samba3sam
-@FROM: """ + s4.substvars["BASEDN"] + """
-@TO: sambaDomainName=TESTS,""" + s3.substvars["BASEDN"] + """
+@FROM: """ + s4.basedn + """
+@TO: sambaDomainName=TESTS,""" + s3.basedn + """
 
 dn: @MODULES
 @LIST: rootdse,paged_results,server_sort,extended_dn,asq,samldb,password_hash,operational,objectguid,rdn_name,samba3sam,partition
 
 dn: @PARTITION
-partition: """ + s4.substvars["BASEDN"] + ":" + s4.url + """
-partition: """ + s3.substvars["BASEDN"] + ":" + s3.url + """
+partition: """ + s4.basedn + ":" + s4.url + """
+partition: """ + s3.basedn + ":" + s3.url + """
 replicateEntries: @SUBCLASSES
 replicateEntries: @ATTRIBUTES
 replicateEntries: @INDEXLIST
 """
         ldb.add_ldif(ldif)
 
-    def test_s3sam_search(self, ldb):
+    def _test_s3sam_search(self, ldb):
         print "Looking up by non-mapped attribute"
         msg = ldb.search(expression="(cn=Administrator)")
         self.assertEquals(len(msg), 1)
@@ -91,7 +89,7 @@ replicateEntries: @INDEXLIST
                    (str(msg[i].dn) == "unixName=nobody,ou=Users,dc=vernstok,dc=nl"))
 
 
-    def test_s3sam_modify(ldb, s3):
+    def _test_s3sam_modify(ldb, s3):
         print "Adding a record that will be fallbacked"
         ldb.add_ldif("""
 dn: cn=Foo
@@ -205,16 +203,15 @@ delete: description
         msg = ldb.search(expression="(cn=Niemand2)")
         self.assertEquals(len(msg), 0)
 
-    def test_map_search(ldb, s3, s4):
+    def _test_map_search(self, ldb, s3, s4):
         print "Running search tests on mapped data"
         ldif = """
-dn: """ + "sambaDomainName=TESTS,""" + s3.substvars["BASEDN"] + """
+dn: """ + "sambaDomainName=TESTS,""" + s3.basedn + """
 objectclass: sambaDomain
 objectclass: top
 sambaSID: S-1-5-21-4231626423-2410014848-2360679739
 sambaNextRid: 2000
 sambaDomainName: TESTS"""
-        self.assertTrue(ldif is not None)
         s3.db.add_ldif(substitute_var(ldif, s3.substvars))
 
         print "Add a set of split records"
@@ -252,7 +249,6 @@ lastLogon: z
 description: y
 """
 
-        self.assertTrue(ldif is not None)
         ldb.add_ldif(substitute_var(ldif, s4.substvars))
 
         print "Add a set of remote records"
@@ -284,7 +280,6 @@ sambaBadPasswordCount: y
 sambaLogonTime: z
 description: y
 """
-        self.assertTrue(ldif is not None)
         s3.add_ldif(substitute_var(ldif, s3.substvars))
 
         print "Testing search by DN"
@@ -678,7 +673,7 @@ description: y
         for dn in dns:
             ldb.delete(dn)
 
-    def test_map_modify(self, ldb, s3, s4):
+    def _test_map_modify(self, ldb, s3, s4):
         print "Running modification tests on mapped data"
 
         print "Testing modification of local records"
@@ -1002,66 +997,70 @@ revision: 2
     def setUp(self):
         super(Samba3SamTestCase, self).setUp()
 
-        def make_dn(rdn):
-            return rdn + ",sambaDomainName=TESTS," + this.substvars["BASEDN"]
-
-        def make_s4dn(rdn):
-            return rdn + "," + this.substvars["BASEDN"]
+        def make_dn(basedn, rdn):
+            return rdn + ",sambaDomainName=TESTS," + basedn
 
-        ldb = Ldb()
+        def make_s4dn(basedn, rdn):
+            return rdn + "," + basedn
 
-        ldbfile = os.path.join(self.tempdir, "test.ldb")
-        ldburl = "tdb://" + ldbfile
+        self.ldbfile = os.path.join(self.tempdir, "test.ldb")
+        self.ldburl = "tdb://" + self.ldbfile
 
         tempdir = self.tempdir
+        print tempdir
 
         class Target:
+            """Simple helper class that contains data for a specific SAM connection."""
             def __init__(self, file, basedn, dn):
                 self.file = os.path.join(tempdir, file)
                 self.url = "tdb://" + self.file
-                self.substvars = {"BASEDN": basedn}
+                self.basedn = basedn
+                self.substvars = {"BASEDN": self.basedn}
                 self.db = Ldb()
-                self.dn = dn
-
-        samba4 = Target("samba4.ldb", "dc=vernstok,dc=nl", make_s4dn)
-        samba3 = Target("samba3.ldb", "cn=Samba3Sam", make_dn)
-        templates = Target("templates.ldb", "cn=templates", None)
-
-        ldb.connect(ldburl)
-        samba3.db.connect(samba3.url)
-        templates.db.connect(templates.url)
-        samba4.db.connect(samba4.url)
-
-        self.setup_data(samba3, open(os.path.join(datadir, "samba3.ldif"), 'r').read())
-        self.setup_data(templates, open(os.path.join(datadir, "provision_samba3sam_templates.ldif"), 'r').read())
-        self.setup_modules(ldb, samba3, samba4, open(os.path.join(datadir, "provision_samba3sam.ldif"), 'r').read())
-
-        ldb = Ldb()
-        ldb.connect(ldburl)
-
-        self.test_s3sam_search(ldb)
-        self.test_s3sam_modify(ldb, samba3)
-
-        os.unlink(ldbfile)
-        os.unlink(samba3.file)
-        os.unlink(templates.file)
-        os.unlink(samba4.file)
+                self._dn = dn
+
+            def dn(self, rdn):
+                return self._dn(rdn, self.basedn)
+
+            def connect(self):
+                return self.db.connect(self.url)
+
+        self.samba4 = Target("samba4.ldb", "dc=vernstok,dc=nl", make_s4dn)
+        self.samba3 = Target("samba3.ldb", "cn=Samba3Sam", make_dn)
+        self.templates = Target("templates.ldb", "cn=templates", None)
+
+        self.samba3.connect()
+        self.templates.connect()
+        self.samba4.connect()
+
+    def tearDown(self):
+        super(Samba3SamTestCase, self).tearDown()
+        os.unlink(self.ldbfile)
+        os.unlink(self.samba3.file)
+        os.unlink(self.templates.file)
+        os.unlink(self.samba4.file)
+
+    def test_s3sam(self):
+        ldb = Ldb(self.ldburl)
+        self.setup_data(self.samba3, open(os.path.join(datadir, "samba3.ldif"), 'r').read())
+        self.setup_data(self.templates, open(os.path.join(datadir, "provision_samba3sam_templates.ldif"), 'r').read())
+        ldif = open(os.path.join(datadir, "provision_samba3sam.ldif"), 'r').read()
+        ldb.add_ldif(substitute_var(ldif, s4.substvars))
+        self.setup_modules(ldb, self.samba3, self.samba4)
 
-        ldb = Ldb()
-        ldb.connect(ldburl)
-        samba3.db = Ldb()
-        samba3.db.connect(samba3.url)
-        templates.db = Ldb()
-        templates.db.connect(templates.url)
-        samba4.db = Ldb()
-        samba4.db.connect(samba4.url)
+        ldb = Ldb(self.ldburl)
 
-        self.setup_data(templates, open(os.path.join(datadir, "provision_samba3sam_templates.ldif"), 'r').read())
-        self.setup_modules(ldb, samba3, samba4, open(os.path.join(datadir, "provision_samba3sam.ldif"), 'r').read())
+        self._test_s3sam_search(ldb)
+        self._test_s3sam_modify(ldb, self.samba3)
 
-        ldb = Ldb()
-        ldb.connect(ldburl)
+    def test_map(self):
+        ldb = Ldb(self.ldburl)
+        self.setup_data(self.templates, open(os.path.join(datadir, "provision_samba3sam_templates.ldif"), 'r').read())
+        ldif = open(os.path.join(datadir, "provision_samba3sam.ldif"), 'r').read()
+        ldb.add_ldif(substitute_var(ldif, s4.substvars))
+        self.setup_modules(ldb, self.samba3, self.samba4)
 
-        test_map_search(ldb, samba3, samba4)
-        test_map_modify(ldb, samba3, samba4)
+        ldb = Ldb(self.ldburl)
+        self._test_map_search(ldb, self.samba3, self.samba4)
+        self._test_map_modify(ldb, self.samba3, self.samba4)
 
diff --git a/source4/lib/ldb_wrap.c b/source4/lib/ldb_wrap.c
index d0abb5808a..63049b06fc 100644
--- a/source4/lib/ldb_wrap.c
+++ b/source4/lib/ldb_wrap.c
@@ -140,7 +140,7 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
 		return NULL;
 	}
 	
-	if (strcmp(lp_sam_url(lp_ctx), url) == 0) {
+	if (lp_ctx != NULL && strcmp(lp_sam_url(lp_ctx), url) == 0) {
 		dsdb_set_global_schema(ldb);
 	}
 
diff --git a/source4/scripting/bin/samba3dump b/source4/scripting/bin/samba3dump
index 157a708ff6..f8d10cbc71 100755
--- a/source4/scripting/bin/samba3dump
+++ b/source4/scripting/bin/samba3dump
@@ -47,7 +47,6 @@ def print_samba3_policy(pol):
 
 def print_samba3_sam(samdb):
     print_header("SAM Database")
-    
     for user in samdb:
         print "%s" % user
 
@@ -55,10 +54,8 @@ def print_samba3_shares(shares):
     print_header("Configured shares")
     for s in shares:
         print "--- %s ---" % s.name
-
         for p in s:
             print "\t%s = %s" % (p.key, p.value)
-
         print ""
 
 def print_samba3_secrets(secrets):
diff --git a/source4/scripting/libjs/upgrade.js b/source4/scripting/libjs/upgrade.js
deleted file mode 100644
index 3a548fe34b..0000000000
--- a/source4/scripting/libjs/upgrade.js
+++ /dev/null
@@ -1,687 +0,0 @@
-/*
-	backend code for upgrading from Samba3
-	Copyright Jelmer Vernooij 2005
-	Released under the GNU GPL v2 or later
-*/
-
-libinclude("base.js");
-
-function regkey_to_dn(name)
-{
-	var dn = "hive=NONE";
-	var i = 0;
-
-	var as = split("/", name);
-
-	for (i in as) {
-		if (i > 0) {
-			dn = sprintf("key=%s,", as[i]) + dn;
-		}
-	}
-
-	return dn;
-}
-
-/* Where prefix is any of:
- * - HKLM
- *   HKU
- *   HKCR
- *   HKPD
- *   HKPT
- */
-
-function upgrade_registry(regdb,prefix,ldb)
-{
-	assert(regdb != undefined);
-	var prefix_up = strupper(prefix);
-	var ldif = new Array();
-
-	for (var i in regdb.keys) {
-		var rk = regdb.keys[i];
-		var pts = split("/", rk.name);
-
-		/* Only handle selected hive */
-		if (strupper(pts[0]) != prefix_up) {
-			continue;
-		}
-
-		var keydn = regkey_to_dn(rk.name);
-
-		var pts = split("/", rk.name);
-
-		/* Convert key name to dn */
-		ldif[rk.name] = sprintf("
-dn: %s
-name: %s
-
-", keydn, pts[0]);
-		
-		for (var j in rk.values) {
-			var rv = rk.values[j];
-
-			ldif[rk.name + " (" + rv.name + ")"] = sprintf("
-dn: %s,value=%s
-value: %s
-type: %d
-data:: %s", keydn, rv.name, rv.name, rv.type, ldb.encode(rv.data));
-		}
-	}
-
-	return ldif;
-}
-
-function upgrade_sam_policy(samba3,dn)
-{
-	var ldif = sprintf("
-dn: %s
-changetype: modify
-replace: minPwdLength
-minPwdLength: %d
-pwdHistoryLength: %d
-minPwdAge: %d
-maxPwdAge: %d
-lockoutDuration: %d
-samba3ResetCountMinutes: %d
-samba3UserMustLogonToChangePassword: %d
-samba3BadLockoutMinutes: %d
-samba3DisconnectTime: %d
-
-", dn, samba3.policy.min_password_length, 
-	samba3.policy.password_history, samba3.policy.minimum_password_age,
-	samba3.policy.maximum_password_age, samba3.policy.lockout_duration,
-	samba3.policy.reset_count_minutes, samba3.policy.user_must_logon_to_change_password,
-	samba3.policy.bad_lockout_minutes, samba3.policy.disconnect_time
-);
-	
-	return ldif;
-}
-
-function upgrade_sam_account(ldb,acc,domaindn,domainsid)
-{
-	if (acc.nt_username == undefined) {
-		acc.nt_username = acc.username;
-	}	
-
-	if (acc.nt_username == "") {
-		acc.nt_username = acc.username;
-	}	
-
-	if (acc.fullname == undefined) {
-		var pw = nss.getpwnam(acc.fullname);
-		acc.fullname = pw.pw_gecos;
-	}
-
-	var pts = split(',', acc.fullname);
-	acc.fullname = pts[0];
-
-	if (acc.fullname == undefined) {
-		acc.fullname = acc.username;
-	}
-	
-	assert(acc.fullname != undefined);
-	assert(acc.nt_username != undefined);
-
-	var ldif = sprintf(
-"dn: cn=%s,%s
-objectClass: top
-objectClass: user
-lastLogon: %d
-lastLogoff: %d
-unixName: %s
-sAMAccountName: %s
-cn: %s
-description: %s
-primaryGroupID: %d
-badPwdcount: %d
-logonCount: %d
-samba3Domain: %s
-samba3DirDrive: %s
-samba3MungedDial: %s
-samba3Homedir: %s
-samba3LogonScript: %s
-samba3ProfilePath: %s
-samba3Workstations: %s
-samba3KickOffTime: %d
-samba3BadPwdTime: %d
-samba3PassLastSetTime: %d
-samba3PassCanChangeTime: %d
-samba3PassMustChangeTime: %d
-objectSid: %s-%d
-lmPwdHash:: %s
-ntPwdHash:: %s
-
-", ldb.dn_escape(acc.fullname), domaindn, acc.logon_time, acc.logoff_time, acc.username, acc.nt_username, acc.nt_username, 
-
-acc.acct_desc, acc.group_rid, acc.bad_password_count, acc.logon_count,
-acc.domain, acc.dir_drive, acc.munged_dial, acc.homedir, acc.logon_script, 
-acc.profile_path, acc.workstations, acc.kickoff_time, acc.bad_password_time, 
-acc.pass_last_set_time, acc.pass_can_change_time, acc.pass_must_change_time, domainsid, acc.user_rid,
-	ldb.encode(acc.lm_pw), ldb.encode(acc.nt_pw)); 
-
-	return ldif;
-}
-
-function upgrade_sam_group(grp,domaindn)
-{
-	var nss = nss_init();
-
-	var gr;
-	if (grp.sid_name_use == 5) { // Well-known group
-		return undefined;
-	}
-
-	if (grp.nt_name == "Domain Guests" ||
-	    grp.nt_name == "Domain Users" ||
-	    grp.nt_name == "Domain Admins") {
-		return undefined;
-	}
-	
-	if (grp.gid == -1) {
-		gr = nss.getgrnam(grp.nt_name);
-	} else {
-		gr = nss.getgrgid(grp.gid);
-	}
-
-	if (gr == undefined) {
-		grp.unixname = "UNKNOWN";
-	} else {
-		grp.unixname = gr.gr_name;
-	}
-
-	assert(grp.unixname != undefined);
-	
-	var ldif = sprintf(
-"dn: cn=%s,%s
-objectClass: top
-objectClass: group
-description: %s
-cn: %s
-objectSid: %s
-unixName: %s
-samba3SidNameUse: %d
-", grp.nt_name, domaindn, 
-grp.comment, grp.nt_name, grp.sid, grp.unixname, grp.sid_name_use);
-
-	return ldif;
-}
-
-function upgrade_winbind(samba3,domaindn)
-{
-	var ldif = sprintf("
-		
-dn: dc=none
-userHwm: %d
-groupHwm: %d
-
-", samba3.idmap.user_hwm, samba3.idmap.group_hwm);
-
-	for (var i in samba3.idmap.mappings) {
-		var m = samba3.idmap.mappings[i];
-		ldif = ldif + sprintf("
-dn: SID=%s,%s
-SID: %s
-type: %d
-unixID: %d", m.sid, domaindn, m.sid, m.type, m.unix_id);
-	}
-	
-	return ldif;
-}
-*/
-
-function upgrade_wins(samba3)
-{
-	var ldif = "";
-	var version_id = 0;
-
-	for (i in samba3.winsentries) {
-		var rType;
-		var rState;
-		var nType;
-		var numIPs = 0;
-		var e = samba3.winsentries[i];
-		var now = sys.nttime();
-		var ttl = sys.unix2nttime(e.ttl);
-
-		version_id++;
-
-		for (var i in e.ips) {
-			numIPs++;
-		}
-
-		if (e.type == 0x1C) {
-			rType = 0x2;
-		} else if (sys.bitAND(e.type, 0x80)) {
-			if (numIPs > 1) {
-				rType = 0x2;
-			} else {
-				rType = 0x1;
-			}
-		} else {
-			if (numIPs > 1) {
-				rType = 0x3;
-			} else {
-				rType = 0x0;
-			}
-		}
-
-		if (ttl > now) {
-			rState = 0x0;/* active */
-		} else {
-			rState = 0x1;/* released */		
-		}
-
-		nType = (sys.bitAND(e.nb_flags,0x60)>>5);
-
-		ldif = ldif + sprintf("
-dn: name=%s,type=0x%02X
-type: 0x%02X
-name: %s
-objectClass: winsRecord
-recordType: %u
-recordState: %u
-nodeType: %u
-isStatic: 0
-expireTime: %s
-versionID: %llu
-", e.name, e.type, e.type, e.name, 
-   rType, rState, nType, 
-   sys.ldaptime(ttl), version_id);
-
-		for (var i in e.ips) {
-			ldif = ldif + sprintf("address: %s\n", e.ips[i]);
-		}
-	}
-
-	ldif = ldif + sprintf("
-dn: CN=VERSION
-objectClass: winsMaxVersion
-maxVersion: %llu
-", version_id);
-
-	return ldif;
-}
-
-function upgrade_provision(samba3)
-{
-	var subobj = new Object();
-	var nss = nss_init();
-	var lp = loadparm_init();
-	var rdn_list;
-
-	var domainname = samba3.configuration.get("workgroup");
-	
-	if (domainname == undefined) {
-		domainname = samba3.secrets.domains[0].name;
-		println("No domain specified in smb.conf file, assuming '" + domainname + "'");
-	}
-	
-	var domsec = samba3.find_domainsecrets(domainname);
-	var hostsec = samba3.find_domainsecrets(hostname());
-	var realm = samba3.configuration.get("realm");
-
-	if (realm == undefined) {
-		realm = domainname;
-		println("No realm specified in smb.conf file, assuming '" + realm + "'");
-	}
-	random_init(local);
-
-	subobj.REALM        = realm;
-	subobj.DOMAIN       = domainname;
-	subobj.HOSTNAME     = hostname();
-
-	assert(subobj.REALM);
-	assert(subobj.DOMAIN);
-	assert(subobj.HOSTNAME);
-
-	subobj.HOSTIP       = hostip();
-	if (domsec != undefined) {
-		subobj.DOMAINGUID   = domsec.guid;
-		subobj.DOMAINSID    = domsec.sid;
-	} else {
-		println("Can't find domain secrets for '" + domainname + "'; using random SID and GUID");
-		subobj.DOMAINGUID = randguid();
-		subobj.DOMAINSID = randsid();
-	}
-	
-	if (hostsec) {
-		subobj.HOSTGUID     = hostsec.guid;
-	} else {
-		subobj.HOSTGUID = randguid();
-	}
-	subobj.INVOCATIONID = randguid();
-	subobj.KRBTGTPASS   = randpass(12);
-	subobj.MACHINEPASS  = randpass(12);
-	subobj.ADMINPASS    = randpass(12);
-	subobj.DEFAULTSITE  = "Default-First-Site-Name";
-	subobj.NEWGUID      = randguid;
-	subobj.NTTIME       = nttime;
-	subobj.LDAPTIME     = ldaptime;
-	subobj.DATESTRING   = datestring;
-	subobj.ROOT         = findnss(nss.getpwnam, "root");
-	subobj.NOBODY       = findnss(nss.getpwnam, "nobody");
-	subobj.NOGROUP      = findnss(nss.getgrnam, "nogroup", "nobody");
-	subobj.WHEEL        = findnss(nss.getgrnam, "wheel", "root");
-	subobj.USERS        = findnss(nss.getgrnam, "users", "guest", "other");
-	subobj.DNSDOMAIN    = strlower(subobj.REALM);
-	subobj.DNSNAME      = sprintf("%s.%s", 
-				      strlower(subobj.HOSTNAME), 
-				      subobj.DNSDOMAIN);
-	subobj.BASEDN       = "DC=" + join(",DC=", split(".", subobj.REALM));
-	rdn_list = split(".", subobj.DNSDOMAIN);
-	subobj.DOMAINDN     = "DC=" + join(",DC=", rdn_list);
-	subobj.DOMAINDN_LDB = "users.ldb";
-	subobj.ROOTDN       = subobj.DOMAINDN;
-
-	modules_list        = new Array("rootdse",
-					"kludge_acl",
-					"paged_results",
-					"server_sort",
-					"extended_dn",
-					"asq",
-					"samldb",
-					"password_hash",
-					"operational",
-					"objectclass",
-					"rdn_name",
-					"show_deleted",
-					"partition");
-	subobj.MODULES_LIST = join(",", modules_list);
-
-	return subobj;
-}
-
-smbconf_keep = new Array(
-	"dos charset", 
-	"unix charset",
-	"display charset",
-	"comment",
-	"path",
-	"directory",
-	"workgroup",
-	"realm",
-	"netbios name",
-	"netbios aliases",
-	"netbios scope",
-	"server string",
-	"interfaces",
-	"bind interfaces only",
-	"security",
-	"auth methods",
-	"encrypt passwords",
-	"null passwords",
-	"obey pam restrictions",
-	"password server",
-	"smb passwd file",
-	"private dir",
-	"passwd chat",
-	"password level",
-	"lanman auth",
-	"ntlm auth",
-	"client NTLMv2 auth",
-	"client lanman auth",
-	"client plaintext auth",
-	"read only",
-	"hosts allow",
-	"hosts deny",
-	"log level",
-	"debuglevel",
-	"log file",
-	"smb ports",
-	"large readwrite",
-	"max protocol",
-	"min protocol",
-	"unicode",
-	"read raw",
-	"write raw",
-	"disable netbios",
-	"nt status support",
-	"announce version",
-	"announce as",
-	"max mux",
-	"max xmit",
-	"name resolve order",
-	"max wins ttl",
-	"min wins ttl",
-	"time server",
-	"unix extensions",
-	"use spnego",
-	"server signing",
-	"client signing",
-	"max connections",
-	"paranoid server security",
-	"socket options",
-	"strict sync",
-	"max print jobs",
-	"printable",
-	"print ok",
-	"printer name",
-	"printer",
-	"map system",
-	"map hidden",
-	"map archive",
-	"preferred master",
-	"prefered master",
-	"local master",
-	"browseable",
-	"browsable",
-	"wins server",
-	"wins support",
-	"csc policy",
-	"strict locking",
-	"preload",
-	"auto services",
-	"lock dir",
-	"lock directory",
-	"pid directory",
-	"socket address",
-	"copy",
-	"include",
-	"available",
-	"volume",
-	"fstype",
-	"panic action",
-	"msdfs root",
-	"host msdfs",
-	"winbind separator");
-
-/*
-   Remove configuration variables not present in Samba4
-	oldconf: Old configuration structure
-	mark: Whether removed configuration variables should be 
-		kept in the new configuration as "samba3:<name>"
- */
-function upgrade_smbconf(oldconf,mark)
-{
-	var data = oldconf.data();
-	var newconf = param_init();
-
-	for (var s in data) {
-		for (var p in data[s]) {
-			var keep = false;
-			for (var k in smbconf_keep) { 
-				if (smbconf_keep[k] == p) {
-					keep = true;
-					break;
-				}
-			}
-
-			if (keep) {
-				newconf.set(s, p, oldconf.get(s, p));
-			} else if (mark) {
-				newconf.set(s, "samba3:"+p, oldconf.get(s,p));
-			}
-		}
-	}
-
-	if (oldconf.get("domain logons") == "True") {
-		newconf.set("server role", "domain controller");
-	} else {
-		if (oldconf.get("security") == "user") {
-			newconf.set("server role", "standalone");
-		} else {
-			newconf.set("server role", "member server");
-		}
-	}
-
-	return newconf;
-}
-
-function upgrade(subobj, samba3, message, paths, session_info, credentials)
-{
-	var ret = 0;
-	var lp = loadparm_init();
-	var samdb = ldb_init();
-	samdb.session_info = session_info;
-	samdb.credentials = credentials;
-	var ok = samdb.connect(paths.samdb);
-	if (!ok) {
-		info.message("samdb connect failed: " + samdb.errstring() + "\n");
-		assert(ok);
-	}
-
-	message("Writing configuration\n");
-	var newconf = upgrade_smbconf(samba3.configuration,true);
-	newconf.save(paths.smbconf);
-
-	message("Importing account policies\n");
-	var ldif = upgrade_sam_policy(samba3,subobj.BASEDN);
-	ok = samdb.modify(ldif);
-	if (!ok) {
-		message("samdb load failed: " + samdb.errstring() + "\n");
-		assert(ok);
-	}
-	var regdb = ldb_init();
-	ok = regdb.connect(paths.hklm);
-	if (!ok) {
-		message("registry connect: " + regdb.errstring() + "\n");
-		assert(ok);
-	}
-
-	ok = regdb.modify(sprintf("
-dn: value=RefusePasswordChange,key=Parameters,key=Netlogon,key=Services,key=CurrentControlSet,key=System,HIVE=NONE
-replace: type
-type: 4
-replace: data
-data: %d
-", samba3.policy.refuse_machine_password_change));
-	if (!ok) {
-		message("registry load failed: " + regdb.errstring() + "\n");
-		assert(ok);
-	}
-
-	message("Importing users\n");
-	for (var i in samba3.samaccounts) {
-		var msg = "... " + samba3.samaccounts[i].username;
-		var ldif = upgrade_sam_account(samdb,samba3.samaccounts[i],subobj.BASEDN,subobj.DOMAINSID);
-		ok = samdb.add(ldif);
-		if (!ok && samdb.errstring() != "Record exists") { 
-			msg = msg + "... error: " + samdb.errstring();
-			ret = ret + 1; 
-		}
-		message(msg + "\n");
-	}
-
-	message("Importing groups\n");
-	for (var i in samba3.groupmappings) {
-		var msg = "... " + samba3.groupmappings[i].nt_name;
-		var ldif = upgrade_sam_group(samba3.groupmappings[i],subobj.BASEDN);
-		if (ldif != undefined) {
-			ok = samdb.add(ldif);
-			if (!ok && samdb.errstring() != "Record exists") { 
-				msg = msg + "... error: " + samdb.errstring();
-				ret = ret + 1; 
-			}
-		}
-		message(msg + "\n");
-	}
-
-	message("Importing registry data\n");
-	var hives = new Array("hkcr","hkcu","hklm","hkpd","hku","hkpt"); 
-	for (var i in hives) {
-		var hn = hives[i];
-		message("... " + hn + "\n");
-		regdb = ldb_init();
-		ok = regdb.connect(paths[hn]);
-		assert(ok);
-		var ldif = upgrade_registry(samba3.registry, hn, regdb);
-		for (var j in ldif) {
-			var msg = "... ... " + j;
-			ok = regdb.add(ldif[j]);
-			if (!ok && regdb.errstring() != "Record exists") { 
-				msg = msg + "... error: " + regdb.errstring();
-				ret = ret + 1; 
-			}
-			message(msg + "\n");
-		}
-	}
-
-
-	message("Importing WINS data\n");
-	var winsdb = ldb_init();
-	ok = winsdb.connect(paths.winsdb);
-	assert(ok);
-	ldb_erase(winsdb);
-
-	var ldif = upgrade_wins(samba3);
-	ok = winsdb.add(ldif);
-	assert(ok);
-
-	// figure out ldapurl, if applicable
-	var ldapurl = undefined;
-	var pdb = samba3.configuration.get_list("passdb backend");
-	if (pdb != undefined) {
-		for (var b in pdb) {
-			if (strlen(pdb[b]) >= 7) {
-				if (substr(pdb[b], 0, 7) == "ldapsam") {
-					ldapurl = substr(pdb[b], 8);
-				}
-			}
-		}
-	}
-
-	// URL was not specified in passdb backend but ldap /is/ used
-	if (ldapurl == "") {
-		ldapurl = "ldap://" + samba3.configuration.get("ldap server");
-	}
-
-	// Enable samba3sam module if original passdb backend was ldap
-	if (ldapurl != undefined) {
-		message("Enabling Samba3 LDAP mappings for SAM database\n");
-
-		ok = samdb.modify("
-dn: @MODULES
-changetype: modify
-replace: @LIST
-@LIST: samldb,operational,objectguid,rdn_name,samba3sam
-");
-		if (!ok) {
-			message("Error enabling samba3sam module: " + samdb.errstring() + "\n");
-			ret = ret + 1;
-		}
-
-		ok = samdb.add(sprintf("
-dn: @MAP=samba3sam
-@MAP_URL: %s", ldapurl));
-		assert(ok);
-
-	}
-
-	return ret;
-}
-
-function upgrade_verify(subobj, samba3,paths,message)
-{
-	message("Verifying account policies\n");
-	var samldb = ldb_init();
-	var ne = 0;
-
-	var ok = samldb.connect(paths.samdb);
-	assert(ok);
-
-	for (var i in samba3.samaccounts) {
-		var msg = samldb.search("(&(sAMAccountName=" + samba3.samaccounts[i].nt_username + ")(objectclass=user))");
-		assert(msg.length >= 1);
-	}
-	
-	// FIXME
-}
diff --git a/source4/scripting/python/samba/samba3.py b/source4/scripting/python/samba/samba3.py
index d125e3164b..b4261f7c74 100644
--- a/source4/scripting/python/samba/samba3.py
+++ b/source4/scripting/python/samba/samba3.py
@@ -167,7 +167,7 @@ class SecretsDatabase:
     def get_auth_user(self):
         return self.tdb.get("SECRETS/AUTH_USER")
 
-    def get_dom_guid(self, host):
+    def get_domain_guid(self, host):
         return self.tdb.get("SECRETS/DOMGUID/%s" % host)
 
     def ldap_dns(self):
diff --git a/source4/scripting/python/samba/tests/__init__.py b/source4/scripting/python/samba/tests/__init__.py
index b01807c02f..5885a3b507 100644
--- a/source4/scripting/python/samba/tests/__init__.py
+++ b/source4/scripting/python/samba/tests/__init__.py
@@ -43,7 +43,7 @@ class TestCaseInTempDir(unittest.TestCase):
 
     def tearDown(self):
         super(TestCaseInTempDir, self).tearDown()
-        # FIXME: Remove all files in self.tempdir
+        os.rmdir(self.tempdir)
 
 
 class SubstituteVarTestCase(unittest.TestCase):
diff --git a/source4/scripting/python/samba/upgrade.py b/source4/scripting/python/samba/upgrade.py
index 375c39eb5a..c13351bc63 100644
--- a/source4/scripting/python/samba/upgrade.py
+++ b/source4/scripting/python/samba/upgrade.py
@@ -255,68 +255,44 @@ maxVersion: %llu
 
     return ldif
 
-def upgrade_provision(lp, samba3):
-    domainname = samba3.configuration.get("workgroup")
+def upgrade_provision(samba3, setup_dir, message, credentials, session_info, paths):
+    oldconf = samba3.get_conf()
+
+    if oldconf.get("domain logons") == "True":
+        serverrole = "domain controller"
+    else:
+        if oldconf.get("security") == "user":
+            serverrole = "standalone"
+        else:
+            serverrole = "member server"
+
+    domainname = oldconf.get("workgroup")
+    realm = oldconf.get("realm")
+    netbiosname = oldconf.get("netbios name")
+
+    secrets_db = samba3.get_secrets_db()
     
     if domainname is None:
-        domainname = samba3.secrets.domains[0].name
-        print "No domain specified in smb.conf file, assuming '%s'\n" % domainname
+        domainname = secrets_db.domains()[0]
+        message("No domain specified in smb.conf file, assuming '%s'" % domainname)
     
-    domsec = samba3.find_domainsecrets(domainname)
-    hostsec = samba3.find_domainsecrets(hostname())
-    realm = samba3.configuration.get("realm")
-
     if realm is None:
-        realm = domainname
-        print "No realm specified in smb.conf file, assuming '%s'\n" % realm
-    random_init(local)
+        realm = domainname.lower()
+        message("No realm specified in smb.conf file, assuming '%s'\n" % realm)
 
-    subobj.realm        = realm
-    subobj.domain       = domainname
-
-    if domsec is not None:
-        subobj.DOMAINGUID   = domsec.guid
-        subobj.DOMAINSID    = domsec.sid
+    domainguid = secrets_db.get_domain_guid(domainname)
+    domainsid = secrets_db.get_sid(domainsid)
+    if domainsid is None:
+        message("Can't find domain secrets for '%s'; using random SID\n" % domainname)
+    
+    if netbiosname is not None:
+        machinepass = secrets_db.get_machine_password(netbiosname)
     else:
-        print "Can't find domain secrets for '%s'; using random SID and GUID\n" % domainname
-        subobj.DOMAINGUID = uuid.random()
-        subobj.DOMAINSID = randsid()
+        netbiosname = None
     
-    if hostsec:
-        hostguid = hostsec.guid
-    subobj.krbtgtpass   = randpass(12)
-    subobj.machinepass  = randpass(12)
-    subobj.adminpass    = randpass(12)
-    subobj.datestring   = datestring()
-    subobj.root         = findnss(pwd.getpwnam, "root")[4]
-    subobj.nobody       = findnss(pwd.getpwnam, "nobody")[4]
-    subobj.nogroup      = findnss(grp.getgrnam, "nogroup", "nobody")[2]
-    subobj.wheel        = findnss(grp.getgrnam, "wheel", "root")[2]
-    subobj.users        = findnss(grp.getgrnam, "users", "guest", "other")[2]
-    subobj.dnsdomain    = subobj.realm.lower()
-    subobj.dnsname      = "%s.%s" % (subobj.hostname.lower(), subobj.dnsdomain)
-    subobj.basedn       = "DC=" + ",DC=".join(subobj.realm.split("."))
-    rdn_list = subobj.dnsdomain.split(".")
-    subobj.domaindn     = "DC=" + ",DC=".join(rdn_list)
-    subobj.domaindn_ldb = "users.ldb"
-    subobj.rootdn       = subobj.domaindn
-
-    modules_list        = ["rootdse",
-                    "kludge_acl",
-                    "paged_results",
-                    "server_sort",
-                    "extended_dn",
-                    "asq",
-                    "samldb",
-                    "password_hash",
-                    "operational",
-                    "objectclass",
-                    "rdn_name",
-                    "show_deleted",
-                    "partition"]
-    subobj.modules_list = ",".join(modules_list)
-
-    return subobj
+    provision(lp, setup_dir, message, blank=True, paths=path, session_info=session_info, 
+              credentials=credentials, realm=realm, domain=domainname, 
+              domainsid=domainsid, domainguid=domainguid, machinepass=machinepass, serverrole=serverrole)
 
 smbconf_keep = [
     "dos charset", 
@@ -435,14 +411,6 @@ def upgrade_smbconf(oldconf,mark):
             elif mark:
                 newconf.set(s, "samba3:"+p, oldconf.get(s,p))
 
-    if oldconf.get("domain logons") == "True":
-        newconf.set("server role", "domain controller")
-    else:
-        if oldconf.get("security") == "user":
-            newconf.set("server role", "standalone")
-        else:
-            newconf.set("server role", "member server")
-
     return newconf
 
 def upgrade(subobj, samba3, message, paths, session_info, credentials):
diff --git a/source4/selftest/samba4_tests.sh b/source4/selftest/samba4_tests.sh
index edcf51a8e0..59c7635c19 100755
--- a/source4/selftest/samba4_tests.sh
+++ b/source4/selftest/samba4_tests.sh
@@ -241,10 +241,6 @@ DATADIR=$samba4srcdir/../testdata
 plantest "parse samba3" none $samba4bindir/smbscript $DATADIR/samba3/verify $CONFIGURATION $DATADIR/samba3
 plantest "js.samba3sam" none $SCRIPTDIR/samba3sam.js $CONFIGURATION `pwd` $DATADIR/samba3/
 
-rm -rf $PREFIX/upgrade
-mkdir -p $PREFIX/upgrade
-#plantest "upgrade" none $samba4bindir/smbscript setup/upgrade $CONFIGURATION --verify --targetdir=$PREFIX ../testdata/samba3 ../testdata/samba3/smb.conf
-
 # Domain Member Tests
 
 plantest "RPC-ECHO against member server with local creds" member $VALGRIND $smb4torture ncacn_np:"\$NETBIOSNAME" -U"\$NETBIOSNAME/\$USERNAME"%"\$PASSWORD" RPC-ECHO "$*"
@@ -301,4 +297,7 @@ then
 	plantest "samba3.python" none PYTHONPATH=bin/python:scripting/python $PYTHON scripting/bin/subunitrun samba.tests.samba3
 	plantest "samba3sam.python" none PYTHONPATH=bin/python:scripting/python $PYTHON dsdb/samdb/ldb_modules/tests/samba3sam.py `pwd` $DATADIR/samba3/
 	plantest "ldap.python" dc $PYTHON $samba4srcdir/lib/ldb/tests/python/ldap.py \$SERVER -U\$USERNAME%\$PASSWORD
+	rm -rf $PREFIX/upgrade
+	mkdir -p $PREFIX/upgrade
+	plantest "blackbox.upgrade" none $PYTHON setup/upgrade.py $CONFIGURATION --verify --targetdir=$PREFIX ../testdata/samba3 ../testdata/samba3/smb.conf
 fi
diff --git a/source4/setup/upgrade b/source4/setup/upgrade
deleted file mode 100755
index f05e22f2e0..0000000000
--- a/source4/setup/upgrade
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/sh
-exec smbscript "$0" ${1+"$@"}
-/*
-	Upgrade from Samba3
-	Copyright Jelmer Vernooij 2005
-	Released under the GNU GPL v2 or later
-*/
-
-options = GetOptions(ARGV,
-		"POPT_AUTOHELP",
-		"POPT_COMMON_SAMBA",
-		"POPT_COMMON_VERSION",
-		"POPT_COMMON_CREDENTIALS",
-		'verify',
-		'targetdir=s',
-		'quiet', 
-		'realm',
-		'blank');
-
-if (options == undefined) {
-   println("Failed to parse options");
-   return -1;
-}
-
-libinclude("base.js");
-libinclude("provision.js");
-libinclude("upgrade.js");
-
-/*
-  print a message if quiet is not set
-*/
-function message() 
-{
-	if (options["quiet"] == undefined) {
-		print(vsprintf(arguments));
-	}
-}
-
-/*
- show some help
-*/
-function ShowHelp()
-{
-	print("
-Samba4 import tool
-
-provision [options] <libdir> <smbconf>
- --targetdir=DIR    Output to specified directory
- --quiet            Be quiet
- --blank            Do not add users or groups, just the structure
- --realm=REALM	    Override realm to use
-
-");
-	exit(1);
-}
-
-if (options.ARGV.length != 2) {
-	ShowHelp();
-	exit(1);
-}
-
-var lp = loadparm_init();
-
-message("Reading Samba3 databases and smb.conf\n");
-var samba3 = samba3_read(options.ARGV[0], options.ARGV[1]);
-
-if (samba3 == undefined) {
-	println("Error reading Samba3 data");
-	exit(1);
-}
-
-
-
-message("Provisioning\n");
-var subobj = upgrade_provision(samba3);
-var paths;
-if (options.targetdir != undefined) {
-	paths = new Object();
-	paths.smbconf = sprintf("%s/smb.conf", options.targetdir);
-	var ldbs = new Array("hklm","hkcr","hku","hkcu","hkpd","hkpt","samdb","rootdse","secrets","wins");
-	for (var i in ldbs) {
-		var n = ldbs[i];
-		paths[n] = sprintf("tdb://%s/%s.ldb", options.targetdir, n);
-	}
-	paths.dns = options.targetdir+"/dns.zone";
-} else {
-	paths = provision_default_paths(subobj);;
-}
-
-var creds = options.get_credentials();
-var system_session = system_session();
-var paths = provision_default_paths(subobj);
-
-if (options.realm != undefined) {
-	subobj.REALM = options.realm;
-}
-
-provision(subobj, message, options.blank, paths, system_session, creds, undefined);
-
-var ret = upgrade(subobj,samba3,message,paths, system_session, creds);
-if (ret > 0) {
-	message("Failed to import %d entries\n", ret);
-} else {
-	provision_dns(subobj, message, paths, system_session, creds);
-
-	message("All OK\n");
-}
-
-if (options.verify != undefined) {
-	message("Verifying...\n");
-	ret = upgrade_verify(subobj, samba3,paths,message);
-}
-
-return ret;
diff --git a/source4/setup/upgrade.py b/source4/setup/upgrade.py
index 96584a1026..186ad3772b 100755
--- a/source4/setup/upgrade.py
+++ b/source4/setup/upgrade.py
@@ -6,15 +6,20 @@
 #
 import getopt
 import optparse
-import sys
+import os, sys
 sys.path.append("scripting/python")
+import param
 import samba
-import samba.getopt
+import samba.getopt as options
+from samba.provision import provision_default_paths
 
-parser = optparse.OptionParser("upgrade [options]")
+parser = optparse.OptionParser("upgrade [options] <libdir> <smbconf>")
 parser.add_option_group(options.SambaOptions(parser))
 parser.add_option_group(options.VersionOptions(parser))
-parser.add_option_group(options.CredentialsOptions(parser))
+credopts = options.CredentialsOptions(parser)
+parser.add_option_group(credopts)
+parser.add_option("--setupdir", type="string", metavar="DIR", 
+		help="directory with setup files")
 parser.add_option("--realm", type="string", metavar="REALM", help="set realm")
 parser.add_option("--quiet", help="Be quiet")
 parser.add_option("--verify", help="Verify resulting configuration")
@@ -23,44 +28,41 @@ parser.add_option("--blank",
 parser.add_option("--targetdir", type="string", metavar="DIR", 
 		          help="Set target directory")
 
-opts = parser.parse_args()[0]
+opts, args = parser.parse_args()
 
 def message(text):
     """Print a message if quiet is not set."""
     if opts.quiet:
         print text
 
+if len(args) < 1:
+    parser.print_usage()
+    sys.exit(1)
+from samba.samba3 import Samba3
 message("Reading Samba3 databases and smb.conf\n")
-samba3 = samba3_read(options.ARGV[0], options.ARGV[1])
-
-message("Provisioning\n")
-subobj = upgrade_provision(samba3)
-if options.targetdir is not None:
-	paths = ProvisionPaths()
-	paths.smbconf = os.path.join(options.targetdir, "smb.conf")
-	ldbs = ["hklm","hkcr","hku","hkcu","hkpd","hkpt","samdb","rootdse","secrets","wins"]
-	for n in ldbs:
-		paths[n] = sprintf("tdb://%s/%s.ldb", options.targetdir, n)
-	paths.dns = os.path.join(options.targetdir, "dns.zone")
+libdir = args[0]
+if not os.path.isdir(libdir):
+    print "error: %s is not a directory"
+    sys.exit(1)
+if len(args) > 1:
+    smbconf = args[1]
 else:
-	paths = provision_default_paths(subobj)
+    smbconf = os.path.join(libdir, "smb.conf")
+samba3 = Samba3(libdir, smbconf)
 
-creds = options.get_credentials()
-system_session = system_session()
-paths = provision_default_paths(subobj)
+from samba.upgrade import upgrade_provision
 
-if options.realm:
-	subobj.realm = options.realm
+message("Provisioning\n")
 
-provision(lp, subobj, message, options.blank, paths, system_session, creds, undefined)
+setup_dir = opts.setupdir
+if setup_dir is None:
+	setup_dir = "setup"
 
-ret = upgrade(subobj,samba3,message,paths, system_session, creds)
-if ret > 0:
-	message("Failed to import %d entries\n", ret)
-else:
-	provision_dns(subobj, message, paths, system_session, creds)
-	message("All OK\n")
+creds = credopts.get_credentials()
+lp = param.LoadParm()
+lp.load(opts.configfile)
+upgrade_provision(samba3, setup_dir, message, credentials=creds, session_info=system_session())
 
-if options.verify:
+if opts.verify:
 	message("Verifying...\n")
 	ret = upgrade_verify(subobj, samba3, paths, message)