selftest: Have only one set of selftest knownfail and skip files

Listing tests that are not ever proposed (eg samba4.* in the samba3 selftest)
is not an error, so just combine the lists.

This is being done because some folks trying to learn how our 'make
test' works are having trouble following the distributed nature of the
selftest system.

Andrew Bartlett

5 files changed, 262 insertions, 26 deletions

diff --git a/selftest/knownfail b/selftest/knownfail
new file mode 100644
index 0000000000..7f023211f6
--- /dev/null
+++ b/selftest/knownfail
@@ -0,0 +1,109 @@
+# This file contains a list of regular expressions matching the names of
+# tests that are expected to fail.
+#
+# "make test" will not report failures for tests listed here and will consider
+# a successful run for any of these tests an error.
+
+^samba3.blackbox.failure # this is designed to fail, for testing our test infrastructure
+.*printer.*print_test_extended # fails on some hosts due to timing issues ?
+.*printer.*print_test # fails on some hosts due to timing issues ?
+^samba3.posix_s3.rap.printing # fails sometimes on sn-devel
+^samba3.posix_s3.rpc.spoolss.*printserver.enum_printers_old # fails on some hosts due to timing issues ?
+^samba3.posix_s3.rpc.spoolss.printer.*addprinterex.print_test # another intermittent failure
+^samba3.posix_s3.smb2.lock.*.rw-exclusive # another intermittent failure
+.*driver.add_driver_timestamps # we only can store dates, not timestamps
+^samba3.raw.mux.* #This test is flaky on the async lock time
+^samba3.smbtorture_s3.*OPLOCK4 # fails sometimes on sn-devel
+^samba3.posix_s3.nbt.dgram.*netlogon2
+^samba3.*rap.sam.*.useradd # Not provided by Samba 3
+^samba3.*rap.sam.*.userdelete # Not provided by Samba 3
+^samba3.*rap.basic.*.netsessiongetinfo # Not provided by Samba 3
+^samba3.posix_s3.libsmbclient .opendir # This requires a workgroup called 'WORKGROUP' and for netbios browse lists to have been registered
+^samba4.local.resolve.*.async
+^samba4.local.iconv.*.next_codepoint()
+# these show that we still have some differences between our system
+# with our internal iconv because it passes except when we bypass our
+# internal iconv modules
+^samba4.local.convert_string_handle.system.iconv.gd_ascii
+^samba4.local.convert_string_handle.system.iconv.gd_iso8859_cp850
+^samba4..*base.delete.*.deltest17
+^samba4..*base.delete.*.deltest20a
+^samba4..*base.delete.*.deltest20b
+^samba4.raw.rename.*.osxrename
+^samba4.raw.rename.*.directory rename
+^samba4.rpc.winreg.*security
+^samba4.local.registry.(dir|ldb).check hive security
+^samba4.local.registry.local.security
+^samba4.rpc.wkssvc
+^samba4.rpc.handles.*.lsarpc-shared
+^samba4.rpc.handles.*.mixed-shared
+^samba4.rpc.epmapper
+^samba4.rpc.drsuapi.*
+^samba4.rpc.lsalookup
+^samba4.rpc.cracknames
+^samba4.rpc.netlogon.*.LogonUasLogon
+^samba4.rpc.netlogon.*.LogonUasLogoff
+^samba4.rpc.netlogon.*.DatabaseSync
+^samba4.rpc.netlogon.*.DatabaseSync2
+^samba4.rpc.netlogon.*.LogonControl
+^samba4.rpc.netlogon.*.LogonControl2
+^samba4.rpc.netlogon.*.DsrEnumerateDomainTrusts
+^samba4.rpc.netlogon.*.NetrEnumerateTrustedDomains
+^samba4.rpc.netlogon.*.NetrEnumerateTrustedDomainsEx
+^samba4.rpc.netlogon.*.GetPassword
+^samba4.rpc.netlogon.*.GetTrustPasswords
+^samba4.rpc.netlogon.*.DatabaseRedo
+^samba4.rpc.netlogon.*.ServerGetTrustInfo
+^samba4.rpc.netlogon.*.GetForestTrustInformation
+^samba4.rpc.samr.passwords.badpwdcount # Not provided by Samba 4 yet
+^samba4.rpc.samr.passwords.lockout
+^samba4.base.charset.*.Testing partial surrogate
+.*net.api.delshare.*				# DelShare isn't implemented yet
+^samba4.rap.*netservergetinfo
+^samba4.rap.*netsessionenum
+^samba4.rap.*netsessiongetinfo
+^samba4.rap.*netremotetod
+^samba4.smb2.persistent.handles1
+^samba4.winbind.struct.*.show_sequence     # Not yet working in winbind
+^samba4.winbind.struct.*.getpwent          # Not yet working in winbind
+^samba4.winbind.struct.*.setpwent          # Not yet working in winbind
+^samba4.winbind.struct.*.lookup_name_sid   # Not yet working in winbind
+^samba4.winbind.struct.*.list_groups
+^samba4.*base.delaywrite.*update of write time and SMBwrite truncate$
+^samba4.*base.delaywrite.*update of write time and SMBwrite truncate expand$
+^samba4.*base.delaywrite.*delayed update of write time 3a$
+^samba4.*base.delaywrite.*delayed update of write time 3c$
+^samba4.*base.delaywrite.*update of write time using SET_END_OF_FILE$
+^samba4.*base.delaywrite.*update of write time using SET_ALLOCATION_SIZE$
+^samba4.ldap.python \(dc\).Test add_ldif\(\) with BASE64 security descriptor input using WRONG domain SID$
+# some operations don't work over the CIFS NTVFS backend yet (eg. root_fid)
+^samba4.ntvfs.cifs.*.base.createx_sharemodes_dir
+^samba4.ntvfs.cifs.*.base.maximum_allowed
+^samba4.*.base.createx_access # this test is broken for non-administrator users
+^samba4.smb2.oplock # oplocks in the s4 SMB2 server are a mess
+^samba4.raw.lock.*.async # bug 6960
+^samba4.smb2.lock.*.multiple-unlock # bug 6959
+^samba4.raw.sfileinfo.*.end-of-file # bug 6962
+^samba4.raw.oplock.*.batch22 # bug 6963
+^samba4.raw.oplock.*.brl4 # bug 7928
+^samba4.raw.lock.*.zerobyteread # bug 6974
+^samba4.smb2.lock.*.zerobyteread # bug 6974
+^samba4.raw.streams.*.delete
+^samba4.raw.streams.*.createdisp
+^samba4.raw.streams.*.sumtab
+^samba4.raw.acls.*.create_dir
+^samba4.raw.acls.*.create_file
+^samba4.smb2.create.*.acldir
+^samba4.smb2.acls.*.generic
+^samba4.smb2.acls.*.inheritflags
+^samba4.smb2.acls.*.owner
+^samba4.smb2.compound.*.related1
+^samba4.smb2.compound.*.related2
+^samba4.smb2.compound.*.invalid2
+^samba4.ldap.acl.*.search.* # ACL search behaviour not enabled by default
+^samba4.ldap.acl.*.ntSecurityDescriptor.* # ACL extended checks on search not enabled by default
+^samba4.nbt.winsreplication.owned # fails sometimes, timing related
+^samba4.ldap.dirsync.python.dc..__main__.ExtendedDirsyncTests.test_dirsync_deleted_items
+#^samba4.ldap.dirsync.python.dc..__main__.ExtendedDirsyncTests.*
+^samba4.drs.fsmo.python
+^samba4.libsmbclient.opendir.opendir # This requires netbios browsing
diff --git a/selftest/quick b/selftest/quick
new file mode 100644
index 0000000000..7605f3f887
--- /dev/null
+++ b/selftest/quick
@@ -0,0 +1,37 @@
+# This file contains regexes matching the tests that should be run
+# when doing a "quicktest" - verifying whether the build is working
+# rather than trying to see what exactly is broken.
+#
+# This should be as quick as possible but cover as much code as possible.
+base.unlink
+base.attr
+base.delete
+base.tcon
+base.open
+base.chkpath
+raw.qfsinfo
+raw.qfileinfo
+raw.mkdir
+raw.seek
+raw.open
+raw.write
+raw.read
+raw.close
+raw.ioctl
+raw.rename
+raw.eas
+base.open
+rpc.altercontext
+rpc.join
+rpc.echo
+rpc.schannel
+rpc.netlogon
+rpc.unixinfo
+rpc.handles
+rpc.altercontext
+rpc.join
+rpc.handles
+rpc.echo
+smb.signing
+drs.unit
+samba4.blackbox.dbcheck.dc
diff --git a/selftest/skip b/selftest/skip
new file mode 100644
index 0000000000..686ccdb0e7
--- /dev/null
+++ b/selftest/skip
@@ -0,0 +1,103 @@
+# This file contains a list of regular expressions matching testsuites that
+# should be skipped during "make test".
+#
+# Possible reasons for adding a testsuite here:
+# * Testsuite functionality not implemented on the server side
+# * Testsuite crashes during run
+# * Testsuite crashes server
+# * Testsuite contains "flapping" tests (sometimes success, sometimes failure)
+# * Testsuite hangs indefinitely
+#
+# If a testsuite is partially succeeding, please list the failing bits
+# in the selftest/knownfail file rather than disabling the testsuite completely.
+# That way those tests that do succeed still get run and we will be notified
+# if a known failing test suddenly starts succeeding.
+#
+# If a testsuite is very slow, please add it to selftest/slow instead.
+# This way it will still get run in "make slowtest"
+#
+# Please add a comment for each testsuite you disable explaining why
+# it is being skipped.
+^samba3.smbtorture_s3.*.randomipc
+^samba3.smbtorture_s3.*.negnowait
+^samba3.smbtorture_s3.*.nbench
+^samba3.smbtorture_s3.*.errmapextract
+^samba3.smbtorture_s3.*.trans2scan
+^samba3.smbtorture_s3.*.nttransscan
+^samba3.smbtorture_s3.*.deny1
+^samba3.smbtorture_s3.*.deny2
+^samba3.smbtorture_s3.*.openattr
+^samba3.smbtorture_s3.*.casetable
+^samba3.smbtorture_s3.*.eatest
+^samba3.smbtorture_s3.*.mangle
+^samba3.smbtorture_s3.*.utable
+^samba3.smbtorture_s3.*.pipe_number
+^samba3.*base.charset
+^samba3.*raw.acls
+^samba3.*raw.composite
+^samba3.*raw.context
+^samba3.*raw.ioctl
+^samba3.*raw.qfileinfo
+^samba3.*raw.qfsinfo
+^samba3.*raw.sfileinfo.base
+^samba4.raw.composite
+^samba4.base.iometer
+^samba4.base.casetable
+^samba4.base.nttrans
+^samba4.base.scan.maxfid
+^samba4.raw.hold-oplock 		# Not a test, but a way to block other clients for a test
+^samba4.smb2.hold-oplock 		# Not a test, but a way to block other clients for a test
+^samba4.raw.ping.pong		# Needs second server to test
+^samba4.rpc.samr.accessmask
+^samba4.raw.scan.eamax
+^samba4.smb2.notify
+^samba4.smb2.scan
+^samba4.smb2.lease
+^samba4.smb2.durable.open
+^samba4.smb2.dir
+^samba4.ntvfs.cifs.*.base.charset
+^samba4.ntvfs.cifs.*.base.iometer
+^samba4.ntvfs.cifs.*.base.casetable
+^samba4.ntvfs.cifs.*.base.nttrans
+^samba4.ntvfs.cifs.*.base.scan-maxfid
+^samba4.ntvfs.cifs.*.base.utable
+^samba4.ntvfs.cifs.*.base.smb
+^samba4.ntvfs.cifs.*.raw.
+^samba4.rpc.samsync
+^samba4.rpc.remact							# Not provided by Samba 4
+^samba4.rpc.oxidresolve						# Not provided by Samba 4
+^samba4.rpc.eventlog						# Not provided by Samba 4
+^samba4.rpc.initshutdown					# Not provided by Samba 4
+^samba4.rpc.spoolss						# Not provided by Samba 4
+^samba4.rpc.svcctl							# Not provided by Samba 4
+^samba4.rpc.atsvc							# Not provided by Samba 4
+^samba4.rpc.frsapi							# Not provided by Samba 4
+^samba4.rpc.ntsvcs							# Not provided by Samba 4
+^samba4.rpc.dfs								# Not provided by Samba 4
+^samba4.rpc.lsa.forest							# Not provided by Samba 4
+^samba4.*.base.samba3.*						# Samba3-specific test
+^samba4.*.raw.samba3.*						# Samba3-specific test
+^samba4.rpc..*samba3.*						# Samba3-specific test
+^samba4.samba-tool.domopen.*$				# Hangs for some reason
+^samba4.nss.test						# Fails
+^samba4.raw.offline 			# Samba 4 doesn't have much offline support yet
+^samba4.rpc.autoidl  # this one just generates a lot of noise, and is no longer useful
+^samba4.rpc.countcalls # this is not useful now we have full IDL
+^samba4.rap.scan # same thing here - we have docs now
+^samba4.rap.printing # Not provided by Samba 4
+^samba4.rap.sam # Not provided by Samba 4
+bench # don't run benchmarks in our selftest
+^samba4..*trans2.scan # uses huge number of file descriptors
+^samba4.*.base.scan.ioctl # bad idea in make test
+^samba4.*.base.scan.pipe_number # bad idea in make test
+^samba4.*.base.secleak # no point on build farm
+^samba4.*.base.delaywrite # This is randomly failing, depending on timing and filesystem features
+^samba4.*.base.winattr
+^samba4.*.base.birthtime
+^samba4.smb2.acls # new test which doesn't pass yet
+# ktutil might not be installed or from mit...
+# we should build a samba4ktutil and use that instead
+^samba4.blackbox.ktpass # this test isn't portable ...
+^samba4.drs.repl_schema.python # flakey test
+^samba4.smb2.ioctl # snapshots not supported by default
+^samba4.drs.delete_object.python # flakey test
diff --git a/selftest/slow b/selftest/slow
new file mode 100644
index 0000000000..7c2090c0e6
--- /dev/null
+++ b/selftest/slow
@@ -0,0 +1,9 @@
+# This file contains regexes matching tests that are very slow and
+# should be skipped during a normal test run.
+.*base.bench.holdcon.*			# Slow
+raw.bench.lookup				# Slow
+base.utable						# Slow
+base.smb						# Slow
+rpc.scanner						# Slow
+ntvfs.cifs.base.delaywrite	# It's a slow test and having it on the proxy share is not needed
+.*stress.*						# Slow
diff --git a/selftest/wscript b/selftest/wscript
index c34658a1a7..4d3d8f2841 100644
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -70,19 +70,6 @@ def set_options(opt):
 def configure(conf):
     conf.env.SELFTEST_PREFIX = Options.options.SELFTEST_PREFIX
 
-def combine_files(file1, file2, outfile):
-
-    f1 = open(file1)
-    f2 = open(file2)
-    of = open(outfile, mode='w')
-    for line in f1:
-        of.write(line)
-    for line in f2:
-        of.write(line)
-    f1.close()
-    f2.close()
-    of.close()
-
 def cmd_testonly(opt):
     '''run tests without doing a build first'''
     env = LOAD_ENVIRONMENT()
@@ -101,7 +88,7 @@ def cmd_testonly(opt):
     env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER')
     if not env.SUBUNIT_FORMATTER:
         env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
-    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${SELFTEST_DIR}/knownfail'
+    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail'
 
     if Options.options.FAIL_IMMEDIATELY:
         env.FILTER_XFAIL += ' --fail-immediately'
@@ -123,9 +110,9 @@ def cmd_testonly(opt):
 
     env.OPTIONS = '--binary-mapping=%s' % binary_mapping 
     if not Options.options.SLOWTEST:
-        env.OPTIONS += ' --exclude=${SELFTEST_DIR}/slow'
+        env.OPTIONS += ' --exclude=${srcdir}/selftest/slow'
     if Options.options.QUICKTEST:
-        env.OPTIONS += ' --quick --include=${SELFTEST_DIR}/quick'
+        env.OPTIONS += ' --quick --include=${srcdir}/selftest/quick'
     if Options.options.LOAD_LIST:
         env.OPTIONS += ' --load-list=%s' % Options.options.LOAD_LIST
     if Options.options.TESTENV:
@@ -185,20 +172,11 @@ def cmd_testonly(opt):
     if not os.path.isdir(env.SELFTEST_PREFIX):
         os.makedirs(env.SELFTEST_PREFIX, int('755', 8))
 
-    env.SELFTEST_TARGET = "samba"
-
-    for f in ["knownfail", "slow", "quick", "skip" ]:
-        combine_files(env.srcdir + "/source4/selftest/" + f,
-                      env.srcdir + "/source3/selftest/" + f,
-                      env.SELFTEST_PREFIX + "/" + f)
-
-    env.SELFTEST_DIR = env.SELFTEST_PREFIX
-
     env.TESTLISTS = ('--testlist="${PYTHON} ${srcdir}/source3/selftest/tests.py|" ' +
                      '--testlist="${PYTHON} ${srcdir}/source4/selftest/tests.py|"')
 
     # We use the full path rather than relative path because it cause problems on some plateforms (ie. solaris 8).
-    cmd = '(${PERL} ${srcdir}/selftest/selftest.pl --target=samba --prefix=${SELFTEST_PREFIX} --srcdir=${srcdir} --exclude=${SELFTEST_DIR}/skip ${TESTLISTS} ${OPTIONS} --socket-wrapper ${TESTS} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS} | tee ${SELFTEST_PREFIX}/subunit'
+    cmd = '(${PERL} ${srcdir}/selftest/selftest.pl --target=samba --prefix=${SELFTEST_PREFIX} --srcdir=${srcdir} --exclude=${srcdir}/selftest/skip ${TESTLISTS} ${OPTIONS} --socket-wrapper ${TESTS} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS} | tee ${SELFTEST_PREFIX}/subunit'
 
     if os.environ.get('RUN_FROM_BUILD_FARM') is None and not Options.options.FILTERED_SUBUNIT:
         cmd += ' | ${FORMAT_TEST_OUTPUT}'