summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJelmer Vernooij <jelmer@samba.org>2011-12-04 00:23:02 +0100
committerJelmer Vernooij <jelmer@samba.org>2011-12-05 23:11:04 +0100
commite62654578b3aa1ddafd8faf85b65d9702f49fba3 (patch)
treef6ab370e1d31389220acc5678f55d4097dc78686
parent5643d37a000971070b22af7c2710f66b49b57eff (diff)
downloadsamba-e62654578b3aa1ddafd8faf85b65d9702f49fba3.tar.gz
samba-e62654578b3aa1ddafd8faf85b65d9702f49fba3.tar.bz2
samba-e62654578b3aa1ddafd8faf85b65d9702f49fba3.zip
selftest: Add --flapping argument to filter-subunit.
-rwxr-xr-xselftest/filter-subunit15
-rw-r--r--selftest/subunithelper.py28
-rw-r--r--selftest/wscript2
-rwxr-xr-xsource3/selftest/s3-selftest.sh2
4 files changed, 40 insertions, 7 deletions
diff --git a/selftest/filter-subunit b/selftest/filter-subunit
index 5f062d12a2..ef3172171e 100755
--- a/selftest/filter-subunit
+++ b/selftest/filter-subunit
@@ -17,9 +17,11 @@ parser = optparse.OptionParser("filter-subunit [options] < instream > outstream"
parser.add_option("--expected-failures", type="string",
help="File containing list of regexes matching tests to consider known "
"failures")
-parser.add_option("--strip-passed-output", action="store_true",
+parser.add_option("--flapping", type="string",
+ help="File containing list of flapping tests, of which to ignore results.")
+parser.add_option("--strip-passed-output", action="store_true",
help="Whether to strip output from tests that passed")
-parser.add_option("--fail-immediately", action="store_true",
+parser.add_option("--fail-immediately", action="store_true",
help="Whether to stop on the first error", default=False)
parser.add_option("--prefix", type="string",
help="Add prefix to all test names")
@@ -42,6 +44,12 @@ if opts.expected_failures:
else:
expected_failures = {}
+
+if opts.flapping:
+ flapping = subunithelper.read_test_regexes(opts.flapping)
+else:
+ flapping = []
+
statistics = {
'TESTS_UNEXPECTED_OK': 0,
'TESTS_EXPECTED_OK': 0,
@@ -58,7 +66,8 @@ signal.signal(signal.SIGINT, handle_sigint)
out = subunithelper.SubunitOps(sys.stdout)
msg_ops = subunithelper.FilterOps(out, opts.prefix, expected_failures,
opts.strip_passed_output,
- fail_immediately=opts.fail_immediately)
+ fail_immediately=opts.fail_immediately,
+ flapping=flapping)
try:
ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py
index c59b6d002c..c29bdbabf0 100644
--- a/selftest/subunithelper.py
+++ b/selftest/subunithelper.py
@@ -288,6 +288,8 @@ class FilterOps(testtools.testresult.TestResult):
def addFailure(self, test, details=None):
test = self._add_prefix(test)
xfail_reason = find_in_list(self.expected_failures, test.id())
+ if xfail_reason is None:
+ xfail_reason = find_in_list(self.flapping, test.id())
if xfail_reason is not None:
self.xfail_added+=1
self.total_xfail+=1
@@ -308,7 +310,21 @@ class FilterOps(testtools.testresult.TestResult):
def addSuccess(self, test, details=None):
test = self._add_prefix(test)
- self._ops.addSuccess(test, details)
+ xfail_reason = find_in_list(self.expected_failures, test.id())
+ if xfail_reason is not None:
+ self.uxsuccess_added += 1
+ self.total_uxsuccess += 1
+ if details is not None:
+ details = subunit.RemoteError(unicode(details[1]) + xfail_reason.decode("utf-8"))
+ else:
+ details = subunit.RemoteError(xfail_reason.decode("utf-8"))
+ self._ops.addUnexpectedSuccess(test, details)
+ if self.output:
+ self._ops.output_msg(self.output)
+ if self.fail_immediately:
+ raise ImmediateFail()
+ else:
+ self._ops.addSuccess(test, details)
self.output = None
def skip_testsuite(self, name, reason=None):
@@ -319,6 +335,7 @@ class FilterOps(testtools.testresult.TestResult):
self.error_added = 0
self.fail_added = 0
self.xfail_added = 0
+ self.uxsuccess_added = 0
def end_testsuite(self, name, result, reason=None):
xfail = False
@@ -346,7 +363,8 @@ class FilterOps(testtools.testresult.TestResult):
self._ops.end_testsuite(name, result, reason)
def __init__(self, out, prefix=None, expected_failures=None,
- strip_ok_output=False, fail_immediately=False):
+ strip_ok_output=False, fail_immediately=False,
+ flapping=None):
self._ops = out
self.seen_output = False
self.output = None
@@ -355,12 +373,18 @@ class FilterOps(testtools.testresult.TestResult):
self.expected_failures = expected_failures
else:
self.expected_failures = {}
+ if flapping is not None:
+ self.flapping = flapping
+ else:
+ self.flapping = {}
self.strip_ok_output = strip_ok_output
self.xfail_added = 0
self.fail_added = 0
+ self.uxsuccess_added = 0
self.total_xfail = 0
self.total_error = 0
self.total_fail = 0
+ self.total_uxsuccess = 0
self.error_added = 0
self.fail_immediately = fail_immediately
diff --git a/selftest/wscript b/selftest/wscript
index e1308e16b0..67ea5a4ddc 100644
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -88,7 +88,7 @@ def cmd_testonly(opt):
env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER')
if not env.SUBUNIT_FORMATTER:
env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
- env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --expected-failure=${srcdir}/selftest/flapping'
+ env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping'
if Options.options.FAIL_IMMEDIATELY:
env.FILTER_XFAIL += ' --fail-immediately'
diff --git a/source3/selftest/s3-selftest.sh b/source3/selftest/s3-selftest.sh
index db0a8a0ac1..0d20324c68 100755
--- a/source3/selftest/s3-selftest.sh
+++ b/source3/selftest/s3-selftest.sh
@@ -1,6 +1,6 @@
#!/bin/sh
-FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail --expected-failures=${SELFTESTDIR}/flapping"
+FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail --flapping=${SELFTESTDIR}/flapping"
if [ "x${SUBUNIT_FORMATTER}" = x"" ]; then
SUBUNIT_FORMATTER="${PYTHON} -u ${SELFTESTDIR}/format-subunit --prefix=${SELFTESTPREFIX} --immediate"
fi