diff options
| -rwxr-xr-x | selftest/filter-subunit | 15 | ||||
| -rw-r--r-- | selftest/subunithelper.py | 28 | ||||
| -rw-r--r-- | selftest/wscript | 2 | ||||
| -rwxr-xr-x | source3/selftest/s3-selftest.sh | 2 |
4 files changed, 40 insertions, 7 deletions
diff --git a/selftest/filter-subunit b/selftest/filter-subunit index 5f062d12a2..ef3172171e 100755 --- a/selftest/filter-subunit +++ b/selftest/filter-subunit @@ -17,9 +17,11 @@ parser = optparse.OptionParser("filter-subunit [options] < instream > outstream" parser.add_option("--expected-failures", type="string", help="File containing list of regexes matching tests to consider known " "failures") -parser.add_option("--strip-passed-output", action="store_true", +parser.add_option("--flapping", type="string", + help="File containing list of flapping tests, of which to ignore results.") +parser.add_option("--strip-passed-output", action="store_true", help="Whether to strip output from tests that passed") -parser.add_option("--fail-immediately", action="store_true", +parser.add_option("--fail-immediately", action="store_true", help="Whether to stop on the first error", default=False) parser.add_option("--prefix", type="string", help="Add prefix to all test names") @@ -42,6 +44,12 @@ if opts.expected_failures: else: expected_failures = {} + +if opts.flapping: + flapping = subunithelper.read_test_regexes(opts.flapping) +else: + flapping = [] + statistics = { 'TESTS_UNEXPECTED_OK': 0, 'TESTS_EXPECTED_OK': 0, @@ -58,7 +66,8 @@ signal.signal(signal.SIGINT, handle_sigint) out = subunithelper.SubunitOps(sys.stdout) msg_ops = subunithelper.FilterOps(out, opts.prefix, expected_failures, opts.strip_passed_output, - fail_immediately=opts.fail_immediately) + fail_immediately=opts.fail_immediately, + flapping=flapping) try: ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin) diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py index c59b6d002c..c29bdbabf0 100644 --- a/selftest/subunithelper.py +++ b/selftest/subunithelper.py @@ -288,6 +288,8 @@ class FilterOps(testtools.testresult.TestResult): def addFailure(self, test, details=None): test = self._add_prefix(test) xfail_reason = find_in_list(self.expected_failures, test.id()) + if xfail_reason is None: + xfail_reason = find_in_list(self.flapping, test.id()) if xfail_reason is not None: self.xfail_added+=1 self.total_xfail+=1 @@ -308,7 +310,21 @@ class FilterOps(testtools.testresult.TestResult): def addSuccess(self, test, details=None): test = self._add_prefix(test) - self._ops.addSuccess(test, details) + xfail_reason = find_in_list(self.expected_failures, test.id()) + if xfail_reason is not None: + self.uxsuccess_added += 1 + self.total_uxsuccess += 1 + if details is not None: + details = subunit.RemoteError(unicode(details[1]) + xfail_reason.decode("utf-8")) + else: + details = subunit.RemoteError(xfail_reason.decode("utf-8")) + self._ops.addUnexpectedSuccess(test, details) + if self.output: + self._ops.output_msg(self.output) + if self.fail_immediately: + raise ImmediateFail() + else: + self._ops.addSuccess(test, details) self.output = None def skip_testsuite(self, name, reason=None): @@ -319,6 +335,7 @@ class FilterOps(testtools.testresult.TestResult): self.error_added = 0 self.fail_added = 0 self.xfail_added = 0 + self.uxsuccess_added = 0 def end_testsuite(self, name, result, reason=None): xfail = False @@ -346,7 +363,8 @@ class FilterOps(testtools.testresult.TestResult): self._ops.end_testsuite(name, result, reason) def __init__(self, out, prefix=None, expected_failures=None, - strip_ok_output=False, fail_immediately=False): + strip_ok_output=False, fail_immediately=False, + flapping=None): self._ops = out self.seen_output = False self.output = None @@ -355,12 +373,18 @@ class FilterOps(testtools.testresult.TestResult): self.expected_failures = expected_failures else: self.expected_failures = {} + if flapping is not None: + self.flapping = flapping + else: + self.flapping = {} self.strip_ok_output = strip_ok_output self.xfail_added = 0 self.fail_added = 0 + self.uxsuccess_added = 0 self.total_xfail = 0 self.total_error = 0 self.total_fail = 0 + self.total_uxsuccess = 0 self.error_added = 0 self.fail_immediately = fail_immediately diff --git a/selftest/wscript b/selftest/wscript index e1308e16b0..67ea5a4ddc 100644 --- a/selftest/wscript +++ b/selftest/wscript @@ -88,7 +88,7 @@ def cmd_testonly(opt): env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER') if not env.SUBUNIT_FORMATTER: env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate' - env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --expected-failure=${srcdir}/selftest/flapping' + env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping' if Options.options.FAIL_IMMEDIATELY: env.FILTER_XFAIL += ' --fail-immediately' diff --git a/source3/selftest/s3-selftest.sh b/source3/selftest/s3-selftest.sh index db0a8a0ac1..0d20324c68 100755 --- a/source3/selftest/s3-selftest.sh +++ b/source3/selftest/s3-selftest.sh @@ -1,6 +1,6 @@ #!/bin/sh -FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail --expected-failures=${SELFTESTDIR}/flapping" +FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail --flapping=${SELFTESTDIR}/flapping" if [ "x${SUBUNIT_FORMATTER}" = x"" ]; then SUBUNIT_FORMATTER="${PYTHON} -u ${SELFTESTDIR}/format-subunit --prefix=${SELFTESTPREFIX} --immediate" fi |