selftest: Add --flapping argument to filter-subunit.

3 files changed, 39 insertions, 6 deletions

diff --git a/selftest/filter-subunit b/selftest/filter-subunit
index 5f062d12a2..ef3172171e 100755
--- a/selftest/filter-subunit
+++ b/selftest/filter-subunit
@@ -17,9 +17,11 @@ parser = optparse.OptionParser("filter-subunit [options] < instream > outstream"
 parser.add_option("--expected-failures", type="string", 
 	help="File containing list of regexes matching tests to consider known "
 	     "failures")
-parser.add_option("--strip-passed-output", action="store_true", 
+parser.add_option("--flapping", type="string",
+	help="File containing list of flapping tests, of which to ignore results.")
+parser.add_option("--strip-passed-output", action="store_true",
     help="Whether to strip output from tests that passed")
-parser.add_option("--fail-immediately", action="store_true", 
+parser.add_option("--fail-immediately", action="store_true",
     help="Whether to stop on the first error", default=False)
 parser.add_option("--prefix", type="string",
 	help="Add prefix to all test names")
@@ -42,6 +44,12 @@ if opts.expected_failures:
 else:
 	expected_failures = {}
 
+
+if opts.flapping:
+	flapping = subunithelper.read_test_regexes(opts.flapping)
+else:
+	flapping = []
+
 statistics = {
 	'TESTS_UNEXPECTED_OK': 0,
 	'TESTS_EXPECTED_OK': 0,
@@ -58,7 +66,8 @@ signal.signal(signal.SIGINT, handle_sigint)
 out = subunithelper.SubunitOps(sys.stdout)
 msg_ops = subunithelper.FilterOps(out, opts.prefix, expected_failures,
 				  opts.strip_passed_output,
-				  fail_immediately=opts.fail_immediately)
+				  fail_immediately=opts.fail_immediately,
+				  flapping=flapping)
 
 try:
 	ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py
index c59b6d002c..c29bdbabf0 100644
--- a/selftest/subunithelper.py
+++ b/selftest/subunithelper.py
@@ -288,6 +288,8 @@ class FilterOps(testtools.testresult.TestResult):
     def addFailure(self, test, details=None):
         test = self._add_prefix(test)
         xfail_reason = find_in_list(self.expected_failures, test.id())
+        if xfail_reason is None:
+            xfail_reason = find_in_list(self.flapping, test.id())
         if xfail_reason is not None:
             self.xfail_added+=1
             self.total_xfail+=1
@@ -308,7 +310,21 @@ class FilterOps(testtools.testresult.TestResult):
 
     def addSuccess(self, test, details=None):
         test = self._add_prefix(test)
-        self._ops.addSuccess(test, details)
+        xfail_reason = find_in_list(self.expected_failures, test.id())
+        if xfail_reason is not None:
+            self.uxsuccess_added += 1
+            self.total_uxsuccess += 1
+            if details is not None:
+                details = subunit.RemoteError(unicode(details[1]) + xfail_reason.decode("utf-8"))
+            else:
+                details = subunit.RemoteError(xfail_reason.decode("utf-8"))
+            self._ops.addUnexpectedSuccess(test, details)
+            if self.output:
+                self._ops.output_msg(self.output)
+            if self.fail_immediately:
+                raise ImmediateFail()
+        else:
+            self._ops.addSuccess(test, details)
         self.output = None
 
     def skip_testsuite(self, name, reason=None):
@@ -319,6 +335,7 @@ class FilterOps(testtools.testresult.TestResult):
         self.error_added = 0
         self.fail_added = 0
         self.xfail_added = 0
+        self.uxsuccess_added = 0
 
     def end_testsuite(self, name, result, reason=None):
         xfail = False
@@ -346,7 +363,8 @@ class FilterOps(testtools.testresult.TestResult):
         self._ops.end_testsuite(name, result, reason)
 
     def __init__(self, out, prefix=None, expected_failures=None,
-                 strip_ok_output=False, fail_immediately=False):
+                 strip_ok_output=False, fail_immediately=False,
+                 flapping=None):
         self._ops = out
         self.seen_output = False
         self.output = None
@@ -355,12 +373,18 @@ class FilterOps(testtools.testresult.TestResult):
             self.expected_failures = expected_failures
         else:
             self.expected_failures = {}
+        if flapping is not None:
+            self.flapping = flapping
+        else:
+            self.flapping = {}
         self.strip_ok_output = strip_ok_output
         self.xfail_added = 0
         self.fail_added = 0
+        self.uxsuccess_added = 0
         self.total_xfail = 0
         self.total_error = 0
         self.total_fail = 0
+        self.total_uxsuccess = 0
         self.error_added = 0
         self.fail_immediately = fail_immediately
 
diff --git a/selftest/wscript b/selftest/wscript
index e1308e16b0..67ea5a4ddc 100644
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -88,7 +88,7 @@ def cmd_testonly(opt):
     env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER')
     if not env.SUBUNIT_FORMATTER:
         env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
-    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --expected-failure=${srcdir}/selftest/flapping'
+    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping'
 
     if Options.options.FAIL_IMMEDIATELY:
         env.FILTER_XFAIL += ' --fail-immediately'