diff options
Diffstat (limited to 'selftest')
| -rw-r--r-- | selftest/Subunit.pm | 29 | ||||
| -rwxr-xr-x | selftest/filter-subunit | 49 | ||||
| -rwxr-xr-x | selftest/filter-subunit.pl | 100 | ||||
| -rwxr-xr-x | selftest/format-subunit | 40 | ||||
| -rwxr-xr-x | selftest/selftest.pl | 6 | ||||
| -rw-r--r-- | selftest/subunithelper.py | 164 |
6 files changed, 260 insertions, 128 deletions
diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm index 2a9fc0e48b..42a9ad08aa 100644 --- a/selftest/Subunit.pm +++ b/selftest/Subunit.pm @@ -95,8 +95,6 @@ sub parse_results($$$) } } elsif (/^testsuite: (.*)\n/) { $msg_ops->start_testsuite($1); - } elsif (/^testsuite-count: (\d+)\n/) { - $msg_ops->testsuite_count($1); } else { $msg_ops->output_msg($_); } @@ -176,6 +174,27 @@ sub report_time($) printf "time: %04d-%02d-%02d %02d:%02d:%02d\n", $year+1900, $mon+1, $mday, $hour, $min, $sec; } +sub progress_pop() +{ + print "progress: pop\n"; +} + +sub progress_push() +{ + print "progress: push\n"; +} + +sub progress($;$) +{ + my ($count, $whence) = @_; + + unless(defined($whence)) { + $whence = ""; + } + + print "progress: $whence$count\n"; +} + # The following are Samba extensions: sub start_testsuite($) @@ -208,10 +227,4 @@ sub end_testsuite($$;$) } } -sub testsuite_count($) -{ - my ($count) = @_; - print "testsuite-count: $count\n"; -} - 1; diff --git a/selftest/filter-subunit b/selftest/filter-subunit new file mode 100755 index 0000000000..605a89840a --- /dev/null +++ b/selftest/filter-subunit @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# Filter a subunit stream +# Copyright (C) Jelmer Vernooij <jelmer@samba.org> +# Published under the GNU GPL, v3 or later + +import optparse +import os +import sys +import signal + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools")) + +import subunithelper + +parser = optparse.OptionParser("filter-subunit [options] < instream > outstream") +parser.add_option("--expected-failures", type="string", + help="File containing list of regexes matching tests to consider known " + "failures") +parser.add_option("--strip-passed-output", action="store_true", + help="Whether to strip output from tests that passed") + +parser.add_option("--prefix", type="string", + help="Add prefix to all test names") + +opts, args = parser.parse_args() + +if opts.expected_failures: + expected_failures = list(subunithelper.read_test_regexes(opts.expected_failures)) +else: + expected_failures = [] + +statistics = { + 'TESTS_UNEXPECTED_OK': 0, + 'TESTS_EXPECTED_OK': 0, + 'TESTS_UNEXPECTED_FAIL': 0, + 'TESTS_EXPECTED_FAIL': 0, + 'TESTS_ERROR': 0, + 'TESTS_SKIP': 0, +} + +def handle_sigint(sig, stack): + sys.exit(0) +signal.signal(signal.SIGINT, handle_sigint) + +msg_ops = subunithelper.FilterOps(opts.prefix, expected_failures, + opts.strip_passed_output) + +sys.exit(subunithelper.parse_results(msg_ops, statistics, sys.stdin)) diff --git a/selftest/filter-subunit.pl b/selftest/filter-subunit.pl deleted file mode 100755 index 5e87ef49f6..0000000000 --- a/selftest/filter-subunit.pl +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/perl -# Filter a subunit stream -# Copyright (C) Jelmer Vernooij <jelmer@samba.org> -# Published under the GNU GPL, v3 or later - -=pod - -=head1 NAME - -filter-subunit - Filter a subunit stream - -=head1 SYNOPSIS - -filter-subunit --help - -filter-subunit --prefix=PREFIX --known-failures=FILE < in-stream > out-stream - -=head1 DESCRIPTION - -Simple Subunit stream filter that will change failures to known failures -based on a list of regular expressions. - -=head1 OPTIONS - -=over 4 - -=item I<--prefix> - -Add the specified prefix to all test names. - -=item I<--expected-failures> - -Specify a file containing a list of tests that are expected to fail. Failures -for these tests will be counted as successes, successes will be counted as -failures. - -The format for the file is, one entry per line: - -TESTSUITE-NAME.TEST-NAME - -The reason for a test can also be specified, by adding a hash sign (#) and the reason -after the test name. - -=head1 LICENSE - -selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>. - - -=head1 AUTHOR - -Jelmer Vernooij - -=cut - -use Getopt::Long; -use strict; -use FindBin qw($RealBin $Script); -use lib "$RealBin"; -use Subunit qw(parse_results); -use Subunit::Filter; - -my $opt_expected_failures = undef; -my $opt_help = 0; -my $opt_prefix = undef; -my $opt_strip_ok_output = 0; -my @expected_failures = (); - -my $result = GetOptions( - 'expected-failures=s' => \$opt_expected_failures, - 'strip-passed-output' => \$opt_strip_ok_output, - 'prefix=s' => \$opt_prefix, - 'help' => \$opt_help, - ); -exit(1) if (not $result); - -if ($opt_help) { - print "Usage: filter-subunit [--prefix=PREFIX] [--expected-failures=FILE]... < instream > outstream\n"; - exit(0); -} - -if (defined($opt_expected_failures)) { - @expected_failures = Subunit::Filter::read_test_regexes($opt_expected_failures); -} - -# we want unbuffered output -$| = 1; - -my $statistics = { - TESTS_UNEXPECTED_OK => 0, - TESTS_EXPECTED_OK => 0, - TESTS_UNEXPECTED_FAIL => 0, - TESTS_EXPECTED_FAIL => 0, - TESTS_ERROR => 0, - TESTS_SKIP => 0, -}; - -my $msg_ops = new Subunit::Filter($opt_prefix, \@expected_failures, - $opt_strip_ok_output); - -exit(parse_results($msg_ops, $statistics, *STDIN)); diff --git a/selftest/format-subunit b/selftest/format-subunit index 3747082839..54949df97a 100755 --- a/selftest/format-subunit +++ b/selftest/format-subunit @@ -6,9 +6,14 @@ import optparse import os +import signal import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools")) + import subunithelper +import subunit class PlainFormatter(object): @@ -25,10 +30,19 @@ class PlainFormatter(object): self.summaryfile = summaryfile self.index = 0 self.name = None + self._progress_level = 0 self.totalsuites = totaltests - def testsuite_count(self, count): - self.totalsuites = count + def progress(self, offset, whence): + if whence == subunit.PROGRESS_POP: + self._progress_level -= 1 + elif whence == subunit.PROGRESS_PUSH: + self._progress_level += 1 + elif whence == subunit.PROGRESS_SET: + if self._progress_level == 0: + self.totalsuites = offset + elif whence == subunit.PROGRESS_CUR: + raise NotImplementedError def report_time(self, time): if self.start_time is None: @@ -71,7 +85,7 @@ class PlainFormatter(object): def end_testsuite(self, name, result, reason): out = "" - unexpected = 0 + unexpected = False if not name in self.test_output: print "no output for name[%s]" % name @@ -79,11 +93,13 @@ class PlainFormatter(object): if result in ("success", "xfail"): self.suites_ok+=1 else: - self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason or '')) + self.output_msg("ERROR: Testsuite[%s]\n" % name) + if reason is not None: + self.output_msg("REASON: %s\n" % (reason,)) self.suitesfailed.append(name) if self.immediate and not self.verbose: out += self.test_output[name] - unexpected = 1 + unexpected = True if not self.immediate: if not unexpected: @@ -96,7 +112,7 @@ class PlainFormatter(object): def start_test(self, testname): pass - def end_test(self, testname, result, unexpected, reason): + def end_test(self, testname, result, unexpected, reason=None): if not unexpected: self.test_output[self.name] = "" if not self.immediate: @@ -107,11 +123,9 @@ class PlainFormatter(object): 'success': '.'}.get(result, "?(%s)" % result)) return - if reason is None: - reason = '' - reason = reason.strip() - - self.test_output[self.name] += "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason) + self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname) + if reason is not None: + self.test_output[self.name] += "REASON: %s\n" % (reason.strip(),) if self.immediate and not self.verbose: print self.test_output[self.name] @@ -188,6 +202,10 @@ statistics = { 'TESTS_SKIP': 0, } +def handle_sigint(sig, stack): + sys.exit(0) +signal.signal(signal.SIGINT, handle_sigint) + msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose, opts.immediate, statistics) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index 7bbad62bbf..634db92c92 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -229,6 +229,7 @@ sub run_testsuite($$$$$) my $pcap_file = setup_pcap($name); Subunit::start_testsuite($name); + Subunit::progress_push(); Subunit::report_time(time()); open(RESULTS, "$cmd 2>&1|"); @@ -249,6 +250,7 @@ sub run_testsuite($$$$$) unless (close(RESULTS)) { if ($!) { + Subunit::progress_pop(); Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!"); return 0; } else { @@ -257,6 +259,7 @@ sub run_testsuite($$$$$) } if ($ret & 127) { + Subunit::progress_pop(); Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without")); return 0; } @@ -271,6 +274,7 @@ sub run_testsuite($$$$$) my $exitcode = $ret >> 8; Subunit::report_time(time()); + Subunit::progress_pop(); if ($exitcode == 0) { Subunit::end_testsuite($name, "success"); } else { @@ -684,7 +688,7 @@ foreach my $fn (@testlists) { } } -Subunit::testsuite_count($#available+1); +Subunit::progress($#available+1); Subunit::report_time(time()); foreach (@available) { diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py index 517bbe2c90..8659f984d8 100644 --- a/selftest/subunithelper.py +++ b/selftest/subunithelper.py @@ -18,6 +18,8 @@ __all__ = ['parse_results'] import re +import sys +import subunit import time VALID_RESULTS = ['success', 'successful', 'failure', 'fail', 'skip', 'knownfail', 'error', 'xfail', 'skip-testsuite', 'testsuite-failure', 'testsuite-xfail', 'testsuite-success', 'testsuite-error'] @@ -101,8 +103,16 @@ def parse_results(msg_ops, statistics, fh): msg_ops.end_testsuite(testname, "error", reason) elif l.startswith("testsuite: "): msg_ops.start_testsuite(l.split(":", 1)[1].strip()) - elif l.startswith("testsuite-count: "): - msg_ops.testsuite_count(int(l.split(":", 1)[1].strip())) + elif l.startswith("progress: "): + arg = l.split(":", 1)[1].strip() + if arg == "pop": + msg_ops.progress(None, subunit.PROGRESS_POP) + elif arg == "push": + msg_ops.progress(None, subunit.PROGRESS_PUSH) + elif arg[0] in '+-': + msg_ops.progress(int(arg), subunit.PROGRESS_CUR) + else: + msg_ops.progress(int(arg), subunit.PROGRESS_SET) else: msg_ops.output_msg(l) @@ -144,8 +154,21 @@ class SubunitOps(object): self.end_test(name, "xfail", reason) def report_time(self, t): - (sec, min, hour, mday, mon, year, wday, yday, isdst) = time.localtimet(t) - print "time: %04d-%02d-%02d %02d:%02d:%02d" % (year+1900, mon+1, mday, hour, min, sec) + (year, mon, mday, hour, min, sec, wday, yday, isdst) = time.localtime(t) + print "time: %04d-%02d-%02d %02d:%02d:%02d" % (year, mon, mday, hour, min, sec) + + def progress(self, offset, whence): + if whence == subunit.PROGRESS_CUR and offset > -1: + prefix = "+" + elif whence == subunit.PROGRESS_PUSH: + prefix = "" + offset = "push" + elif whence == subunit.PROGRESS_POP: + prefix = "" + offset = "pop" + else: + prefix = "" + print "progress: %s%s" % (prefix, offset) # The following are Samba extensions: def start_testsuite(self, name): @@ -159,11 +182,136 @@ class SubunitOps(object): def end_testsuite(self, name, result, reason=None): if reason: - print "testsuite-$result: %s [" % name + print "testsuite-%s: %s [" % (result, name) print "%s" % reason print "]" else: - print "testsuite-$result: %s" % name + print "testsuite-%s: %s" % (result, name) + + +def read_test_regexes(name): + f = open(name, 'r') + try: + for l in f: + l = l.strip() + if l == "" or l[0] == "#": + continue + if "#" in l: + (regex, reason) = l.split("#", 1) + yield (regex.strip(), reason.strip()) + else: + yield l, None + finally: + f.close() + + +def find_in_list(regexes, fullname): + for regex, reason in regexes: + if re.match(regex, fullname): + if reason is None: + return "" + return reason + return None + + +class FilterOps(object): + + def control_msg(self, msg): + pass # We regenerate control messages, so ignore this + + def report_time(self, time): + self._ops.report_time(time) + + def progress(self, delta, whence): + self._ops.progress(delta, whence) + + def output_msg(self, msg): + if self.output is None: + sys.stdout.write(msg) + else: + self.output+=msg + + def start_test(self, testname): + if self.prefix is not None: + testname = self.prefix + testname + + if self.strip_ok_output: + self.output = "" + + self._ops.start_test(testname) + + def end_test(self, testname, result, unexpected, reason): + if self.prefix is not None: + testname = self.prefix + testname + + if result in ("fail", "failure") and not unexpected: + result = "xfail" + self.xfail_added+=1 + self.total_xfail+=1 + xfail_reason = find_in_list(self.expected_failures, testname) + if xfail_reason is not None and result in ("fail", "failure"): + result = "xfail" + self.xfail_added+=1 + self.total_xfail+=1 + reason += xfail_reason + + if result in ("fail", "failure"): + self.fail_added+=1 + self.total_fail+=1 + + if result == "error": + self.error_added+=1 + self.total_error+=1 + + if self.strip_ok_output: + if result not in ("success", "xfail", "skip"): + print self.output + self.output = None + + self._ops.end_test(testname, result, reason) + + def skip_testsuite(self, name, reason=None): + self._ops.skip_testsuite(name, reason) + + def start_testsuite(self, name): + self._ops.start_testsuite(name) + + self.error_added = 0 + self.fail_added = 0 + self.xfail_added = 0 + + def end_testsuite(self, name, result, reason=None): + xfail = False + + if self.xfail_added > 0: + xfail = True + if self.fail_added > 0 or self.error_added > 0: + xfail = False + + if xfail and result in ("fail", "failure"): + result = "xfail" + + if self.fail_added > 0 and result != "failure": + result = "failure" + if reason is None: + reason = "Subunit/Filter Reason" + reason += "\n failures[%d]" % self.fail_added + + if self.error_added > 0 and result != "error": + result = "error" + if reason is None: + reason = "Subunit/Filter Reason" + reason += "\n errors[%d]" % self.error_added + + self._ops.end_testsuite(name, result, reason) - def testsuite_count(self, count): - print "testsuite-count: %d" % count + def __init__(self, prefix, expected_failures, strip_ok_output): + self._ops = SubunitOps() + self.output = None + self.prefix = prefix + self.expected_failures = expected_failures + self.strip_ok_output = strip_ok_output + self.xfail_added = 0 + self.total_xfail = 0 + self.total_error = 0 + self.total_fail = 0 |