diff options
| -rwxr-xr-x | selftest/format-subunit | 176 | ||||
| -rw-r--r-- | selftest/output/plain.pm | 246 | ||||
| -rw-r--r-- | selftest/subunithelper.py | 21 | ||||
| -rw-r--r-- | source3/Makefile.in | 2 | ||||
| -rw-r--r-- | source4/selftest/config.mk | 2 |
5 files changed, 179 insertions, 268 deletions
diff --git a/selftest/format-subunit b/selftest/format-subunit index 1967fb4f85..8581dec6cb 100755 --- a/selftest/format-subunit +++ b/selftest/format-subunit @@ -1,4 +1,5 @@ #!/usr/bin/env python +# vim: expandtab # Pretty-format subunit output # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org> # Published under the GNU GPL, v3 or later @@ -9,24 +10,179 @@ import sys import subunithelper +class PlainFormatter(object): + + def __init__(self, summaryfile, verbose, immediate, statistics, totaltests=None): + self.verbose = verbose + self.immediate = immediate + self.statistics = statistics + self.start_time = None + self.test_output = {} + self.suitesfailed = [] + self.suites_ok = 0 + self.skips = {} + self.summaryfile = summaryfile + self.index = 0 + self.NAME = None + self.totalsuites = totaltests + + def testsuite_count(self, count): + self.totalsuites = count + + def report_time(self, time): + if self.start_time is None: + self.start_time = time + self.last_time = time + + def start_testsuite(self, name): + self.index += 1 + self.NAME = name + self.START_TIME = self.last_time + + duration = self.START_TIME - self.start_time + + if not self.verbose: + self.test_output[name] = "" + + out = "" + out += "[%d" % self.index + if self.totalsuites is not None: + out += "/%d" % self.totalsuites + out += " in %ds" % duration + if self.suitesfailed: + out += ", %d errors" % (len(self.suitesfailed),) + out += "] %s" % name + if self.immediate: + sys.stdout.write(out + "\n") + else: + sys.stdout.write(out + ": ") + + def output_msg(self, output): + if self.verbose: + sys.stdout.write(output) + elif self.NAME is not None: + self.test_output[self.NAME] += output + else: + sys.stdout.write(output) + + def control_msg(self, output): + #$self->output_msg($output) + pass + + def end_testsuite(self, name, result, reason): + out = "" + unexpected = 0 + + if not name in self.test_output: + print "no output for name[%s]" % name + + if result in ("success", "xfail"): + self.suites_ok+=1 + else: + self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason)) + self.suitesfailed.append(name) + if self.immediate and not self.verbose: + out += self.test_output[name] + unexpected = 1 + + if not self.immediate: + if not unexpected: + out += " ok\n" + else: + out += " " + result.upper() + "\n" + + sys.stdout.write(out) + + def start_test(self, testname): + pass + + def end_test(self, testname, result, unexpected, reason): + append = "" + + if not unexpected: + self.test_output[self.NAME] = "" + if not self.immediate: + sys.stdout.write({ + 'failure': 'f', + 'xfail': 'X', + 'skip': 's', + 'success': '.'}.get(result, "?(%s)" % result)) + return + + reason = reason.strip() + + append = "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason) + + self.test_output[self.NAME] += append + + if self.immediate and not self.verbose: + print self.test_output[self.NAME] + self.test_output[self.NAME] = "" + + if not self.immediate: + sys.stdout.write({ + 'error': 'E', + 'failure': 'F', + 'success': 'S'}.get(result, "?")) + + def summary(self): + f = open(self.summaryfile, 'w+') + + if self.suitesfailed: + f.write("= Failed tests =\n") + + for suite in self.suitesfailed: + f.write("== %s ==\n" % suite) + f.write(self.test_output[suite]+"\n\n") + + f.write("\n") + + if not self.immediate and not self.verbose: + for suite in self.suitesfailed: + print "===============================================================================" + print "FAIL: %s" % suite + print self.test_output[suite] + print "" + + f.write("= Skipped tests =\n") + for reason in self.skips.keys(): + f.write(reason + "\n") + for name in self.skips[reason]: + f.write("\t%s\n" % name) + f.write("\n") + f.close() + + print "\nA summary with detailed information can be found in:\n %s\n" % self.summaryfile + + if not self.suitesfailed: + ok = self.statistics['TESTS_EXPECTED_OK'] + self.statistics['TESTS_EXPECTED_FAIL'] + print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok) + else: + print "\nFAILED (%d failures and %d errors in %d testsuites)" % (self.statistics['TESTS_UNEXPECTED_FAIL'], self.statistics['TESTS_ERROR'], len(self.suitesfailed)) + + def skip_testsuite(self, name, reason="UNKNOWN"): + self.skips.setdefault(reason, []).append(name) + if self.totalsuites: + self.totalsuites-=1 + parser = optparse.OptionParser("format-subunit [options]") parser.add_option("--verbose", action="store_true", - help="Be verbose") + help="Be verbose") parser.add_option("--immediate", action="store_true", - help="Show failures immediately, don't wait until test run has finished") + help="Show failures immediately, don't wait until test run has finished") parser.add_option("--prefix", type="string", default=".", - help="Prefix to write summary to") + help="Prefix to write summary to") opts, args = parser.parse_args() statistics = { - 'SUITES_FAIL': 0, - 'TESTS_UNEXPECTED_OK': 0, - 'TESTS_EXPECTED_OK': 0, - 'TESTS_UNEXPECTED_FAIL': 0, - 'TESTS_EXPECTED_FAIL': 0, - 'TESTS_ERROR': 0, - 'TESTS_SKIP': 0, + 'SUITES_FAIL': 0, + 'TESTS_UNEXPECTED_OK': 0, + 'TESTS_EXPECTED_OK': 0, + 'TESTS_UNEXPECTED_FAIL': 0, + 'TESTS_EXPECTED_FAIL': 0, + 'TESTS_ERROR': 0, + 'TESTS_SKIP': 0, } msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose, opts.immediate, statistics) diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm deleted file mode 100644 index eae1e7aaac..0000000000 --- a/selftest/output/plain.pm +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/perl -# Plain text output for selftest -# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -package output::plain; -use Exporter; -@ISA = qw(Exporter); - -use FindBin qw($RealBin); -use lib "$RealBin/.."; - -use strict; - -sub new($$$$$$$) { - my ($class, $summaryfile, $verbose, $immediate, $statistics, $totaltests) = @_; - my $self = { - verbose => $verbose, - immediate => $immediate, - statistics => $statistics, - start_time => undef, - test_output => {}, - suitesfailed => [], - suites_ok => 0, - skips => {}, - summaryfile => $summaryfile, - index => 0, - totalsuites => $totaltests, - }; - bless($self, $class); -} - -sub testsuite_count($$) -{ - my ($self, $count) = @_; - $self->{totalsuites} = $count; -} - -sub report_time($$) -{ - my ($self, $time) = @_; - unless ($self->{start_time}) { - $self->{start_time} = $time; - } - $self->{last_time} = $time; -} - -sub output_msg($$); - -sub start_testsuite($$) -{ - my ($self, $name) = @_; - - $self->{index}++; - $self->{NAME} = $name; - $self->{START_TIME} = $self->{last_time}; - - my $duration = $self->{START_TIME} - $self->{start_time}; - - $self->{test_output}->{$name} = "" unless($self->{verbose}); - - my $out = ""; - $out .= "[$self->{index}"; - if ($self->{totalsuites}) { - $out .= "/$self->{totalsuites}"; - } - $out.= " in ".$duration."s"; - $out .= sprintf(", %d errors", ($#{$self->{suitesfailed}}+1)) if ($#{$self->{suitesfailed}} > -1); - $out .= "] $name"; - if ($self->{immediate}) { - print "$out\n"; - } else { - print "$out: "; - } -} - -sub output_msg($$) -{ - my ($self, $output) = @_; - - if ($self->{verbose}) { - require FileHandle; - print $output; - STDOUT->flush(); - } elsif (defined($self->{NAME})) { - $self->{test_output}->{$self->{NAME}} .= $output; - } else { - print $output; - } -} - -sub control_msg($$) -{ - my ($self, $output) = @_; - - #$self->output_msg($output); -} - -sub end_testsuite($$$$) -{ - my ($self, $name, $result, $reason) = @_; - my $out = ""; - my $unexpected = 0; - - if (not defined($self->{test_output}->{$name})) { - print "no output for name[$name]\n"; - } - - if ($result eq "success" or $result eq "xfail") { - $self->{suites_ok}++; - } else { - $self->output_msg("ERROR: Testsuite[$name]\nREASON: $reason\n"); - push (@{$self->{suitesfailed}}, $name); - if ($self->{immediate} and not $self->{verbose}) { - $out .= $self->{test_output}->{$name}; - } - $unexpected = 1; - } - - if (not $self->{immediate}) { - unless($unexpected) { - $out .= " ok\n"; - } else { - $out .= " " . uc($result) . "\n"; - } - } - - print $out; -} - -sub start_test($$$) -{ - my ($self, $testname) = @_; -} - -sub end_test($$$$$) -{ - my ($self, $testname, $result, $unexpected, $reason) = @_; - - my $append = ""; - - unless ($unexpected) { - $self->{test_output}->{$self->{NAME}} = ""; - if (not $self->{immediate}) { - if ($result eq "failure") { print "f"; } - elsif ($result eq "xfail") { print "X"; } - elsif ($result eq "skip") { print "s"; } - elsif ($result eq "success") { print "."; } - else { print "?($result)"; } - } - return; - } - - chomp $reason; - - $append = "UNEXPECTED($result): $testname\nREASON: $reason\n"; - - $self->{test_output}->{$self->{NAME}} .= $append; - - if ($self->{immediate} and not $self->{verbose}) { - print $self->{test_output}->{$self->{NAME}}; - $self->{test_output}->{$self->{NAME}} = ""; - } - - if (not $self->{immediate}) { - if ($result eq "error") { print "E"; } - elsif ($result eq "failure") { print "F"; } - elsif ($result eq "success") { print "S"; } - else { print "?"; } - } -} - -sub summary($) -{ - my ($self) = @_; - - open(SUMMARY, ">$self->{summaryfile}"); - - if ($#{$self->{suitesfailed}} > -1) { - print SUMMARY "= Failed tests =\n"; - - foreach (@{$self->{suitesfailed}}) { - print SUMMARY "== $_ ==\n"; - print SUMMARY $self->{test_output}->{$_}."\n\n"; - } - - print SUMMARY "\n"; - } - - if (not $self->{immediate} and not $self->{verbose}) { - foreach (@{$self->{suitesfailed}}) { - print "===============================================================================\n"; - print "FAIL: $_\n"; - print $self->{test_output}->{$_}; - print "\n"; - } - } - - print SUMMARY "= Skipped tests =\n"; - foreach my $reason (keys %{$self->{skips}}) { - print SUMMARY "$reason\n"; - foreach my $name (@{$self->{skips}->{$reason}}) { - print SUMMARY "\t$name\n"; - } - print SUMMARY "\n"; - } - close(SUMMARY); - - print "\nA summary with detailed information can be found in:\n $self->{summaryfile}\n"; - - if ($#{$self->{suitesfailed}} == -1) { - my $ok = $self->{statistics}->{TESTS_EXPECTED_OK} + - $self->{statistics}->{TESTS_EXPECTED_FAIL}; - print "\nALL OK ($ok tests in $self->{suites_ok} testsuites)\n"; - } else { - print "\nFAILED ($self->{statistics}->{TESTS_UNEXPECTED_FAIL} failures and $self->{statistics}->{TESTS_ERROR} errors in ". ($#{$self->{suitesfailed}}+1) ." testsuites)\n"; - } - -} - -sub skip_testsuite($$$) -{ - my ($self, $name, $reason) = @_; - - unless (defined($reason)) { - $reason = "UNKNOWN"; - } - push (@{$self->{skips}->{$reason}}, $name); - - if ($self->{totalsuites}) { - $self->{totalsuites}--; - } -} - -1; diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py index 3cd0f013d0..770b14befa 100644 --- a/selftest/subunithelper.py +++ b/selftest/subunithelper.py @@ -28,24 +28,29 @@ def parse_results(msg_ops, statistics, fh): while fh: l = fh.readline() + if l == "": + break if l.startswith("test: "): msg_ops.control_msg(l) name = l.split(":", 1)[1].strip() msg_ops.start_test(name) open_tests.append(name) elif l.startswith("time: "): - (year, month, day, hour, minute, second) = re.match( - "^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)\n/", l) - msg_ops.report_time(time.mktime(second, minute, hour, day, month-1, year-1900)) + grp = re.match( + "^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)\n", l) + msg_ops.report_time(time.mktime((int(grp.group(1)), int(grp.group(2)), int(grp.group(3)), int(grp.group(4)), int(grp.group(5)), int(grp.group(6)), 0, 0, 0))) elif re.match("^(" + "|".join(VALID_RESULTS) + "): (.*?)( \[)?([ \t]*)( multipart)?\n", l): msg_ops.control_msg(l) - (result, testname, hasreason) = re.match("^(" + "|".join(VALID_RESULTS) + "): (.*?)( \[)?([ \t]*)( multipart)?\n", l) + grp = re.match("^(" + "|".join(VALID_RESULTS) + "): (.*?)( \[)?([ \t]*)( multipart)?\n", l) + (result, testname, hasreason) = (grp.group(1), grp.group(2), grp.group(3)) if hasreason: reason = "" # reason may be specified in next lines terminated = False while fh: l = fh.readline() + if l == "": + break msg_ops.control_msg(l) if l == "]\n": terminated = True @@ -58,6 +63,8 @@ def parse_results(msg_ops, statistics, fh): msg_ops.end_test(testname, "error", 1, "reason (%s) interrupted" % result) return 1 + else: + reason = None if result in ("success", "successful"): open_tests.pop() #FIXME: Check that popped value == $testname statistics['TESTS_EXPECTED_OK']+=1 @@ -104,12 +111,6 @@ def parse_results(msg_ops, statistics, fh): "was started but never finished!") statistics['TESTS_ERROR']+=1 - # if the Filter module is in use, it will have the right counts - if 'total_error' in msg_ops: - statistics['TESTS_ERROR'] = msg_ops['total_error'] - statistics['TESTS_UNEXPECTED_FAIL'] = msg_ops['total_fail'] - statistics['TESTS_EXPECTED_FAIL'] = msg_ops['total_xfail'] - if statistics['TESTS_ERROR'] > 0: return 1 if statistics['TESTS_UNEXPECTED_FAIL'] > 0: diff --git a/source3/Makefile.in b/source3/Makefile.in index 9913806fe7..97b3275e46 100644 --- a/source3/Makefile.in +++ b/source3/Makefile.in @@ -3253,7 +3253,7 @@ selftest:: all torture timelimit --socket-wrapper $(TESTS) | \ $(PERL) $(selftestdir)/filter-subunit.pl \ --expected-failures=$(srcdir)/selftest/knownfail | \ - $(PERL) $(selftestdir)/format-subunit --immediate + $(PYTHON) $(selftestdir)/format-subunit --immediate selftest-%: $(MAKE) selftest TESTS=$* diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk index 4359585e9e..6057de68f9 100644 --- a/source4/selftest/config.mk +++ b/source4/selftest/config.mk @@ -14,7 +14,7 @@ ST_DONE_TEST = @test -f $(selftest_prefix)/st_done || { echo "SELFTEST FAILED"; SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(srcdir)/selftest/knownfail -SUBUNIT_FORMATTER ?= $(PERL) $(selftestdir)/format-subunit --prefix=${selftest_prefix} --immediate +SUBUNIT_FORMATTER ?= $(PYTHON) $(selftestdir)/format-subunit --prefix=${selftest_prefix} --immediate FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(SUBUNIT_FORMATTER) test-subunit:: everything |