diff options
Diffstat (limited to 'selftest/output')
| -rw-r--r-- | selftest/output/buildfarm.pm | 120 | ||||
| -rw-r--r-- | selftest/output/html.pm | 354 | ||||
| -rw-r--r-- | selftest/output/plain.pm | 212 | ||||
| -rw-r--r-- | selftest/output/testresults.css | 129 |
4 files changed, 815 insertions, 0 deletions
diff --git a/selftest/output/buildfarm.pm b/selftest/output/buildfarm.pm new file mode 100644 index 0000000000..cee6c1e63a --- /dev/null +++ b/selftest/output/buildfarm.pm @@ -0,0 +1,120 @@ +#!/usr/bin/perl + +package output::buildfarm; + +use Exporter; +@ISA = qw(Exporter); + +use FindBin qw($RealBin); +use lib "$RealBin/.."; + +use Subunit qw(parse_results); + +use strict; + +sub new($$$) { + my ($class) = @_; + my $self = { + test_output => {}, + start_time => time() + }; + bless($self, $class); +} + +sub start_testsuite($$) +{ + my ($self, $name) = @_; + my $out = ""; + + $self->{NAME} = $name; + $self->{START_TIME} = time(); + + my $duration = $self->{START_TIME} - $self->{start_time}; + $out .= "--==--==--==--==--==--==--==--==--==--==--\n"; + $out .= "Running test $name (level 0 stdout)\n"; + $out .= "--==--==--==--==--==--==--==--==--==--==--\n"; + $out .= scalar(localtime())."\n"; + $out .= "SELFTEST RUNTIME: " . $duration . "s\n"; + $out .= "NAME: $name\n"; + + $self->{test_output}->{$name} = ""; + + print $out; +} + +sub output_msg($$) +{ + my ($self, $output) = @_; + + $self->{test_output}->{$self->{NAME}} .= $output; +} + +sub control_msg($$) +{ + my ($self, $output) = @_; + + $self->{test_output}->{$self->{NAME}} .= $output; +} + +sub end_testsuite($$$$$$) +{ + my ($self, $name, $result, $unexpected, $reason) = @_; + my $out = ""; + + $out .= "TEST RUNTIME: " . (time() - $self->{START_TIME}) . "s\n"; + + if (not $unexpected) { + $out .= "ALL OK\n"; + } else { + $out .= "ERROR: $reason\n"; + $out .= $self->{test_output}->{$name}; + } + + $out .= "==========================================\n"; + if (not $unexpected) { + $out .= "TEST PASSED: $name\n"; + } else { + $out .= "TEST FAILED: $name (status $reason)\n"; + } + $out .= "==========================================\n"; + + print $out; +} + +sub start_test($$$) +{ + my ($self, $parents, $testname) = @_; + + if ($#$parents == -1) { + $self->start_testsuite($testname); + } +} + +sub end_test($$$$$) +{ + my ($self, $parents, $testname, $result, $unexpected, $reason) = @_; + + if ($unexpected) { + $self->{test_output}->{$self->{NAME}} .= "UNEXPECTED($result): $testname\n"; + } + + if ($#$parents == -1) { + $self->end_testsuite($testname, $result, $unexpected, $reason); + } +} + +sub summary($) +{ + my ($self) = @_; + + print "DURATION: " . (time() - $self->{start_time}) . " seconds\n"; +} + +sub skip_testsuite($$$$) +{ + my ($self, $name, $reason) = @_; + + print "SKIPPED: $name\n"; +} + +1; diff --git a/selftest/output/html.pm b/selftest/output/html.pm new file mode 100644 index 0000000000..1049527129 --- /dev/null +++ b/selftest/output/html.pm @@ -0,0 +1,354 @@ +#!/usr/bin/perl + +package output::html; +use Exporter; +@ISA = qw(Exporter); + +use strict; +use warnings; + +use FindBin qw($RealBin); +use lib "$RealBin/.."; + +use Subunit qw(parse_results); + +sub new($$$) { + my ($class, $dirname, $statistics) = @_; + my $self = { + dirname => $dirname, + active_test => undef, + local_statistics => {}, + statistics => $statistics, + msg => "", + error_summary => { + skip => [], + expected_success => [], + unexpected_success => [], + expected_failure => [], + unexpected_failure => [], + skip_testsuites => [], + error => [] + } + }; + + link("$RealBin/output/testresults.css", "$dirname/testresults.css"); + + open(INDEX, ">$dirname/index.html"); + + bless($self, $class); + + $self->print_html_header("Samba Testsuite Run", *INDEX); + + print INDEX " <center>"; + print INDEX " <table>\n"; + print INDEX " <tr>\n"; + print INDEX " <td class=\"tableHead\">Test</td>\n"; + print INDEX " <td class=\"tableHead\">Result</td>\n"; + print INDEX " </tr>\n"; + + return $self; +} + +sub print_html_header($$$) +{ + my ($self, $title, $fh) = @_; + + print $fh "<html lang=\"en\">\n"; + print $fh "<head>\n"; + print $fh " <title>$title</title>\n"; + print $fh " <link rel=\"stylesheet\" type=\"text/css\" href=\"testresults.css\"/>\n"; + print $fh "</head>\n"; + print $fh "<body>\n"; + print $fh "<table width=\"100%\" border=\"0\" cellspacing=\"0\">\n"; + print $fh " <tr><td class=\"title\">$title</td></tr>\n"; + print $fh " <tr><td>\n"; +} + +sub print_html_footer($$) +{ + my ($self, $fh) = @_; + + print $fh "</td></tr>\n"; + print $fh "</table>\n"; + print $fh "</body>\n"; + print $fh "</html>\n"; +} + +sub output_msg($$); + +sub start_testsuite($$) +{ + my ($self, $name) = @_; + + $self->{local_statistics} = { + success => 0, + skip => 0, + error => 0, + failure => 0 + }; + + $self->{NAME} = $name; + $self->{HTMLFILE} = "$name.html"; + $self->{HTMLFILE} =~ s/[:\t\n \/]/_/g; + + open(TEST, ">$self->{dirname}/$self->{HTMLFILE}") or die("Unable to open $self->{HTMLFILE} for writing"); + + $self->print_html_header("Test Results for $name", *TEST); + + print TEST "<h2>Tests</h2>\n"; + + print TEST " <table>\n"; +} + +sub control_msg($$) +{ + my ($self, $output) = @_; + + $self->{msg} .= "<span class=\"control\">$output<br/></span>\n"; +} + +sub output_msg($$) +{ + my ($self, $output) = @_; + + unless (defined($self->{active_test})) { + print TEST "$output<br/>"; + } else { + $self->{msg} .= "$output<br/>"; + } +} + +sub end_testsuite($$$$) +{ + my ($self, $name, $result, $unexpected, $reason) = @_; + + print TEST "</table>\n"; + + print TEST "<div class=\"duration\">Duration: " . (time() - $self->{START_TIME}) . "s</div>\n"; + + $self->print_html_footer(*TEST); + + close(TEST); + + print INDEX "<tr>\n"; + print INDEX " <td class=\"testSuite\"><a href=\"$self->{HTMLFILE}\">$name</a></td>\n"; + my $st = $self->{local_statistics}; + + if (not $unexpected) { + if ($result eq "failure") { + print INDEX " <td class=\"resultExpectedFailure\">"; + } else { + print INDEX " <td class=\"resultOk\">"; + } + } else { + print INDEX " <td class=\"resultFailure\">"; + } + + my $l = 0; + if ($st->{success} > 0) { + print INDEX "$st->{success} ok"; + $l++; + } + if ($st->{skip} > 0) { + print INDEX ", " if ($l); + print INDEX "$st->{skip} skipped"; + $l++; + } + if ($st->{failure} > 0) { + print INDEX ", " if ($l); + print INDEX "$st->{failure} failures"; + $l++; + } + if ($st->{error} > 0) { + print INDEX ", " if ($l); + print INDEX "$st->{error} errors"; + $l++; + } + + if ($l == 0) { + if (not $unexpected) { + print INDEX "OK"; + } else { + print INDEX "FAIL"; + } + } + + print INDEX "</td>"; + + print INDEX "</tr>\n"; +} + +sub start_test($$) +{ + my ($self, $parents, $testname) = @_; + + if ($#$parents == -1) { + $self->{START_TIME} = time(); + $self->start_testsuite($testname); + return; + } + + $self->{active_test} = $testname; + $self->{msg} = ""; +} + +sub end_test($$$$$$) +{ + my ($self, $parents, $testname, $result, $unexpected, $reason) = @_; + + if ($#$parents == -1) { + $self->end_testsuite($testname, $result, $unexpected, $reason); + return; + } + + print TEST "<tr>"; + + $self->{local_statistics}->{$result}++; + + my $track_class; + + if ($result eq "skip") { + print TEST "<td class=\"outputSkipped\">\n"; + $track_class = "skip"; + } elsif ($unexpected) { + print TEST "<td class=\"outputFailure\">\n"; + if ($result eq "error") { + $track_class = "error"; + } else { + $track_class = "unexpected_$result"; + } + } else { + if ($result eq "failure") { + print TEST "<td class=\"outputExpectedFailure\">\n"; + } else { + print TEST "<td class=\"outputOk\">\n"; + } + $track_class = "expected_$result"; + } + + push(@{$self->{error_summary}->{$track_class}}, , + [$self->{HTMLFILE}, $testname, $self->{NAME}, + $reason]); + + print TEST "<a name=\"$testname\"><h3>$testname</h3></a>\n"; + + print TEST $self->{msg}; + + if (defined($reason)) { + print TEST "<div class=\"reason\">$reason</div>\n"; + } + + print TEST "</td></tr>\n"; + + $self->{active_test} = undef; +} + +sub summary($) +{ + my ($self) = @_; + + my $st = $self->{statistics}; + print INDEX "<tr>\n"; + print INDEX " <td class=\"testSuiteTotal\">Total</td>\n"; + + if ($st->{TESTS_UNEXPECTED_OK} == 0 and + $st->{TESTS_UNEXPECTED_FAIL} == 0 and + $st->{TESTS_ERROR} == 0) { + print INDEX " <td class=\"resultOk\">"; + } else { + print INDEX " <td class=\"resultFailure\">"; + } + print INDEX ($st->{TESTS_EXPECTED_OK} + $st->{TESTS_UNEXPECTED_OK}) . " ok"; + if ($st->{TESTS_UNEXPECTED_OK} > 0) { + print INDEX " ($st->{TESTS_UNEXPECTED_OK} unexpected)"; + } + if ($st->{TESTS_SKIP} > 0) { + print INDEX ", $st->{TESTS_SKIP} skipped"; + } + if (($st->{TESTS_UNEXPECTED_FAIL} + $st->{TESTS_EXPECTED_FAIL}) > 0) { + print INDEX ", " . ($st->{TESTS_UNEXPECTED_FAIL} + $st->{TESTS_EXPECTED_FAIL}) . " failures"; + if ($st->{TESTS_UNEXPECTED_FAIL} > 0) { + print INDEX " ($st->{TESTS_EXPECTED_FAIL} expected)"; + } + } + if ($st->{TESTS_ERROR} > 0) { + print INDEX ", $st->{TESTS_ERROR} errors"; + } + + print INDEX "</td>"; + + print INDEX "</tr>\n"; + + print INDEX "</table>\n"; + print INDEX "<a href=\"summary.html\">Summary</a>\n"; + print INDEX "</center>\n"; + $self->print_html_footer(*INDEX); + close(INDEX); + + my $summ = $self->{error_summary}; + open(SUMMARY, ">$self->{dirname}/summary.html"); + $self->print_html_header("Summary", *SUMMARY); + sub print_table($$) { + my ($title, $list) = @_; + return if ($#$list == -1); + print SUMMARY "<h3>$title</h3>\n"; + print SUMMARY "<table>\n"; + print SUMMARY "<tr>\n"; + print SUMMARY " <td class=\"tableHead\">Testsuite</td>\n"; + print SUMMARY " <td class=\"tableHead\">Test</td>\n"; + print SUMMARY " <td class=\"tableHead\">Reason</td>\n"; + print SUMMARY "</tr>\n"; + + foreach (@$list) { + print SUMMARY "<tr>\n"; + print SUMMARY " <td><a href=\"" . $$_[0] . "\">$$_[2]</a></td>\n"; + print SUMMARY " <td><a href=\"" . $$_[0] . "#$$_[1]\">$$_[1]</a></td>\n"; + if (defined($$_[3])) { + print SUMMARY " <td>$$_[3]</td>\n"; + } else { + print SUMMARY " <td></td>\n"; + } + print SUMMARY "</tr>\n"; + } + + print SUMMARY "</table>"; + } + print_table("Errors", $summ->{error}); + print_table("Unexpected successes", $summ->{unexpected_success}); + print_table("Unexpected failures", $summ->{unexpected_failure}); + print_table("Skipped tests", $summ->{skip}); + print_table("Expected failures", $summ->{expected_failure}); + + print SUMMARY "<h3>Skipped testsuites</h3>\n"; + print SUMMARY "<table>\n"; + print SUMMARY "<tr>\n"; + print SUMMARY " <td class=\"tableHead\">Testsuite</td>\n"; + print SUMMARY " <td class=\"tableHead\">Reason</td>\n"; + print SUMMARY "</tr>\n"; + + foreach (@{$summ->{skip_testsuites}}) { + print SUMMARY "<tr>\n"; + print SUMMARY " <td>$$_[0]</td>\n"; + if (defined($$_[1])) { + print SUMMARY " <td>$$_[1]</td>\n"; + } else { + print SUMMARY " <td></td>\n"; + } + print SUMMARY "</tr>\n"; + } + + print SUMMARY "</table>"; + + $self->print_html_footer(*SUMMARY); + close(SUMMARY); +} + +sub skip_testsuite($$$$) +{ + my ($self, $name, $reason) = @_; + + push (@{$self->{error_summary}->{skip_testsuites}}, + [$name, $reason]); +} + +1; diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm new file mode 100644 index 0000000000..82a73ab932 --- /dev/null +++ b/selftest/output/plain.pm @@ -0,0 +1,212 @@ +#!/usr/bin/perl + +package output::plain; +use Exporter; +@ISA = qw(Exporter); + +use FindBin qw($RealBin); +use lib "$RealBin/.."; + +use strict; + +sub new($$$$$$$) { + my ($class, $summaryfile, $verbose, $immediate, $statistics, $totaltests) = @_; + my $self = { + verbose => $verbose, + immediate => $immediate, + statistics => $statistics, + start_time => time(), + test_output => {}, + suitesfailed => [], + suites_ok => 0, + skips => {}, + summaryfile => $summaryfile, + index => 0, + totalsuites => $totaltests, + }; + bless($self, $class); +} + +sub output_msg($$); + +sub start_testsuite($$) +{ + my ($self, $name) = @_; + + $self->{index}++; + $self->{NAME} = $name; + $self->{START_TIME} = time(); + + my $duration = $self->{START_TIME} - $self->{start_time}; + + $self->{test_output}->{$name} = "" unless($self->{verbose}); + + my $out = ""; + $out .= "[$self->{index}/$self->{totalsuites} in ".$duration."s"; + $out .= sprintf(", %d errors", ($#{$self->{suitesfailed}}+1)) if ($#{$self->{suitesfailed}} > -1); + $out .= "] $name"; + if ($self->{immediate}) { + print "$out\n"; + } else { + print "$out: "; + } +} + +sub output_msg($$) +{ + my ($self, $output) = @_; + + if ($self->{verbose}) { + require FileHandle; + print $output; + STDOUT->flush(); + } else { + $self->{test_output}->{$self->{NAME}} .= $output; + } +} + +sub control_msg($$) +{ + my ($self, $output) = @_; + + $self->output_msg($output); +} + +sub end_testsuite($$$$$) +{ + my ($self, $name, $result, $unexpected, $reason) = @_; + my $out = ""; + + if ($unexpected) { + if ($result eq "success" and not defined($reason)) { + $reason = "Expected negative exit code, got positive exit code"; + } + $self->output_msg("ERROR: $reason\n"); + push (@{$self->{suitesfailed}}, $name); + } else { + $self->{suites_ok}++; + } + + if ($unexpected and $self->{immediate} and not $self->{verbose}) { + $out .= $self->{test_output}->{$name}; + } + + if (not $self->{immediate}) { + if (not $unexpected) { + $out .= " ok\n"; + } else { + $out .= " " . uc($result) . "\n"; + } + } + + print $out; +} + +sub start_test($$$) +{ + my ($self, $parents, $testname) = @_; + + if ($#$parents == -1) { + $self->start_testsuite($testname); + } +} + +sub end_test($$$$$) +{ + my ($self, $parents, $testname, $result, $unexpected, $reason) = @_; + + if ($#$parents == -1) { + $self->end_testsuite($testname, $result, $unexpected, $reason); + return; + } + + my $append = ""; + + unless ($unexpected) { + $self->{test_output}->{$self->{NAME}} = ""; + if (not $self->{immediate}) { + if ($result eq "failure") { print "f"; } + elsif ($result eq "skip") { print "s"; } + elsif ($result eq "success") { print "."; } + else { print "?($result)"; } + } + return; + } + + my $fullname = join(".", @$parents).".$testname"; + + $append = "UNEXPECTED($result): $testname ($fullname)\n"; + + $self->{test_output}->{$self->{NAME}} .= $append; + + if ($self->{immediate} and not $self->{verbose}) { + print $self->{test_output}->{$self->{NAME}}; + $self->{test_output}->{$self->{NAME}} = ""; + } + + if (not $self->{immediate}) { + if ($result eq "error") { print "E"; } + elsif ($result eq "failure") { print "F"; } + elsif ($result eq "success") { print "S"; } + else { print "?"; } + } +} + +sub summary($) +{ + my ($self) = @_; + + open(SUMMARY, ">$self->{summaryfile}"); + + if ($#{$self->{suitesfailed}} > -1) { + print SUMMARY "= Failed tests =\n"; + + foreach (@{$self->{suitesfailed}}) { + print SUMMARY "== $_ ==\n"; + print SUMMARY $self->{test_output}->{$_}."\n\n"; + } + + print SUMMARY "\n"; + } + + if (not $self->{immediate} and not $self->{verbose}) { + foreach (@{$self->{suitesfailed}}) { + print "===============================================================================\n"; + print "FAIL: $_\n"; + print $self->{test_output}->{$_}; + print "\n"; + } + } + + print SUMMARY "= Skipped tests =\n"; + foreach my $reason (keys %{$self->{skips}}) { + print SUMMARY "$reason\n"; + foreach my $name (@{$self->{skips}->{$reason}}) { + print SUMMARY "\t$name\n"; + } + print SUMMARY "\n"; + } + close(SUMMARY); + + print "\nA summary with detailed information can be found in:\n $self->{summaryfile}\n"; + + if ($#{$self->{suitesfailed}} == -1) { + my $ok = $self->{statistics}->{TESTS_EXPECTED_OK} + + $self->{statistics}->{TESTS_EXPECTED_FAIL}; + print "\nALL OK ($ok tests in $self->{suites_ok} testsuites)\n"; + } else { + print "\nFAILED ($self->{statistics}->{TESTS_UNEXPECTED_FAIL} failures and $self->{statistics}->{TESTS_ERROR} errors in ". ($#{$self->{suitesfailed}}+1) ." testsuites)\n"; + } + +} + +sub skip_testsuite($$) +{ + my ($self, $name, $reason) = @_; + + push (@{$self->{skips}->{$reason}}, $name); + + $self->{totalsuites}--; +} + +1; diff --git a/selftest/output/testresults.css b/selftest/output/testresults.css new file mode 100644 index 0000000000..66d1d6b2ad --- /dev/null +++ b/selftest/output/testresults.css @@ -0,0 +1,129 @@ +/* Stylesheet for Samba test results. + * + * Partially based on the CSS file from lcov. + */ + +/* All views: main title format */ +td.title +{ + text-align: center; + padding-bottom: 10px; + font-family: sans-serif; + font-size: 20pt; + font-style: italic; + font-weight: bold; +} + +/* Index table headers */ +td.tableHead +{ + text-align: center; + color: #FFFFFF; + background-color: #6688D4; + font-family: sans-serif; + font-size: 120%; + font-weight: bold; +} + +/* Testsuite names */ +td.testSuite +{ + text-align: left; + padding-left: 10px; + padding-right: 20px; + color: #284FA8; + background-color: #DAE7FE; + font-family: monospace; +} + +/* Successful */ +td.resultOk +{ + text-align: right; + padding-left: 10px; + padding-right: 10px; + background-color: #A7FC9D; + font-weight: bold; +} + +/* Failure */ +td.resultFailure +{ + text-align: right; + padding-left: 10px; + padding-right: 10px; + background-color: #FF0000; + font-weight: bold; +} + +/* Expected failure */ +td.resultExpectedFailure +{ + text-align: right; + padding-left: 10px; + padding-right: 10px; + background-color: #FFA500; + font-weight: bold; +} + +/* Skipped */ +td.resultSkipped +{ + text-align: center; + padding-left: 10px; + padding-right: 10px; + background-color: #FFEA20; + font-weight: bold; +} + +td.duration +{ + text-align: right; +} + +td.durationSkipped +{ + text-align: right; +} + +td.outputSkipped +{ + background-color: #FFEA20; +} + +td.outputOk +{ + background-color: #A7FC9D; +} + +td.outputFailure +{ + background-color: #FF0000; +} + +td.outputExpectedFailure +{ + background-color: #FFA500; +} + +div.reason +{ + text-align: center; + font-weight: bold; +} + +span.control +{ + display: none; +} + +div.duration +{ + text-align: right; + font-weight: bold; +} + +div.command +{ + background-color: gray; +} |