summaryrefslogtreecommitdiff
path: root/selftest/output
diff options
context:
space:
mode:
Diffstat (limited to 'selftest/output')
-rw-r--r--selftest/output/buildfarm.pm120
-rw-r--r--selftest/output/html.pm354
-rw-r--r--selftest/output/plain.pm212
-rw-r--r--selftest/output/testresults.css129
4 files changed, 815 insertions, 0 deletions
diff --git a/selftest/output/buildfarm.pm b/selftest/output/buildfarm.pm
new file mode 100644
index 0000000000..cee6c1e63a
--- /dev/null
+++ b/selftest/output/buildfarm.pm
@@ -0,0 +1,120 @@
+#!/usr/bin/perl
+
+package output::buildfarm;
+
+use Exporter;
+@ISA = qw(Exporter);
+
+use FindBin qw($RealBin);
+use lib "$RealBin/..";
+
+use Subunit qw(parse_results);
+
+use strict;
+
+sub new($$$) {
+ my ($class) = @_;
+ my $self = {
+ test_output => {},
+ start_time => time()
+ };
+ bless($self, $class);
+}
+
+sub start_testsuite($$)
+{
+ my ($self, $name) = @_;
+ my $out = "";
+
+ $self->{NAME} = $name;
+ $self->{START_TIME} = time();
+
+ my $duration = $self->{START_TIME} - $self->{start_time};
+ $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
+ $out .= "Running test $name (level 0 stdout)\n";
+ $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
+ $out .= scalar(localtime())."\n";
+ $out .= "SELFTEST RUNTIME: " . $duration . "s\n";
+ $out .= "NAME: $name\n";
+
+ $self->{test_output}->{$name} = "";
+
+ print $out;
+}
+
+sub output_msg($$)
+{
+ my ($self, $output) = @_;
+
+ $self->{test_output}->{$self->{NAME}} .= $output;
+}
+
+sub control_msg($$)
+{
+ my ($self, $output) = @_;
+
+ $self->{test_output}->{$self->{NAME}} .= $output;
+}
+
+sub end_testsuite($$$$$$)
+{
+ my ($self, $name, $result, $unexpected, $reason) = @_;
+ my $out = "";
+
+ $out .= "TEST RUNTIME: " . (time() - $self->{START_TIME}) . "s\n";
+
+ if (not $unexpected) {
+ $out .= "ALL OK\n";
+ } else {
+ $out .= "ERROR: $reason\n";
+ $out .= $self->{test_output}->{$name};
+ }
+
+ $out .= "==========================================\n";
+ if (not $unexpected) {
+ $out .= "TEST PASSED: $name\n";
+ } else {
+ $out .= "TEST FAILED: $name (status $reason)\n";
+ }
+ $out .= "==========================================\n";
+
+ print $out;
+}
+
+sub start_test($$$)
+{
+ my ($self, $parents, $testname) = @_;
+
+ if ($#$parents == -1) {
+ $self->start_testsuite($testname);
+ }
+}
+
+sub end_test($$$$$)
+{
+ my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
+
+ if ($unexpected) {
+ $self->{test_output}->{$self->{NAME}} .= "UNEXPECTED($result): $testname\n";
+ }
+
+ if ($#$parents == -1) {
+ $self->end_testsuite($testname, $result, $unexpected, $reason);
+ }
+}
+
+sub summary($)
+{
+ my ($self) = @_;
+
+ print "DURATION: " . (time() - $self->{start_time}) . " seconds\n";
+}
+
+sub skip_testsuite($$$$)
+{
+ my ($self, $name, $reason) = @_;
+
+ print "SKIPPED: $name\n";
+}
+
+1;
diff --git a/selftest/output/html.pm b/selftest/output/html.pm
new file mode 100644
index 0000000000..1049527129
--- /dev/null
+++ b/selftest/output/html.pm
@@ -0,0 +1,354 @@
+#!/usr/bin/perl
+
+package output::html;
+use Exporter;
+@ISA = qw(Exporter);
+
+use strict;
+use warnings;
+
+use FindBin qw($RealBin);
+use lib "$RealBin/..";
+
+use Subunit qw(parse_results);
+
+sub new($$$) {
+ my ($class, $dirname, $statistics) = @_;
+ my $self = {
+ dirname => $dirname,
+ active_test => undef,
+ local_statistics => {},
+ statistics => $statistics,
+ msg => "",
+ error_summary => {
+ skip => [],
+ expected_success => [],
+ unexpected_success => [],
+ expected_failure => [],
+ unexpected_failure => [],
+ skip_testsuites => [],
+ error => []
+ }
+ };
+
+ link("$RealBin/output/testresults.css", "$dirname/testresults.css");
+
+ open(INDEX, ">$dirname/index.html");
+
+ bless($self, $class);
+
+ $self->print_html_header("Samba Testsuite Run", *INDEX);
+
+ print INDEX " <center>";
+ print INDEX " <table>\n";
+ print INDEX " <tr>\n";
+ print INDEX " <td class=\"tableHead\">Test</td>\n";
+ print INDEX " <td class=\"tableHead\">Result</td>\n";
+ print INDEX " </tr>\n";
+
+ return $self;
+}
+
+sub print_html_header($$$)
+{
+ my ($self, $title, $fh) = @_;
+
+ print $fh "<html lang=\"en\">\n";
+ print $fh "<head>\n";
+ print $fh " <title>$title</title>\n";
+ print $fh " <link rel=\"stylesheet\" type=\"text/css\" href=\"testresults.css\"/>\n";
+ print $fh "</head>\n";
+ print $fh "<body>\n";
+ print $fh "<table width=\"100%\" border=\"0\" cellspacing=\"0\">\n";
+ print $fh " <tr><td class=\"title\">$title</td></tr>\n";
+ print $fh " <tr><td>\n";
+}
+
+sub print_html_footer($$)
+{
+ my ($self, $fh) = @_;
+
+ print $fh "</td></tr>\n";
+ print $fh "</table>\n";
+ print $fh "</body>\n";
+ print $fh "</html>\n";
+}
+
+sub output_msg($$);
+
+sub start_testsuite($$)
+{
+ my ($self, $name) = @_;
+
+ $self->{local_statistics} = {
+ success => 0,
+ skip => 0,
+ error => 0,
+ failure => 0
+ };
+
+ $self->{NAME} = $name;
+ $self->{HTMLFILE} = "$name.html";
+ $self->{HTMLFILE} =~ s/[:\t\n \/]/_/g;
+
+ open(TEST, ">$self->{dirname}/$self->{HTMLFILE}") or die("Unable to open $self->{HTMLFILE} for writing");
+
+ $self->print_html_header("Test Results for $name", *TEST);
+
+ print TEST "<h2>Tests</h2>\n";
+
+ print TEST " <table>\n";
+}
+
+sub control_msg($$)
+{
+ my ($self, $output) = @_;
+
+ $self->{msg} .= "<span class=\"control\">$output<br/></span>\n";
+}
+
+sub output_msg($$)
+{
+ my ($self, $output) = @_;
+
+ unless (defined($self->{active_test})) {
+ print TEST "$output<br/>";
+ } else {
+ $self->{msg} .= "$output<br/>";
+ }
+}
+
+sub end_testsuite($$$$)
+{
+ my ($self, $name, $result, $unexpected, $reason) = @_;
+
+ print TEST "</table>\n";
+
+ print TEST "<div class=\"duration\">Duration: " . (time() - $self->{START_TIME}) . "s</div>\n";
+
+ $self->print_html_footer(*TEST);
+
+ close(TEST);
+
+ print INDEX "<tr>\n";
+ print INDEX " <td class=\"testSuite\"><a href=\"$self->{HTMLFILE}\">$name</a></td>\n";
+ my $st = $self->{local_statistics};
+
+ if (not $unexpected) {
+ if ($result eq "failure") {
+ print INDEX " <td class=\"resultExpectedFailure\">";
+ } else {
+ print INDEX " <td class=\"resultOk\">";
+ }
+ } else {
+ print INDEX " <td class=\"resultFailure\">";
+ }
+
+ my $l = 0;
+ if ($st->{success} > 0) {
+ print INDEX "$st->{success} ok";
+ $l++;
+ }
+ if ($st->{skip} > 0) {
+ print INDEX ", " if ($l);
+ print INDEX "$st->{skip} skipped";
+ $l++;
+ }
+ if ($st->{failure} > 0) {
+ print INDEX ", " if ($l);
+ print INDEX "$st->{failure} failures";
+ $l++;
+ }
+ if ($st->{error} > 0) {
+ print INDEX ", " if ($l);
+ print INDEX "$st->{error} errors";
+ $l++;
+ }
+
+ if ($l == 0) {
+ if (not $unexpected) {
+ print INDEX "OK";
+ } else {
+ print INDEX "FAIL";
+ }
+ }
+
+ print INDEX "</td>";
+
+ print INDEX "</tr>\n";
+}
+
+sub start_test($$)
+{
+ my ($self, $parents, $testname) = @_;
+
+ if ($#$parents == -1) {
+ $self->{START_TIME} = time();
+ $self->start_testsuite($testname);
+ return;
+ }
+
+ $self->{active_test} = $testname;
+ $self->{msg} = "";
+}
+
+sub end_test($$$$$$)
+{
+ my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
+
+ if ($#$parents == -1) {
+ $self->end_testsuite($testname, $result, $unexpected, $reason);
+ return;
+ }
+
+ print TEST "<tr>";
+
+ $self->{local_statistics}->{$result}++;
+
+ my $track_class;
+
+ if ($result eq "skip") {
+ print TEST "<td class=\"outputSkipped\">\n";
+ $track_class = "skip";
+ } elsif ($unexpected) {
+ print TEST "<td class=\"outputFailure\">\n";
+ if ($result eq "error") {
+ $track_class = "error";
+ } else {
+ $track_class = "unexpected_$result";
+ }
+ } else {
+ if ($result eq "failure") {
+ print TEST "<td class=\"outputExpectedFailure\">\n";
+ } else {
+ print TEST "<td class=\"outputOk\">\n";
+ }
+ $track_class = "expected_$result";
+ }
+
+ push(@{$self->{error_summary}->{$track_class}}, ,
+ [$self->{HTMLFILE}, $testname, $self->{NAME},
+ $reason]);
+
+ print TEST "<a name=\"$testname\"><h3>$testname</h3></a>\n";
+
+ print TEST $self->{msg};
+
+ if (defined($reason)) {
+ print TEST "<div class=\"reason\">$reason</div>\n";
+ }
+
+ print TEST "</td></tr>\n";
+
+ $self->{active_test} = undef;
+}
+
+sub summary($)
+{
+ my ($self) = @_;
+
+ my $st = $self->{statistics};
+ print INDEX "<tr>\n";
+ print INDEX " <td class=\"testSuiteTotal\">Total</td>\n";
+
+ if ($st->{TESTS_UNEXPECTED_OK} == 0 and
+ $st->{TESTS_UNEXPECTED_FAIL} == 0 and
+ $st->{TESTS_ERROR} == 0) {
+ print INDEX " <td class=\"resultOk\">";
+ } else {
+ print INDEX " <td class=\"resultFailure\">";
+ }
+ print INDEX ($st->{TESTS_EXPECTED_OK} + $st->{TESTS_UNEXPECTED_OK}) . " ok";
+ if ($st->{TESTS_UNEXPECTED_OK} > 0) {
+ print INDEX " ($st->{TESTS_UNEXPECTED_OK} unexpected)";
+ }
+ if ($st->{TESTS_SKIP} > 0) {
+ print INDEX ", $st->{TESTS_SKIP} skipped";
+ }
+ if (($st->{TESTS_UNEXPECTED_FAIL} + $st->{TESTS_EXPECTED_FAIL}) > 0) {
+ print INDEX ", " . ($st->{TESTS_UNEXPECTED_FAIL} + $st->{TESTS_EXPECTED_FAIL}) . " failures";
+ if ($st->{TESTS_UNEXPECTED_FAIL} > 0) {
+ print INDEX " ($st->{TESTS_EXPECTED_FAIL} expected)";
+ }
+ }
+ if ($st->{TESTS_ERROR} > 0) {
+ print INDEX ", $st->{TESTS_ERROR} errors";
+ }
+
+ print INDEX "</td>";
+
+ print INDEX "</tr>\n";
+
+ print INDEX "</table>\n";
+ print INDEX "<a href=\"summary.html\">Summary</a>\n";
+ print INDEX "</center>\n";
+ $self->print_html_footer(*INDEX);
+ close(INDEX);
+
+ my $summ = $self->{error_summary};
+ open(SUMMARY, ">$self->{dirname}/summary.html");
+ $self->print_html_header("Summary", *SUMMARY);
+ sub print_table($$) {
+ my ($title, $list) = @_;
+ return if ($#$list == -1);
+ print SUMMARY "<h3>$title</h3>\n";
+ print SUMMARY "<table>\n";
+ print SUMMARY "<tr>\n";
+ print SUMMARY " <td class=\"tableHead\">Testsuite</td>\n";
+ print SUMMARY " <td class=\"tableHead\">Test</td>\n";
+ print SUMMARY " <td class=\"tableHead\">Reason</td>\n";
+ print SUMMARY "</tr>\n";
+
+ foreach (@$list) {
+ print SUMMARY "<tr>\n";
+ print SUMMARY " <td><a href=\"" . $$_[0] . "\">$$_[2]</a></td>\n";
+ print SUMMARY " <td><a href=\"" . $$_[0] . "#$$_[1]\">$$_[1]</a></td>\n";
+ if (defined($$_[3])) {
+ print SUMMARY " <td>$$_[3]</td>\n";
+ } else {
+ print SUMMARY " <td></td>\n";
+ }
+ print SUMMARY "</tr>\n";
+ }
+
+ print SUMMARY "</table>";
+ }
+ print_table("Errors", $summ->{error});
+ print_table("Unexpected successes", $summ->{unexpected_success});
+ print_table("Unexpected failures", $summ->{unexpected_failure});
+ print_table("Skipped tests", $summ->{skip});
+ print_table("Expected failures", $summ->{expected_failure});
+
+ print SUMMARY "<h3>Skipped testsuites</h3>\n";
+ print SUMMARY "<table>\n";
+ print SUMMARY "<tr>\n";
+ print SUMMARY " <td class=\"tableHead\">Testsuite</td>\n";
+ print SUMMARY " <td class=\"tableHead\">Reason</td>\n";
+ print SUMMARY "</tr>\n";
+
+ foreach (@{$summ->{skip_testsuites}}) {
+ print SUMMARY "<tr>\n";
+ print SUMMARY " <td>$$_[0]</td>\n";
+ if (defined($$_[1])) {
+ print SUMMARY " <td>$$_[1]</td>\n";
+ } else {
+ print SUMMARY " <td></td>\n";
+ }
+ print SUMMARY "</tr>\n";
+ }
+
+ print SUMMARY "</table>";
+
+ $self->print_html_footer(*SUMMARY);
+ close(SUMMARY);
+}
+
+sub skip_testsuite($$$$)
+{
+ my ($self, $name, $reason) = @_;
+
+ push (@{$self->{error_summary}->{skip_testsuites}},
+ [$name, $reason]);
+}
+
+1;
diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm
new file mode 100644
index 0000000000..82a73ab932
--- /dev/null
+++ b/selftest/output/plain.pm
@@ -0,0 +1,212 @@
+#!/usr/bin/perl
+
+package output::plain;
+use Exporter;
+@ISA = qw(Exporter);
+
+use FindBin qw($RealBin);
+use lib "$RealBin/..";
+
+use strict;
+
+sub new($$$$$$$) {
+ my ($class, $summaryfile, $verbose, $immediate, $statistics, $totaltests) = @_;
+ my $self = {
+ verbose => $verbose,
+ immediate => $immediate,
+ statistics => $statistics,
+ start_time => time(),
+ test_output => {},
+ suitesfailed => [],
+ suites_ok => 0,
+ skips => {},
+ summaryfile => $summaryfile,
+ index => 0,
+ totalsuites => $totaltests,
+ };
+ bless($self, $class);
+}
+
+sub output_msg($$);
+
+sub start_testsuite($$)
+{
+ my ($self, $name) = @_;
+
+ $self->{index}++;
+ $self->{NAME} = $name;
+ $self->{START_TIME} = time();
+
+ my $duration = $self->{START_TIME} - $self->{start_time};
+
+ $self->{test_output}->{$name} = "" unless($self->{verbose});
+
+ my $out = "";
+ $out .= "[$self->{index}/$self->{totalsuites} in ".$duration."s";
+ $out .= sprintf(", %d errors", ($#{$self->{suitesfailed}}+1)) if ($#{$self->{suitesfailed}} > -1);
+ $out .= "] $name";
+ if ($self->{immediate}) {
+ print "$out\n";
+ } else {
+ print "$out: ";
+ }
+}
+
+sub output_msg($$)
+{
+ my ($self, $output) = @_;
+
+ if ($self->{verbose}) {
+ require FileHandle;
+ print $output;
+ STDOUT->flush();
+ } else {
+ $self->{test_output}->{$self->{NAME}} .= $output;
+ }
+}
+
+sub control_msg($$)
+{
+ my ($self, $output) = @_;
+
+ $self->output_msg($output);
+}
+
+sub end_testsuite($$$$$)
+{
+ my ($self, $name, $result, $unexpected, $reason) = @_;
+ my $out = "";
+
+ if ($unexpected) {
+ if ($result eq "success" and not defined($reason)) {
+ $reason = "Expected negative exit code, got positive exit code";
+ }
+ $self->output_msg("ERROR: $reason\n");
+ push (@{$self->{suitesfailed}}, $name);
+ } else {
+ $self->{suites_ok}++;
+ }
+
+ if ($unexpected and $self->{immediate} and not $self->{verbose}) {
+ $out .= $self->{test_output}->{$name};
+ }
+
+ if (not $self->{immediate}) {
+ if (not $unexpected) {
+ $out .= " ok\n";
+ } else {
+ $out .= " " . uc($result) . "\n";
+ }
+ }
+
+ print $out;
+}
+
+sub start_test($$$)
+{
+ my ($self, $parents, $testname) = @_;
+
+ if ($#$parents == -1) {
+ $self->start_testsuite($testname);
+ }
+}
+
+sub end_test($$$$$)
+{
+ my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
+
+ if ($#$parents == -1) {
+ $self->end_testsuite($testname, $result, $unexpected, $reason);
+ return;
+ }
+
+ my $append = "";
+
+ unless ($unexpected) {
+ $self->{test_output}->{$self->{NAME}} = "";
+ if (not $self->{immediate}) {
+ if ($result eq "failure") { print "f"; }
+ elsif ($result eq "skip") { print "s"; }
+ elsif ($result eq "success") { print "."; }
+ else { print "?($result)"; }
+ }
+ return;
+ }
+
+ my $fullname = join(".", @$parents).".$testname";
+
+ $append = "UNEXPECTED($result): $testname ($fullname)\n";
+
+ $self->{test_output}->{$self->{NAME}} .= $append;
+
+ if ($self->{immediate} and not $self->{verbose}) {
+ print $self->{test_output}->{$self->{NAME}};
+ $self->{test_output}->{$self->{NAME}} = "";
+ }
+
+ if (not $self->{immediate}) {
+ if ($result eq "error") { print "E"; }
+ elsif ($result eq "failure") { print "F"; }
+ elsif ($result eq "success") { print "S"; }
+ else { print "?"; }
+ }
+}
+
+sub summary($)
+{
+ my ($self) = @_;
+
+ open(SUMMARY, ">$self->{summaryfile}");
+
+ if ($#{$self->{suitesfailed}} > -1) {
+ print SUMMARY "= Failed tests =\n";
+
+ foreach (@{$self->{suitesfailed}}) {
+ print SUMMARY "== $_ ==\n";
+ print SUMMARY $self->{test_output}->{$_}."\n\n";
+ }
+
+ print SUMMARY "\n";
+ }
+
+ if (not $self->{immediate} and not $self->{verbose}) {
+ foreach (@{$self->{suitesfailed}}) {
+ print "===============================================================================\n";
+ print "FAIL: $_\n";
+ print $self->{test_output}->{$_};
+ print "\n";
+ }
+ }
+
+ print SUMMARY "= Skipped tests =\n";
+ foreach my $reason (keys %{$self->{skips}}) {
+ print SUMMARY "$reason\n";
+ foreach my $name (@{$self->{skips}->{$reason}}) {
+ print SUMMARY "\t$name\n";
+ }
+ print SUMMARY "\n";
+ }
+ close(SUMMARY);
+
+ print "\nA summary with detailed information can be found in:\n $self->{summaryfile}\n";
+
+ if ($#{$self->{suitesfailed}} == -1) {
+ my $ok = $self->{statistics}->{TESTS_EXPECTED_OK} +
+ $self->{statistics}->{TESTS_EXPECTED_FAIL};
+ print "\nALL OK ($ok tests in $self->{suites_ok} testsuites)\n";
+ } else {
+ print "\nFAILED ($self->{statistics}->{TESTS_UNEXPECTED_FAIL} failures and $self->{statistics}->{TESTS_ERROR} errors in ". ($#{$self->{suitesfailed}}+1) ." testsuites)\n";
+ }
+
+}
+
+sub skip_testsuite($$)
+{
+ my ($self, $name, $reason) = @_;
+
+ push (@{$self->{skips}->{$reason}}, $name);
+
+ $self->{totalsuites}--;
+}
+
+1;
diff --git a/selftest/output/testresults.css b/selftest/output/testresults.css
new file mode 100644
index 0000000000..66d1d6b2ad
--- /dev/null
+++ b/selftest/output/testresults.css
@@ -0,0 +1,129 @@
+/* Stylesheet for Samba test results.
+ *
+ * Partially based on the CSS file from lcov.
+ */
+
+/* All views: main title format */
+td.title
+{
+ text-align: center;
+ padding-bottom: 10px;
+ font-family: sans-serif;
+ font-size: 20pt;
+ font-style: italic;
+ font-weight: bold;
+}
+
+/* Index table headers */
+td.tableHead
+{
+ text-align: center;
+ color: #FFFFFF;
+ background-color: #6688D4;
+ font-family: sans-serif;
+ font-size: 120%;
+ font-weight: bold;
+}
+
+/* Testsuite names */
+td.testSuite
+{
+ text-align: left;
+ padding-left: 10px;
+ padding-right: 20px;
+ color: #284FA8;
+ background-color: #DAE7FE;
+ font-family: monospace;
+}
+
+/* Successful */
+td.resultOk
+{
+ text-align: right;
+ padding-left: 10px;
+ padding-right: 10px;
+ background-color: #A7FC9D;
+ font-weight: bold;
+}
+
+/* Failure */
+td.resultFailure
+{
+ text-align: right;
+ padding-left: 10px;
+ padding-right: 10px;
+ background-color: #FF0000;
+ font-weight: bold;
+}
+
+/* Expected failure */
+td.resultExpectedFailure
+{
+ text-align: right;
+ padding-left: 10px;
+ padding-right: 10px;
+ background-color: #FFA500;
+ font-weight: bold;
+}
+
+/* Skipped */
+td.resultSkipped
+{
+ text-align: center;
+ padding-left: 10px;
+ padding-right: 10px;
+ background-color: #FFEA20;
+ font-weight: bold;
+}
+
+td.duration
+{
+ text-align: right;
+}
+
+td.durationSkipped
+{
+ text-align: right;
+}
+
+td.outputSkipped
+{
+ background-color: #FFEA20;
+}
+
+td.outputOk
+{
+ background-color: #A7FC9D;
+}
+
+td.outputFailure
+{
+ background-color: #FF0000;
+}
+
+td.outputExpectedFailure
+{
+ background-color: #FFA500;
+}
+
+div.reason
+{
+ text-align: center;
+ font-weight: bold;
+}
+
+span.control
+{
+ display: none;
+}
+
+div.duration
+{
+ text-align: right;
+ font-weight: bold;
+}
+
+div.command
+{
+ background-color: gray;
+}