summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJelmer Vernooij <jelmer@samba.org>2009-06-03 18:19:01 +0200
committerJelmer Vernooij <jelmer@samba.org>2009-06-11 19:59:58 +0200
commit24f01e70aaefb390d081bcd8cec6cd42282430b4 (patch)
tree66280c4c8b9f059b8ae612efe2537db23cd8a783
parent4e276fb60a61f559e1d343261fc15c794c921dd7 (diff)
downloadsamba-24f01e70aaefb390d081bcd8cec6cd42282430b4.tar.gz
samba-24f01e70aaefb390d081bcd8cec6cd42282430b4.tar.bz2
samba-24f01e70aaefb390d081bcd8cec6cd42282430b4.zip
selftest: Use filter-xfail for known failures
-rw-r--r--selftest/Subunit.pm35
-rwxr-xr-xselftest/filter-xfail.pl2
-rwxr-xr-xselftest/format-subunit.pl2
-rwxr-xr-xselftest/selftest.pl31
-rw-r--r--source4/selftest/config.mk7
5 files changed, 21 insertions, 56 deletions
diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm
index f7284d3f55..0eafc44146 100644
--- a/selftest/Subunit.pm
+++ b/selftest/Subunit.pm
@@ -23,9 +23,9 @@ require Exporter;
use strict;
-sub parse_results($$$$$)
+sub parse_results($$$$)
{
- my ($msg_ops, $statistics, $fh, $expecting_failure, $open_tests) = @_;
+ my ($msg_ops, $statistics, $fh, $open_tests) = @_;
my $unexpected_ok = 0;
my $expected_fail = 0;
my $unexpected_fail = 0;
@@ -39,7 +39,7 @@ sub parse_results($$$$$)
push (@$open_tests, $1);
} elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) {
$msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1));
- } elsif (/^(success|successful|failure|fail|skip|knownfail|error): (.*?)( \[)?([ \t]*)\n/) {
+ } elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) {
$msg_ops->control_msg($_);
my $reason = undef;
if ($3) {
@@ -60,29 +60,18 @@ sub parse_results($$$$$)
my $result = $1;
if ($1 eq "success" or $1 eq "successful") {
pop(@$open_tests); #FIXME: Check that popped value == $2
- if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
- $statistics->{TESTS_UNEXPECTED_OK}++;
- $msg_ops->end_test($open_tests, $2, $1, 1, $reason);
- $unexpected_ok++;
- } else {
- $statistics->{TESTS_EXPECTED_OK}++;
- $msg_ops->end_test($open_tests, $2, $1, 0, $reason);
- }
- } elsif ($1 eq "failure" or $1 eq "fail") {
- pop(@$open_tests); #FIXME: Check that popped value == $2
- if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
- $statistics->{TESTS_EXPECTED_FAIL}++;
- $msg_ops->end_test($open_tests, $2, $1, 0, $reason);
- $expected_fail++;
- } else {
- $statistics->{TESTS_UNEXPECTED_FAIL}++;
- $msg_ops->end_test($open_tests, $2, $1, 1, $reason);
- $unexpected_fail++;
- }
- } elsif ($1 eq "knownfail") {
+ $statistics->{TESTS_EXPECTED_OK}++;
+ $msg_ops->end_test($open_tests, $2, $1, 0, $reason);
+ } elsif ($1 eq "xfail" or $1 eq "knownfail") {
pop(@$open_tests); #FIXME: Check that popped value == $2
$statistics->{TESTS_EXPECTED_FAIL}++;
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
+ $expected_fail++;
+ } elsif ($1 eq "failure" or $1 eq "fail") {
+ pop(@$open_tests); #FIXME: Check that popped value == $2
+ $statistics->{TESTS_UNEXPECTED_FAIL}++;
+ $msg_ops->end_test($open_tests, $2, $1, 1, $reason);
+ $unexpected_fail++;
} elsif ($1 eq "skip") {
$statistics->{TESTS_SKIP}++;
pop(@$open_tests); #FIXME: Check that popped value == $2
diff --git a/selftest/filter-xfail.pl b/selftest/filter-xfail.pl
index 0e0c4e22bc..edf0e050a7 100755
--- a/selftest/filter-xfail.pl
+++ b/selftest/filter-xfail.pl
@@ -157,6 +157,6 @@ sub end_test($$$$$)
my $msg_ops = {};
bless $msg_ops;
-parse_results($msg_ops, $statistics, *STDIN, sub { return 0; }, []);
+parse_results($msg_ops, $statistics, *STDIN, []);
0;
diff --git a/selftest/format-subunit.pl b/selftest/format-subunit.pl
index 000346a9af..c60902f8d7 100755
--- a/selftest/format-subunit.pl
+++ b/selftest/format-subunit.pl
@@ -61,7 +61,7 @@ if ($opt_format eq "buildfarm") {
}
my $expected_ret = parse_results(
- $msg_ops, $statistics, *STDIN, sub { return 0; }, []);
+ $msg_ops, $statistics, *STDIN, []);
$msg_ops->summary();
diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index b786c9a50f..8a2c8804bc 100755
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -77,23 +77,10 @@ when the server is running locally.
Will prevent TCP and UDP ports being opened on the local host but
(transparently) redirects these calls to use unix domain sockets.
-=item I<--expected-failures>
-
-Specify a file containing a list of tests that are expected to fail. Failures for
-these tests will be counted as successes, successes will be counted as failures.
-
-The format for the file is, one entry per line:
-
-TESTSUITE-NAME.TEST-NAME
-
-The reason for a test can also be specified, by adding a hash sign (#) and the reason
-after the test name.
-
=item I<--exclude>
Specify a file containing a list of tests that should be skipped. Possible
-candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
-for the --expected-failures flag.
+candidates are tests that segfault the server, flip or don't end.
=item I<--include>
@@ -157,7 +144,6 @@ my $opt_socket_wrapper_pcap = undef;
my $opt_socket_wrapper_keep_pcap = undef;
my $opt_one = 0;
my $opt_immediate = 0;
-my $opt_expected_failures = undef;
my @opt_exclude = ();
my @opt_include = ();
my $opt_verbose = 0;
@@ -176,7 +162,6 @@ my $builddir = ".";
my $exeext = "";
my $prefix = "./st";
-my @expected_failures = ();
my @includes = ();
my @excludes = ();
@@ -205,12 +190,6 @@ sub find_in_list($$)
return undef;
}
-sub expecting_failure($)
-{
- my ($name) = @_;
- return find_in_list(\@expected_failures, $name);
-}
-
sub skip($)
{
my ($name) = @_;
@@ -265,7 +244,7 @@ sub run_testsuite($$$$$$)
}
my $expected_ret = parse_results(
- $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
+ $msg_ops, $statistics, *RESULT, [$name]);
my $envlog = getlog_env($envname);
$msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
@@ -323,7 +302,6 @@ Target Specific:
failed
--socket-wrapper enable socket wrapper
--bindir=PATH path to target binaries
- --expected-failures=FILE specify list of tests that is guaranteed to fail
Samba4 Specific:
--ldap=openldap|fedora-ds back samba onto specified ldap server
@@ -351,7 +329,6 @@ my $result = GetOptions (
'quick' => \$opt_quick,
'one' => \$opt_one,
'immediate' => \$opt_immediate,
- 'expected-failures=s' => \$opt_expected_failures,
'exclude=s' => \@opt_exclude,
'include=s' => \@opt_include,
'srcdir=s' => \$srcdir,
@@ -537,10 +514,6 @@ sub read_test_regexes($)
return @ret;
}
-if (defined($opt_expected_failures)) {
- @expected_failures = read_test_regexes($opt_expected_failures);
-}
-
foreach (@opt_exclude) {
push (@excludes, read_test_regexes($_));
}
diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 033fd7938d..0b1f340f32 100644
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -3,14 +3,17 @@ TEST_FORMAT = plain
SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
$(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
--builddir=$(builddir) --srcdir=$(srcdir) \
- --expected-failures=$(srcdir)/selftest/knownfail \
--format=subunit \
--exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
$(TEST_OPTIONS)
SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
-FORMAT_TEST_OUTPUT = $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
+FILTER_XFAIL = $(PERL) $(selftestdir)/filter-xfail.pl --expected-failures=$(srcdir)/selftest/knownfail
+FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
+
+subunittest:: everything
+ $(SELFTEST) $(TESTS)
slowtest:: everything
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate