selftest: Use filter-xfail for known failures

5 files changed, 21 insertions, 56 deletions

diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm
index f7284d3f55..0eafc44146 100644
--- a/selftest/Subunit.pm
+++ b/selftest/Subunit.pm
@@ -23,9 +23,9 @@ require Exporter;
 
 use strict;
 
-sub parse_results($$$$$)
+sub parse_results($$$$)
 {
-	my ($msg_ops, $statistics, $fh, $expecting_failure, $open_tests) = @_;
+	my ($msg_ops, $statistics, $fh, $open_tests) = @_;
 	my $unexpected_ok = 0;
 	my $expected_fail = 0;
 	my $unexpected_fail = 0;
@@ -39,7 +39,7 @@ sub parse_results($$$$$)
 			push (@$open_tests, $1);
 		} elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) {
 			$msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1));
-		} elsif (/^(success|successful|failure|fail|skip|knownfail|error): (.*?)( \[)?([ \t]*)\n/) {
+		} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) {
 			$msg_ops->control_msg($_);
 			my $reason = undef;
 			if ($3) {
@@ -60,29 +60,18 @@ sub parse_results($$$$$)
 			my $result = $1;
 			if ($1 eq "success" or $1 eq "successful") {
 				pop(@$open_tests); #FIXME: Check that popped value == $2
-				if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
-					$statistics->{TESTS_UNEXPECTED_OK}++;
-					$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
-					$unexpected_ok++;
-				} else {
-					$statistics->{TESTS_EXPECTED_OK}++;
-					$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-				}
-			} elsif ($1 eq "failure" or $1 eq "fail") {
-				pop(@$open_tests); #FIXME: Check that popped value == $2
-				if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
-					$statistics->{TESTS_EXPECTED_FAIL}++;
-					$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-					$expected_fail++;
-				} else {
-					$statistics->{TESTS_UNEXPECTED_FAIL}++;
-					$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
-					$unexpected_fail++;
-				}
-			} elsif ($1 eq "knownfail") {
+				$statistics->{TESTS_EXPECTED_OK}++;
+				$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
+			} elsif ($1 eq "xfail" or $1 eq "knownfail") {
 				pop(@$open_tests); #FIXME: Check that popped value == $2
 				$statistics->{TESTS_EXPECTED_FAIL}++;
 				$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
+				$expected_fail++;
+			} elsif ($1 eq "failure" or $1 eq "fail") {
+				pop(@$open_tests); #FIXME: Check that popped value == $2
+				$statistics->{TESTS_UNEXPECTED_FAIL}++;
+				$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
+				$unexpected_fail++;
 			} elsif ($1 eq "skip") {
 				$statistics->{TESTS_SKIP}++;
 				pop(@$open_tests); #FIXME: Check that popped value == $2
diff --git a/selftest/filter-xfail.pl b/selftest/filter-xfail.pl
index 0e0c4e22bc..edf0e050a7 100755
--- a/selftest/filter-xfail.pl
+++ b/selftest/filter-xfail.pl
@@ -157,6 +157,6 @@ sub end_test($$$$$)
 my $msg_ops = {};
 bless $msg_ops;
 
-parse_results($msg_ops, $statistics, *STDIN, sub { return 0; }, []);
+parse_results($msg_ops, $statistics, *STDIN, []);
 
 0;
diff --git a/selftest/format-subunit.pl b/selftest/format-subunit.pl
index 000346a9af..c60902f8d7 100755
--- a/selftest/format-subunit.pl
+++ b/selftest/format-subunit.pl
@@ -61,7 +61,7 @@ if ($opt_format eq "buildfarm") {
 }
 
 my $expected_ret = parse_results(
-	$msg_ops, $statistics, *STDIN, sub { return 0; }, []);
+	$msg_ops, $statistics, *STDIN, []);
 
 $msg_ops->summary();
 
diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index b786c9a50f..8a2c8804bc 100755
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -77,23 +77,10 @@ when the server is running locally.
 Will prevent TCP and UDP ports being opened on the local host but 
 (transparently) redirects these calls to use unix domain sockets.
 
-=item I<--expected-failures>
-
-Specify a file containing a list of tests that are expected to fail. Failures for 
-these tests will be counted as successes, successes will be counted as failures.
-
-The format for the file is, one entry per line:
-
-TESTSUITE-NAME.TEST-NAME
-
-The reason for a test can also be specified, by adding a hash sign (#) and the reason 
-after the test name.
-
 =item I<--exclude>
 
 Specify a file containing a list of tests that should be skipped. Possible 
-candidates are tests that segfault the server, flip or don't end. The format of this file is the same as 
-for the --expected-failures flag.
+candidates are tests that segfault the server, flip or don't end. 
 
 =item I<--include>
 
@@ -157,7 +144,6 @@ my $opt_socket_wrapper_pcap = undef;
 my $opt_socket_wrapper_keep_pcap = undef;
 my $opt_one = 0;
 my $opt_immediate = 0;
-my $opt_expected_failures = undef;
 my @opt_exclude = ();
 my @opt_include = ();
 my $opt_verbose = 0;
@@ -176,7 +162,6 @@ my $builddir = ".";
 my $exeext = "";
 my $prefix = "./st";
 
-my @expected_failures = ();
 my @includes = ();
 my @excludes = ();
 
@@ -205,12 +190,6 @@ sub find_in_list($$)
 	return undef;
 }
 
-sub expecting_failure($)
-{
-	my ($name) = @_;
-	return find_in_list(\@expected_failures, $name);
-}
-
 sub skip($)
 {
 	my ($name) = @_;
@@ -265,7 +244,7 @@ sub run_testsuite($$$$$$)
 	}
 
 	my $expected_ret = parse_results(
-		$msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
+		$msg_ops, $statistics, *RESULT, [$name]);
 
 	my $envlog = getlog_env($envname);
 	$msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
@@ -323,7 +302,6 @@ Target Specific:
                             failed
  --socket-wrapper           enable socket wrapper
  --bindir=PATH              path to target binaries
- --expected-failures=FILE   specify list of tests that is guaranteed to fail
 
 Samba4 Specific:
  --ldap=openldap|fedora-ds  back samba onto specified ldap server
@@ -351,7 +329,6 @@ my $result = GetOptions (
 		'quick' => \$opt_quick,
 		'one' => \$opt_one,
 		'immediate' => \$opt_immediate,
-		'expected-failures=s' => \$opt_expected_failures,
 		'exclude=s' => \@opt_exclude,
 		'include=s' => \@opt_include,
 		'srcdir=s' => \$srcdir,
@@ -537,10 +514,6 @@ sub read_test_regexes($)
 	return @ret;
 }
 
-if (defined($opt_expected_failures)) {
-	@expected_failures = read_test_regexes($opt_expected_failures);
-}
-
 foreach (@opt_exclude) {
 	push (@excludes, read_test_regexes($_));
 }
diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 033fd7938d..0b1f340f32 100644
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -3,14 +3,17 @@ TEST_FORMAT = plain
 SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
     $(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
     --builddir=$(builddir) --srcdir=$(srcdir) \
-    --expected-failures=$(srcdir)/selftest/knownfail \
 	--format=subunit \
     --exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
     $(TEST_OPTIONS) 
 
 SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
 SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
-FORMAT_TEST_OUTPUT = $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
+FILTER_XFAIL = $(PERL) $(selftestdir)/filter-xfail.pl --expected-failures=$(srcdir)/selftest/knownfail
+FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
+
+subunittest:: everything
+	$(SELFTEST) $(TESTS)
 
 slowtest:: everything
 	$(SELFTEST) $(DEFAULT_TEST_OPTIONS) --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate