selftest: Properly recognize xfail testsuites.

5 files changed, 48 insertions, 10 deletions

diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm
index 16aab73926..bb23483d12 100644
--- a/selftest/Subunit.pm
+++ b/selftest/Subunit.pm
@@ -39,7 +39,7 @@ sub parse_results($$$)
 			push (@$open_tests, $1);
 		} elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) {
 			$msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1-1900));
-		} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail|skip-testsuite|testsuite-failure|testsuite-success|testsuite-error): (.*?)( \[)?([ \t]*)\n/) {
+		} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail|skip-testsuite|testsuite-failure|testsuite-xfail|testsuite-success|testsuite-error): (.*?)( \[)?([ \t]*)\n/) {
 			$msg_ops->control_msg($_);
 			my $result = $1;
 			my $testname = $2;
@@ -91,6 +91,8 @@ sub parse_results($$$)
 				$msg_ops->end_testsuite($testname, "success", $reason);
 			} elsif ($result eq "testsuite-failure") {
 				$msg_ops->end_testsuite($testname, "failure", $reason);
+			} elsif ($result eq "testsuite-xfail") {
+				$msg_ops->end_testsuite($testname, "xfail", $reason);
 			} elsif ($result eq "testsuite-error") {
 				$msg_ops->end_testsuite($testname, "error", $reason);
 			} 
diff --git a/selftest/Subunit/Filter.pm b/selftest/Subunit/Filter.pm
index 3a9e4f9409..1bd4e5fed2 100644
--- a/selftest/Subunit/Filter.pm
+++ b/selftest/Subunit/Filter.pm
@@ -76,10 +76,14 @@ sub end_test($$$$$)
 		$testname = $self->{prefix}.$testname;
 	}
 
-	if (($result eq "fail" or $result eq "failure") and not $unexpected) { $result = "xfail"; }
+	if (($result eq "fail" or $result eq "failure") and not $unexpected) {
+		$result = "xfail";
+		$self->{xfail_added}++;
+	}
 	my $xfail_reason = find_in_list($self->{expected_failures}, $testname);
 	if (defined($xfail_reason) and ($result eq "fail" or $result eq "failure")) {
 		$result = "xfail";
+		$self->{xfail_added}++;
 		$reason .= $xfail_reason;
 	}
 
@@ -95,11 +99,16 @@ sub start_testsuite($;$)
 {
 	my ($self, $name) = @_;
 	Subunit::start_testsuite($name);
+	$self->{xfail_added} = 0;
 }
 
 sub end_testsuite($$;$)
 {
 	my ($self, $name, $result, $reason) = @_;
+	if ($self->{xfail_added} and ($result eq "fail" or $result eq "failure")) {
+		$result = "xfail";
+	}
+		
 	Subunit::end_testsuite($name, $result, $reason);
 }
 
@@ -115,6 +124,7 @@ sub new {
 	my $self = { 
 		prefix => $prefix,
 		expected_failures => $expected_failures,
+		xfail_added => 0,
 	};
 	bless($self, $class);
 }
diff --git a/selftest/output/buildfarm.pm b/selftest/output/buildfarm.pm
index 95c5423383..bd21fc683d 100644
--- a/selftest/output/buildfarm.pm
+++ b/selftest/output/buildfarm.pm
@@ -86,6 +86,9 @@ sub end_testsuite($$$$$$)
 	BuildFarm::end_testsuite($name, ($self->{last_time} - $self->{START_TIME}), 
 		                     (not $unexpected), $self->{test_output}->{$name}, 
 							 $reason);
+	if ($result ne "success") {
+		$self->{statistics}->{SUITES_FAIL}++;
+	}
 }
 
 sub start_test($$$)
diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm
index bfdced2936..5e75ce9511 100644
--- a/selftest/output/plain.pm
+++ b/selftest/output/plain.pm
@@ -125,7 +125,7 @@ sub end_testsuite($$$$)
 	}
 
 	if (not $self->{immediate}) {
-		if (not $unexpected) {
+		unless($unexpected) {
 			$out .= " ok\n";
 		} else {
 			$out .= " " . uc($result) . "\n";
diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index 93a3ca27a9..28e0460a5a 100755
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -129,7 +129,8 @@ use Getopt::Long;
 use POSIX;
 use Cwd qw(abs_path);
 use lib "$RealBin";
-use Subunit;
+use Subunit qw(parse_results);
+use Subunit::Filter;
 use SocketWrapper;
 
 my $opt_help = 0;
@@ -219,12 +220,34 @@ sub run_testsuite($$$$$)
 	Subunit::report_time(time());
 	Subunit::start_testsuite($name);
 
-	system("$cmd 2>&1 | $RealBin/filter-subunit.pl --prefix \"$name.\"");
-	my $ret = $?;
-	if ($ret == -1) {
-		Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
-		return 0;
-	} elsif ($ret & 127) {
+	open(RESULTS, "$cmd 2>&1|");
+	my $statistics = {
+		SUITES_FAIL => 0,
+
+		TESTS_UNEXPECTED_OK => 0,
+		TESTS_EXPECTED_OK => 0,
+		TESTS_UNEXPECTED_FAIL => 0,
+		TESTS_EXPECTED_FAIL => 0,
+		TESTS_ERROR => 0,
+		TESTS_SKIP => 0,
+	};
+
+	my $msg_ops = new Subunit::Filter("$name\.", []);
+
+	parse_results($msg_ops, $statistics, *RESULTS);
+
+	my $ret = 0;
+
+	unless (close(RESULTS)) {
+		if ($!) {
+			Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
+			return 0;
+		} else {
+			$ret = $?;
+		}
+	} 
+
+	if ($ret & 127) {
 		Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without"));
 		return 0;
 	}