r21707: Finally merge my (long-living) perlselftest branch.

This changes the main selftest code to be in perl rather than in shell script.

The selftest script is now no longer a black box but a regular executable that takes
--help.

This adds the following features:

 * "make test TESTS=foo" will run only the tests that match the regex "foo"
 * ability to deal with expected failures. the suite will not warn about tests
   that fail and are known to fail, but will warn about other failing tests and
   tests that are succeeding tests but incorrectly marked as failing.
 * ability to print a summary with all failures at the end of the run

It also opens up the way to the following features, which I hope to implement later:
 * "environments", for example having a complete domains with DCs and domain members
 in a testenvironment
 * only set up smbd if necessary (not when running LOCAL tests, for example)
 * different mktestsetup scripts per target. except for the mktestsetup script, we can
   use the same infrastructure for samba 3 or windows.
(This used to be commit 38f867880beb40c691e9713f854426031310629c)

1 files changed, 373 insertions, 155 deletions

diff --git a/source4/script/tests/selftest.pl b/source4/script/tests/selftest.pl
index b14333e0b5..6c2ea6d0b1 100755
--- a/source4/script/tests/selftest.pl
+++ b/source4/script/tests/selftest.pl
@@ -2,6 +2,107 @@
 # Bootstrap Samba and run a number of tests against it.
 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
 # Published under the GNU GPL, v3 or later.
+
+=pod
+
+=head1 NAME
+
+selftest - Samba test runner
+
+=head1 SYNOPSIS
+
+selftest --help
+
+selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
+
+=head1 DESCRIPTION
+
+A simple test runner. TESTS is a regular expression with tests to run.
+
+=head1 OPTIONS
+
+=over 4
+
+=item I<--help>
+
+Show list of available options.
+
+=item I<--srcdir=DIR>
+
+Source directory.
+
+=item I<--builddir=DIR>
+
+Build directory.
+
+=item I<--prefix=DIR>
+
+Change directory to run tests in. Default is 'st'.
+
+=item I<--immediate>
+
+Show errors as soon as they happen rather than at the end of the test run.
+		
+=item I<--target samba4|samba3|win>
+
+Specify test target against which to run. Default is 'samba4'.
+
+=item I<--quick>
+
+Run only a limited number of tests. Intended to run in about 30 seconds on 
+moderately recent systems.
+		
+=item I<--socket-wrapper>
+
+Use socket wrapper library for communication with server. Only works 
+when the server is running locally.
+
+Will prevent TCP and UDP ports being opened on the local host but 
+(transparently) redirects these calls to use unix domain sockets.
+
+=item I<--expected-failures>
+
+Specify a file containing a list of tests that are expected to fail. Failures for 
+these tests will be counted as successes, successes will be counted as failures.
+
+The format for the file is, one entry per line:
+
+TESTSUITE-NAME/TEST-NAME
+
+=item I<--one>
+
+Abort as soon as one test fails.
+
+=back
+
+=head1 ENVIRONMENT
+
+=over 4
+
+=item I<SMBD_VALGRIND>
+
+=item I<TORTURE_MAXTIME>
+
+=item I<VALGRIND>
+
+=item I<TEST_LDAP>
+
+=item I<TLS_ENABLED>
+
+=item I<srcdir>
+
+=back
+
+=head1 LICENSE
+
+selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
+
+=head1 AUTHOR
+
+Jelmer Vernooij
+
+=cut
+
 use strict;
 use warnings;
 
@@ -10,86 +111,163 @@ use File::Spec;
 use Getopt::Long;
 use POSIX;
 use Cwd;
+use lib "$RealBin";
+use Samba4;
+use SocketWrapper;
 
-sub slapd_start($$) {
-	my ($conf, $uri) = @_;
-    my $oldpath = $ENV{PATH};
-    $ENV{PATH} = "/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
-	# running slapd in the background means it stays in the same process group, so it can be
-	# killed by timelimit
-    system("slapd -d0 -f $conf -h $uri &");
-    $ENV{PATH} = $oldpath;
-    return $? >> 8;
-}
+my $opt_help = 0;
+my $opt_target = "samba4";
+my $opt_quick = 0;
+my $opt_socket_wrapper = 0;
+my $opt_socket_wrapper_pcap = undef;
+my $opt_one = 0;
+my $opt_immediate = 0;
+my $opt_expected_failures = undef;
+my $opt_verbose = 0;
 
-sub smbd_check_or_start($$$$$$) 
+my $srcdir = ".";
+my $builddir = ".";
+my $prefix = "st";
+
+my $suitesfailed = [];
+my $start = time();
+my @expected_failures = ();
+
+my $statistics = {
+	SUITES_FAIL => 0,
+	SUITES_OK => 0,
+
+	TESTS_UNEXPECTED_OK => 0,
+	TESTS_EXPECTED_OK => 0,
+	TESTS_UNEXPECTED_FAIL => 0,
+	TESTS_EXPECTED_FAIL => 0,
+	TESTS_ERROR => 0
+};
+
+sub expecting_failure($)
 {
-	my ($bindir, $test_fifo, $test_log, $socket_wrapper_dir, $max_time, $conffile) = @_;
-	return 0 if ( -p $test_fifo );
+	my $fullname = shift;
 
-	if (defined($socket_wrapper_dir)) {
-		if ( -d $socket_wrapper_dir ) {
-			unlink <$socket_wrapper_dir/*>;
-		} else {
-			mkdir($socket_wrapper_dir);
-		}
+	foreach (@expected_failures) {
+		return 1 if $fullname =~ /^$_$/;
 	}
 
-	unlink($test_fifo);
-	system("mkfifo $test_fifo");
-
-	unlink($test_log);
-	
-	my $valgrind = "";
-	if (defined($ENV{SMBD_VALGRIND})) {
-		$valgrind = $ENV{SMBD_VALGRIND};
-	} 
-
-	print "STARTING SMBD...";
-	my $pid = fork();
-	if ($pid == 0) {
-		my $ret = system("$valgrind $bindir/smbd --maximum-runtime=$max_time -s $conffile -M single -i --leak-report-full < $test_fifo > $test_log");
-		open LOG, ">>$test_log";
-		if ($? == -1) {
-			print LOG "Unable to start smbd: $ret: $!\n";
-			print "Unable to start smbd: $ret: $!\n";
-			exit 1;
-		}
-		unlink($test_fifo);
-		unlink(<$socket_wrapper_dir/*>) if (defined($socket_wrapper_dir) and -d $socket_wrapper_dir);
-		my $exit = $? >> 8;
-		if ( $ret == 0 ) {
-			print "smbd exits with status $exit\n";
-			print LOG "smbd exits with status $exit\n";
-		} elsif ( $ret & 127 ) {
-			print "smbd got signal ".($ret & 127)." and exits with $exit!\n";
-			print LOG "smbd got signal".($ret & 127). " and exits with $exit!\n";
-		} else {
-			$ret = $? >> 8;
-			print "smbd failed with status $exit!\n";
-			print LOG "smbd failed with status $exit!\n";
+	return 0;
+}
+
+sub run_test_buildfarm($$$$)
+{
+	my ($name, $cmd, $i, $suitestotal) = @_;
+	print "--==--==--==--==--==--==--==--==--==--==--\n";
+	print "Running test $name (level 0 stdout)\n";
+	print "--==--==--==--==--==--==--==--==--==--==--\n";
+	system("date");
+
+	my $expected_ret = 1;
+	my $open_tests = {};
+	open(RESULT, "$cmd|");
+	while (<RESULT>) { 
+		print;
+		if (/^test: (.+)\n/) {
+			$open_tests->{$1} = 1;
+		} elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
+			my $result = $1;
+			if ($1 eq "success") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_UNEXPECTED_OK}++;
+				} else {
+					$statistics->{TESTS_EXPECTED_OK}++;
+				}
+			} elsif ($1 eq "failure") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_EXPECTED_FAIL}++;
+					$expected_ret = 0;
+				} else {
+					$statistics->{TESTS_UNEXPECTED_FAIL}++;
+				}
+			} elsif ($1 eq "skip") {
+				delete $open_tests->{$2};
+			} elsif ($1 eq "error") {
+				$statistics->{TESTS_ERROR}++;
+				delete $open_tests->{$2};
+			}
 		}
-		close(LOG);
-		exit $exit;
 	}
-	print "DONE\n";
+	print "COMMAND: $cmd\n";
+	foreach (keys %$open_tests) {
+		print "$_ was started but never finished!\n";		
+		$statistics->{TESTS_ERROR}++;
+	}
+	my $ret = close(RESULT);
 
-	return $pid;
+	print "==========================================\n";
+	if ($ret == $expected_ret) {
+		print "TEST PASSED: $name\n";
+	} else {
+		print "TEST FAILED: $name (status $ret)\n";
+	}
+	print "==========================================\n";
 }
 
-sub teststatus($$) {
-	my ($name, $failed) = @_;
-
-	print "TEST STATUS: $failed failures\n";
-	if ($failed > 0) {
-print <<EOF	    
-************************
-*** TESTSUITE FAILED ***
-************************
-EOF
-;
+my $test_output = {};
+sub run_test_plain($$$$)
+{
+	my ($name, $cmd, $i, $totalsuites) = @_;
+	my $err = "";
+	if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
+	printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
+	open(RESULT, "$cmd 2>&1|");
+	my $expected_ret = 1;
+	my $open_tests = {};
+	$test_output->{$name} = "";
+	while (<RESULT>) { 
+		$test_output->{$name}.=$_;
+		print if ($opt_verbose);
+		if (/^test: (.+)\n/) {
+			$open_tests->{$1} = 1;
+		} elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
+			my $result = $1;
+			if ($1 eq "success") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_UNEXPECTED_OK}++;
+				} else {
+					$statistics->{TESTS_EXPECTED_OK}++;
+				}
+			} elsif ($1 eq "failure") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_EXPECTED_FAIL}++;
+					$expected_ret = 0;
+				} else {
+					$statistics->{TESTS_UNEXPECTED_FAIL}++;
+				}
+			} elsif ($1 eq "skip") {
+				delete $open_tests->{$2};
+			} elsif ($1 eq "error") {
+				$statistics->{TESTS_ERROR}++;
+				delete $open_tests->{$2};
+			}
+		}
+	}
+	$test_output->{$name}.="COMMAND: $cmd\n";
+	foreach (keys %$open_tests) {
+		$test_output->{$name}.="$_ was started but never finished!\n";		
+		$statistics->{TESTS_ERROR}++;
+	}
+	my $ret = close(RESULT);
+	if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
+		print "$test_output->{$name}\n";
+	}
+	if ($ret != $expected_ret) {
+		push(@$suitesfailed, $name);
+		$statistics->{SUITES_FAIL}++;
+		exit(1) if ($opt_one);
+	} else {
+		$statistics->{SUITES_OK}++;
 	}
-	exit $failed;
 }
 
 sub ShowHelp()
@@ -97,51 +275,60 @@ sub ShowHelp()
 	print "Samba test runner
 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 
-Usage: $Script PREFIX
+Usage: $Script [OPTIONS] PREFIX
 
 Generic options:
  --help                     this help page
+
+Paths:
+ --prefix=DIR               prefix to run tests in [st]
+ --srcdir=DIR               source directory [.]
+ --builddir=DIR             output directory [.]
+
+Target Specific:
  --target=samba4|samba3|win Samba version to target
+ --socket-wrapper-pcap=FILE save traffic to pcap file
  --socket-wrapper           enable socket wrapper
+ --expected-failures=FILE   specify list of tests that is guaranteed to fail
+
+Behaviour:
  --quick                    run quick overall test
  --one                      abort when the first test fails
+ --immediate                print test output for failed tests during run
+ --verbose                  be verbose
 ";
 	exit(0);
 }
 
-my $opt_help = 0;
-my $opt_target = "samba4";
-my $opt_quick = 0;
-my $opt_socket_wrapper = 0;
-my $opt_one = 0;
-
 my $result = GetOptions (
 	    'help|h|?' => \$opt_help,
-		'target' => \$opt_target,
+		'target=s' => \$opt_target,
+		'prefix=s' => \$prefix,
 		'socket-wrapper' => \$opt_socket_wrapper,
+		'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
 		'quick' => \$opt_quick,
-		'one' => \$opt_one
+		'one' => \$opt_one,
+		'immediate' => \$opt_immediate,
+		'expected-failures=s' => \$opt_expected_failures,
+		'srcdir=s' => \$srcdir,
+		'builddir=s' => \$builddir,
+		'verbose' => \$opt_verbose
 	    );
 
-if (not $result) {
-	exit(1);
-}
+exit(1) if (not $result);
 
 ShowHelp() if ($opt_help);
-ShowHelp() if ($#ARGV < 0);
 
-my $prefix = shift;
+my $tests = shift;
 
 my $torture_maxtime = $ENV{TORTURE_MAXTIME};
 unless (defined($torture_maxtime)) {
 	$torture_maxtime = 1200;
 }
 
-# disable rpc validation when using valgrind - its way too slow
-my $valgrind = $ENV{VALGRIND};
-my $validate = undef;
-unless (defined($valgrind)) {
-	$validate = "validate";
+# quick hack to disable rpc validation when using valgrind - its way too slow
+unless (defined($ENV{VALGRIND})) {
+	$ENV{VALIDATE} = "validate";
 }
 
 my $old_pwd = "$RealBin/../..";
@@ -150,10 +337,6 @@ my $ldap = (defined($ENV{TEST_LDAP}) and ($ENV{TEST_LDAP} eq "yes"))?1:0;
 $prefix =~ s+//+/+;
 $ENV{PREFIX} = $prefix;
 
-my $srcdir = "$RealBin/../..";
-if (defined($ENV{srcdir})) {
-	$srcdir = $ENV{srcdir};
-}
 $ENV{SRCDIR} = $srcdir;
 
 my $bindir = "$srcdir/bin";
@@ -161,6 +344,8 @@ my $setupdir = "$srcdir/setup";
 my $testsdir = "$srcdir/script/tests";
 
 my $tls_enabled = not $opt_quick;
+my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and 
+                      ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 
 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
@@ -173,14 +358,12 @@ if (defined($ENV{LD_LIBRARY_PATH})) {
 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 
+my @torture_options = ();
+
+my $testenv_vars = {};
+
 if ($opt_target eq "samba4") {
-	print "PROVISIONING...";
-	open(IN, "$RealBin/mktestsetup.sh $prefix|") or die("Unable to setup");
-	while (<IN>) {
-		next unless (/^([A-Z_]+)=(.*)$/);
-		$ENV{$1} = $2;
-	}
-	close(IN);
+	$testenv_vars = Samba4::provision($prefix);
 } elsif ($opt_target eq "win") {
 	die ("Windows tests will not run without root privileges.") 
 		if (`whoami` ne "root");
@@ -196,77 +379,82 @@ if ($opt_target eq "samba4") {
 	die ("$ENV{WINTESTCONF} could not be read.") if (! -r $ENV{WINTESTCONF});
 
 	$ENV{WINTEST_DIR}="$ENV{SRCDIR}/script/tests/win";
+} elsif ($opt_target eq "none") {
 } else {
 	die("unknown target `$opt_target'");
 }
 
-my $socket_wrapper_dir = undef;
+foreach (keys %$testenv_vars) { $ENV{$_} = $testenv_vars->{$_}; }
+
+if ($opt_socket_wrapper_pcap) {
+	$ENV{SOCKET_WRAPPER_PCAP_FILE} = $opt_socket_wrapper_pcap;
+	# Socket wrapper pcap implies socket wrapper
+	$opt_socket_wrapper = 1;
+}
 
-if ( $opt_socket_wrapper) 
+my $socket_wrapper_dir;
+if ($opt_socket_wrapper) 
 {
-	$socket_wrapper_dir = "$prefix/w";
-	$ENV{SOCKET_WRAPPER_DIR} = $socket_wrapper_dir;
-	print "SOCKET_WRAPPER_DIR=$ENV{SOCKET_WRAPPER_DIR}\n";
-} else {
-	print "NOT USING SOCKET_WRAPPER\n";
+	$socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
+	print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 }
 
 # Start slapd before smbd
 if ($ldap) {
-    slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
+	Samba4::slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
+
     print "LDAP PROVISIONING...";
-    system("$bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} --ldap-backend=$ENV{LDAPI}") or
-		die("LDAP PROVISIONING failed: $bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} --ldap-backend=$ENV{LDAPI}");
+	Samba4::provision_ldap($bindir, $setupdir);
 
     # LDAP is slow
 	$torture_maxtime *= 2;
 }
 
+if (defined($opt_expected_failures)) {
+	open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
+	while (<KNOWN>) { 
+		chomp; 
+		s/([ \t]+)\#(.*)$//;
+		push (@expected_failures, $_); }
+	close(KNOWN);
+}
+
 my $test_fifo = "$prefix/smbd_test.fifo";
 
 $ENV{SMBD_TEST_FIFO} = $test_fifo;
 $ENV{SMBD_TEST_LOG} = "$prefix/smbd_test.log";
 
-$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = 1;
+SocketWrapper::set_default_iface(1);
 my $max_time = 5400;
 if (defined($ENV{SMBD_MAX_TIME})) {
 	$max_time = $ENV{SMBD_MAX_TIME};
 }
-smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG}, $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
+Samba4::smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG}, 
+	                $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
+
+SocketWrapper::set_default_iface(6);
 
-$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = 6;
-$ENV{TORTURE_INTERFACES} = '127.0.0.6/8,127.0.0.7/8,127.0.0.8/8,127.0.0.9/8,127.0.0.10/8,127.0.0.11/8';
+my $interfaces = join(',', ("127.0.0.6/8", 
+		                 "127.0.0.7/8",
+						 "127.0.0.8/8",
+						 "127.0.0.9/8",
+						 "127.0.0.10/8",
+						 "127.0.0.11/8"));
 
-my @torture_options = ("--option=interfaces=$ENV{TORTURE_INTERFACES} $ENV{CONFIGURATION}");
+push (@torture_options, "--option=interfaces=$interfaces");
+push (@torture_options, $ENV{CONFIGURATION});
 # ensure any one smbtorture call doesn't run too long
 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 push (@torture_options, "--target=$opt_target");
-push (@torture_options, "--option=torture:progress=no") 
-	if (defined($ENV{RUN_FROM_BUILD_FARM}) and $ENV{RUN_FROM_BUILD_FARM} eq "yes");
+push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
+push (@torture_options, "--format=subunit");
+push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 
 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 
-my $start = time();
-
 open(DATA, ">$test_fifo");
 
-# give time for nbt server to register its names
-print "delaying for nbt name registration\n";
-sleep(4);
-
-# This will return quickly when things are up, but be slow if we need to wait for (eg) SSL init 
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{NETBIOSNAME}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
-
-# start off with 0 failures
-$ENV{failed} = 0;
-my $totalfailed = 0;
-
 my @todo = ();
 
 if ($opt_target eq "win") {
@@ -283,27 +471,35 @@ if ($opt_target eq "win") {
 			$name =~ s/\n//g;
 			my $cmdline = <IN>;
 			$cmdline =~ s/\n//g;
-			push (@todo, [$name, $cmdline]);
+			push (@todo, [$name, $cmdline]) 
+				if (not defined($tests) or $name =~ /$tests/);
 		} else {
 			print;
 		}
 	}
-	close(IN);
+	close(IN) or die("Error creating recipe");
 }
 
-my $total = $#todo + 1;
+Samba4::wait_for_start();
+
+# start off with 0 failures
+$ENV{failed} = 0;
+
+my $suitestotal = $#todo + 1;
 my $i = 0;
 $| = 1;
 
+delete $ENV{DOMAIN};
+
 foreach (@todo) {
-	$i = $i + 1;
-	my $err = "";
-	if ($totalfailed > 0) { $err = ", $totalfailed errors"; }
-	printf "[$i/$total in " . (time() - $start)."s$err] $$_[0]\n";
-	my $ret = system("$$_[1] >/dev/null 2>/dev/null");
-	if ($ret != 0) {
-		$totalfailed++;
-		exit(1) if ($opt_one);
+	$i++;
+	my $cmd = $$_[1];
+	$cmd =~ s/([\(\)])/\\$1/g;
+	my $name = $$_[0];
+	if ($from_build_farm) {
+		run_test_buildfarm($name, $cmd, $i, $suitestotal);
+	} else {
+		run_test_plain($name, $cmd, $i, $suitestotal);
 	}
 }
 
@@ -311,6 +507,8 @@ print "\n";
 
 close(DATA);
 
+sleep(2);
+
 my $failed = $? >> 8;
 
 if (-f "$ENV{PIDDIR}/smbd.pid" ) {
@@ -319,15 +517,37 @@ if (-f "$ENV{PIDDIR}/smbd.pid" ) {
 	close(IN);
 }
 
-if ($ldap) {
-    open(IN, "<$ENV{PIDDIR}/slapd.pid") or die("unable to open slapd pid file");
-	kill 9, <IN>;
-	close(IN);
-}
+Samba4::slapd_stop() if ($ldap);
 
-my $end=time();
-print "DURATION: " . ($end-$start). " seconds\n";
-print "$totalfailed failures\n";
+my $end = time();
+my $duration = ($end-$start);
+my $numfailed = $#$suitesfailed+1;
+if ($numfailed == 0) {
+	my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
+	print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
+} else {
+
+	unless ($from_build_farm) {
+		if (not $opt_immediate and not $opt_verbose) {
+			foreach (@$suitesfailed) {
+				print "===============================================================================\n";
+				print "FAIL: $_\n";
+				print $test_output->{$_};
+				print "\n";
+			}
+		}
+
+		print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
+	} else {
+		print <<EOF	    
+************************
+*** TESTSUITE FAILED ***
+************************
+EOF
+;
+	}
+}
+print "DURATION: $duration seconds\n";
 
 # if there were any valgrind failures, show them
 foreach (<$prefix/valgrind.log*>) {
@@ -340,6 +560,4 @@ foreach (<$prefix/valgrind.log*>) {
 	}
 }
 
-teststatus($Script, $failed);
-
-exit $failed;
+exit $numfailed;