fixed make test reporting success when provision fails

We were getting build test 'pass' on the build farm when the provision
is failing. The cause was the use of filter pipelines in the selftest
code in the makefile. A bourne shell will by default report the error
status of the last element of the pipeline, which means when provision
failed, make thought everything had passed.

We could use the bash 'pipefail' option to fix this, but it isn't
portable, so instead I've modified the makefile to touch st/st_done
when the first element in the pipeline completes successfully. That
means we now get errors in make test, at the expense of making the
makefile even more horrible than it was.

1 files changed, 51 insertions, 24 deletions

diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 51aa771219..9d4fb4b91e 100644
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -1,10 +1,16 @@
 TEST_FORMAT = plain
 
-SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
+SELFTEST = ( rm -f $(selftest_prefix)/st_done && $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
     $(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
     --builddir=$(builddir) --srcdir=$(srcdir) \
     --exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
-    $(TEST_OPTIONS) 
+    $(TEST_OPTIONS)
+# this strange code is to handle the fact that the bash pipefail option is 
+# not portable. When we run selftest we use a pipeline, and the first element
+# of that pipeline might abort. We need to catch this and mark the test
+# as having failed
+ST_TOUCH = && touch $(selftest_prefix)/st_done ) 
+ST_DONE_TEST = @test -f $(selftest_prefix)/st_done || { echo "SELFTEST FAILED"; exit 1; }
 
 SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
 SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
@@ -12,56 +18,71 @@ FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(sr
 FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
 
 test-subunit:: everything
-	$(SELFTEST) --socket-wrapper $(TESTS)
+	$(SELFTEST) --socket-wrapper $(TESTS) $(ST_TOUCH)
+	$(ST_DONE_TEST)
 
 slowtest:: everything
-	$(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 ifeq ($(RUN_FROM_BUILD_FARM),yes)
 test:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FILTER_XFAIL) --strip-passed-output
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FILTER_XFAIL) --strip-passed-output
+	$(ST_DONE_TEST)
 else
 test:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 endif
 
 kvmtest:: everything
 	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) \
-		--target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) --immediate 
+		--target=kvm --image=$(KVM_IMAGE) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 kvmquicktest:: everything
 	$(SELFTEST) $(DEFAULT_TEST_OPTIONS) \
-		$(SELFTEST_QUICK_OPTS) --target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) | $(FORMAT_TEST_OUTPUT) --immediate 
+		$(SELFTEST_QUICK_OPTS) --target=kvm --image=$(KVM_IMAGE) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 testone:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --one $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT)
+	$(ST_DONE_TEST)
 
 test-swrap:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 test-swrap-pcap:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 test-swrap-keep-pcap:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 test-noswrap:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 quicktest:: all
-	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 quicktest-subunit:: all
-	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS)
+	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH)
+	$(ST_DONE_TEST)
 
 quicktestone:: all
-	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
+	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT)
+	$(ST_DONE_TEST)
 
 testenv:: everything
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
 
 testenv-%:: everything
-	SELFTEST_TESTENV=$* $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+	SELFTEST_TESTENV=$* $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+	$(ST_DONE_TEST)
 
 test-%:: 
 	$(MAKE) test TESTS=$*
@@ -71,29 +92,35 @@ valgrindtest:: valgrindtest-all
 valgrindtest-quick:: all
 	SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
 	VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
-	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 valgrindtest-all:: everything
 	SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
 	VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 valgrindtest-env:: everything
 	SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
 	VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+	$(ST_DONE_TEST)
 
 gdbtest:: gdbtest-all
 
 gdbtest-quick:: all
 	SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
-	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 gdbtest-all:: everything
 	SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+	$(ST_DONE_TEST)
 
 gdbtest-env:: everything
 	SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
-	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+	$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+	$(ST_DONE_TEST)