r19343: Add support for external scripts/binaries that write results using the

'subunit' protocol. This allows us to easily plug EJS scripts or binaries that
can't depend on -ltorture into smbtorture. The protocol is very simple:

- write "comments" to stderr

Example output on stdout:
test: foo
success: foo
test: bar
success: bar
test: blah
failure: blah [
dummy.c:30: Expression 1 != 2 failed!
]
test: blie
skip: blie [
Iconv support not built in
]

I've already converted the talloc testsuite.
(This used to be commit e1742c14a247fabba969f8698108e73997d3f420)

7 files changed, 303 insertions, 29 deletions

diff --git a/source4/torture/config.mk b/source4/torture/config.mk
index 32205c5566..1edaa69447 100644
--- a/source4/torture/config.mk
+++ b/source4/torture/config.mk
@@ -6,7 +6,8 @@ VERSION = 0.0.1
 PUBLIC_HEADERS = torture.h
 PUBLIC_PROTO_HEADER = proto.h
 OBJ_FILES = \
-		torture.o 
+		torture.o \
+		subunit.o
 PUBLIC_DEPENDENCIES = \
 		LIBSAMBA-CONFIG \
 		LIBSAMBA-UTIL
diff --git a/source4/torture/local/config.mk b/source4/torture/local/config.mk
index 682fb55416..a997732c29 100644
--- a/source4/torture/local/config.mk
+++ b/source4/torture/local/config.mk
@@ -7,7 +7,6 @@ PRIVATE_PROTO_HEADER = \
 		proto.h
 OBJ_FILES = \
 		iconv.o \
-		../../lib/talloc/testsuite.o \
 		../../lib/replace/test/testsuite.o \
 		../../lib/replace/test/os2_delete.o \
 		../../lib/crypto/md4test.o \
diff --git a/source4/torture/local/local.c b/source4/torture/local/local.c
index 42fe94bc92..2408971199 100644
--- a/source4/torture/local/local.c
+++ b/source4/torture/local/local.c
@@ -52,12 +52,7 @@ NTSTATUS torture_local_init(void)
 	struct torture_suite *suite = torture_suite_create(
 										talloc_autofree_context(),
 										"LOCAL");
-	struct torture_suite *talloc_suite = torture_suite_create(
-												talloc_autofree_context(),
-												"TALLOC");
 
-	torture_local_talloc(talloc_suite);
-	torture_suite_add_suite(suite, talloc_suite);
 	torture_suite_add_simple_test(suite, "REPLACE", torture_local_replace);
 	torture_suite_add_simple_test(suite, "CRYPTO-SHA1", 
 								  torture_local_crypto_sha1);
diff --git a/source4/torture/smbtorture.c b/source4/torture/smbtorture.c
index 4c2cfa5524..3d7a75f8cf 100644
--- a/source4/torture/smbtorture.c
+++ b/source4/torture/smbtorture.c
@@ -27,6 +27,7 @@
 #include "libcli/libcli.h"
 #include "lib/ldb/include/ldb.h"
 #include "lib/events/events.h"
+#include "dynconfig.h"
 
 #include "torture/torture.h"
 #include "build.h"
@@ -326,25 +327,27 @@ static void subunit_test_result (struct torture_context *context,
 {
 	switch (res) {
 	case TORTURE_OK:
-		printf("success: %s\n", context->active_test->name);
+		printf("success: %s", context->active_test->name);
 		break;
 	case TORTURE_FAIL:
-		printf("failure: %s [ %s ]\n", context->active_test->name, reason);
+		printf("failure: %s", context->active_test->name);
 		break;
 	case TORTURE_TODO:
-		printf("todo: %s\n", context->active_test->name);
+		printf("todo: %s", context->active_test->name);
 		break;
 	case TORTURE_SKIP:
-		printf("skip: %s\n", context->active_test->name);
+		printf("skip: %s", context->active_test->name);
 		break;
 	}
+	if (reason)
+		printf(" [ %s ]", reason);
+	printf("\n");
 }
 
 static void subunit_comment (struct torture_context *test, 
 							 const char *comment)
 {
-	/* FIXME Add # sign before each line */
-	printf("%s", comment);
+	fprintf(stderr, "%s", comment);
 }
 
 const static struct torture_ui_ops subunit_ui_ops = {
@@ -438,6 +441,7 @@ const static struct torture_ui_ops quiet_ui_ops = {
 	char **argv_new;
 	poptContext pc;
 	static const char *target = "other";
+	const char **subunit_dir;
 	static const char *ui_ops_name = "simple";
 	enum {OPT_LOADFILE=1000,OPT_UNCLIST,OPT_TIMELIMIT,OPT_DNS,
 	      OPT_DANGEROUS,OPT_SMB_PORTS,OPT_ASYNC};
@@ -525,6 +529,17 @@ const static struct torture_ui_ops quiet_ui_ops = {
 	}
 
 	torture_init();
+
+	subunit_dir = lp_parm_string_list(-1, "torture", "subunitdir", ":");
+	if (subunit_dir == NULL) 
+		torture_subunit_load_testsuites(dyn_TORTUREDIR);
+	else {
+		for (i = 0; subunit_dir[i]; i++)
+			torture_subunit_load_testsuites(subunit_dir[i]);
+	}
+
+
+
 	ldb_global_init();
 
 	if (torture_seed == 0) {
diff --git a/source4/torture/subunit.c b/source4/torture/subunit.c
new file mode 100644
index 0000000000..72e39101e2
--- /dev/null
+++ b/source4/torture/subunit.c
@@ -0,0 +1,239 @@
+/* 
+   Unix SMB/CIFS implementation.
+   Run subunit tests
+   Copyright (C) Jelmer Vernooij 2006
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include "includes.h"
+#include "system/dir.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "torture/ui.h"
+#include "torture/proto.h"
+
+NTSTATUS torture_register_subunit_testsuite(const char *path)
+{
+	struct torture_suite *suite = talloc_zero(talloc_autofree_context(), 
+											  struct torture_suite);
+
+	suite->path = talloc_strdup(suite, path);
+	suite->name = talloc_strdup(suite, strrchr(path, '/')?strrchr(path, '/')+1:
+									   path);
+	suite->description = talloc_asprintf(suite, "Subunit test %s", suite->name);
+
+	return torture_register_suite(suite);
+}
+
+int torture_subunit_load_testsuites(const char *directory)
+{
+	DIR *dir;
+	struct dirent *entry;
+	char *filename;
+	int success = 0;
+
+	dir = opendir(directory);
+	if (dir == NULL)
+		return -1;
+
+	while((entry = readdir(dir))) {
+		if (ISDOT(entry->d_name) || ISDOTDOT(entry->d_name))
+			continue;
+
+		filename = talloc_asprintf(NULL, "%s/%s", directory, entry->d_name);
+	
+		if (NT_STATUS_IS_OK(torture_register_subunit_testsuite(filename))) {
+			success++;
+		}
+
+		talloc_free(filename);
+	}
+
+	closedir(dir);
+
+	return success;
+}
+
+static pid_t piped_child(char* const command[], int *f_in)
+{
+	pid_t pid;
+	int sock[2];
+
+	if (socketpair(PF_UNIX, SOCK_STREAM, AF_LOCAL, sock) == -1) {
+		DEBUG(0, ("socketpair: %s", strerror(errno)));
+		return -1;
+	}
+
+	*f_in = sock[0];
+
+	fcntl(sock[0], F_SETFL, O_NONBLOCK);
+
+	pid = fork();
+
+	if (pid == -1) {
+		DEBUG(0, ("fork: %s", strerror(errno)));
+		return -1;
+	}
+
+	if (pid == 0) {
+		close(0);
+		close(1);
+		close(2);
+		close(sock[0]);
+
+		dup2(sock[1], 0);
+		dup2(sock[1], 1);
+		execvp(command[0], command);
+		exit(-1);
+	}
+
+	close(sock[1]);
+
+	return pid;
+}
+
+enum subunit_field { SUBUNIT_TEST, SUBUNIT_SUCCESS, SUBUNIT_FAILURE, 
+					 SUBUNIT_SKIP };
+
+static void run_subunit_message(struct torture_context *context,
+								enum subunit_field field, 
+								const char *name, 
+								const char *comment)
+{
+	struct torture_test test;
+
+	ZERO_STRUCT(test);
+	test.name = name;
+
+	switch (field) {
+	case SUBUNIT_TEST:
+		torture_ui_test_start(context, NULL, &test);
+		break;
+	case SUBUNIT_FAILURE:
+		context->active_test = &test;
+		torture_ui_test_result(context, TORTURE_FAIL, comment);
+		context->active_test = NULL;
+		break;
+	case SUBUNIT_SUCCESS:
+		context->active_test = &test;
+		torture_ui_test_result(context, TORTURE_OK, comment);
+		context->active_test = NULL;
+		break;
+	case SUBUNIT_SKIP:
+		context->active_test = &test;
+		torture_ui_test_result(context, TORTURE_SKIP, comment);
+		context->active_test = NULL;
+		break;
+	}
+}
+
+bool torture_subunit_run_suite(struct torture_context *context, 
+					   struct torture_suite *suite)
+{
+	static char *command[2];
+	int fd;
+	pid_t pid;
+	size_t size;
+	char *p, *q;
+	char *comment = NULL;
+	char *name = NULL;
+	enum subunit_field lastfield;
+	int status;
+	char buffer[4096];
+	size_t offset = 0;
+
+	command[0] = talloc_strdup(context, suite->path);
+	command[1] = NULL;
+
+	pid = piped_child(command, &fd);
+	if (pid == -1)
+		return false;
+
+	if (waitpid(pid, &status, 0) == -1) {
+		torture_comment(context, "waitpid(%d) failed\n", pid);
+		return false;
+	}
+
+	if (WEXITSTATUS(status) != 0) {
+		torture_comment(context, "failed with status %d\n", WEXITSTATUS(status));
+		return false;
+	}
+
+	while ((size = read(fd, buffer+offset, sizeof(buffer-offset) > 0))) {
+		char *eol;
+		buffer[offset+size] = '\0';
+
+		for (p = buffer; p; p = eol+1) {
+			eol = strchr(p, '\n');
+			if (eol == NULL) 
+				break;
+
+			*eol = '\0';
+
+			if (comment != NULL && strcmp(p, "]") == 0) {
+				run_subunit_message(context, lastfield, name, comment);
+				talloc_free(name); name = NULL;
+				talloc_free(comment); comment = NULL;
+			} else if (comment != NULL) {
+				comment = talloc_append_string(context, comment, p);
+			} else {
+				q = strchr(p, ':');
+				if (q == NULL) {
+					torture_comment(context, "Invalid line `%s'\n", p);
+					continue;
+				}
+
+				*q = '\0';
+				if (!strcmp(p, "test")) {
+					lastfield = SUBUNIT_TEST;
+				} else if (!strcmp(p, "failure")) {
+					lastfield = SUBUNIT_FAILURE;
+				} else if (!strcmp(p, "success")) {
+					lastfield = SUBUNIT_SUCCESS;
+				} else if (!strcmp(p, "skip")) {
+					lastfield = SUBUNIT_SKIP;
+				} else {
+					torture_comment(context, "Invalid subunit field `%s'\n", p);
+					continue;
+				}
+
+				p = q+1;
+
+				name = talloc_strdup(context, p+1);
+
+				q = strrchr(p, '[');
+				if (q != NULL) {
+					*q = '\0';
+					comment = talloc_strdup(context, "");
+				} else {
+					run_subunit_message(context, lastfield, name, NULL);
+					talloc_free(name);
+					name = NULL;
+				}
+			}
+		}
+
+		offset += size-(p-buffer);
+		memcpy(buffer, p, offset);
+	}
+
+	if (name != NULL) {
+		torture_comment(context, "Interrupted during %s\n", name);
+		return false;
+	}
+
+	return true;
+}
diff --git a/source4/torture/ui.c b/source4/torture/ui.c
index c105f4ec9d..32b632a3e8 100644
--- a/source4/torture/ui.c
+++ b/source4/torture/ui.c
@@ -21,6 +21,7 @@
 
 #include "includes.h"
 #include "torture/ui.h"
+#include "torture/torture.h"
 #include "lib/util/dlinklist.h"
 
 void torture_comment(struct torture_context *context, 
@@ -67,7 +68,7 @@ void _torture_skip_ext(struct torture_context *context,
 
 struct torture_suite *torture_suite_create(TALLOC_CTX *ctx, const char *name)
 {
-	struct torture_suite *suite = talloc(ctx, struct torture_suite);
+	struct torture_suite *suite = talloc_zero(ctx, struct torture_suite);
 
 	suite->name = talloc_strdup(suite, name);
 	suite->testcases = NULL;
@@ -146,6 +147,9 @@ BOOL torture_run_suite(struct torture_context *context,
 	if (context->ui_ops->suite_start)
 		context->ui_ops->suite_start(context, suite);
 
+	if (suite->path)
+		torture_subunit_run_suite(context, suite);
+
 	for (tcase = suite->testcases; tcase; tcase = tcase->next) {
 		ret &= torture_run_tcase(context, tcase);
 	}
@@ -162,6 +166,30 @@ BOOL torture_run_suite(struct torture_context *context,
 	return ret;
 }
 
+void torture_ui_test_start(struct torture_context *context,
+							   struct torture_tcase *tcase,
+							   struct torture_test *test)
+{
+	if (context->ui_ops->test_start)
+		context->ui_ops->test_start(context, tcase, test);
+}
+
+void torture_ui_test_result(struct torture_context *context,
+								enum torture_result result,
+								const char *comment)
+{
+	if (context->ui_ops->test_result)
+		context->ui_ops->test_result(context, result, comment);
+
+
+	switch (result) {
+		case TORTURE_SKIP: context->success++; break;
+		case TORTURE_FAIL: context->failed++; break;
+		case TORTURE_TODO: context->todo++; break;
+		case TORTURE_OK: context->success++; break;
+	}
+}
+
 static BOOL internal_torture_run_test(struct torture_context *context, 
 					  struct torture_tcase *tcase,
 					  struct torture_test *test,
@@ -182,8 +210,7 @@ static BOOL internal_torture_run_test(struct torture_context *context,
 	context->active_tcase = tcase;
 	context->active_test = test;
 
-	if (context->ui_ops->test_start)
-		context->ui_ops->test_start(context, tcase, test);
+	torture_ui_test_start(context, tcase, test);
 
 	context->last_reason = NULL;
 	context->last_result = TORTURE_OK;
@@ -195,19 +222,8 @@ static BOOL internal_torture_run_test(struct torture_context *context,
 		context->last_result = TORTURE_FAIL;
 	}
 
-	if (context->ui_ops->test_result)
-		context->ui_ops->test_result(context, 
-									 context->last_result, 
-									 context->last_reason);
-
-
-	switch (context->last_result) {
-		case TORTURE_SKIP: context->success++; break;
-		case TORTURE_FAIL: context->failed++; break;
-		case TORTURE_TODO: context->todo++; break;
-		case TORTURE_OK: context->success++; break;
-	}
-
+	torture_ui_test_result(context, context->last_result, context->last_reason);
+	
 	talloc_free(context->last_reason);
 
 	context->active_test = NULL;
diff --git a/source4/torture/ui.h b/source4/torture/ui.h
index 36457e6a84..b4e3585031 100644
--- a/source4/torture/ui.h
+++ b/source4/torture/ui.h
@@ -52,6 +52,14 @@ struct torture_ui_ops
 						 enum torture_result, const char *reason);
 };
 
+void torture_ui_test_start(struct torture_context *context,
+							   struct torture_tcase *tcase,
+							   struct torture_test *test);
+
+void torture_ui_test_result(struct torture_context *context,
+								enum torture_result result,
+								const char *comment);
+
 /*
  * Holds information about a specific run of the testsuite. 
  * The data in this structure should be considered private to 
@@ -121,6 +129,7 @@ struct torture_tcase {
 struct torture_suite
 {
 	const char *name;
+	const char *path; /* Used by subunit tests only */
 	const char *description;
 	struct torture_tcase *testcases;
 	struct torture_suite *children;