r19381: Print out the specific tests that failed after a smbtorture run.

Support listing known failures as a list of wildcards in the file
`KNOWN_FAILURES'.
(This used to be commit 23f66efd564d1ad549fc0cd60348f54808f5cafa)

5 files changed, 143 insertions, 28 deletions

diff --git a/source4/torture/smbtorture.c b/source4/torture/smbtorture.c
index 5740829d7d..fdd44ddc08 100644
--- a/source4/torture/smbtorture.c
+++ b/source4/torture/smbtorture.c
@@ -53,8 +53,7 @@ static bool run_matching(struct torture_context *torture,
 				continue;
 			}
 
-			ret &= run_matching(torture, 
-								o->name, expr, o, matched);
+			ret &= run_matching(torture, o->name, expr, o, matched);
 		}
 	} else {
 		char *name;
@@ -435,6 +434,7 @@ const static struct torture_ui_ops quiet_ui_ops = {
 	int max_runtime=0;
 	int argc_new;
 	struct torture_context *torture;
+	const struct torture_ui_ops *ui_ops;
 	char **argv_new;
 	poptContext pc;
 	static const char *target = "other";
@@ -581,39 +581,73 @@ const static struct torture_ui_ops quiet_ui_ops = {
 		lp_set_cmdline("torture:binding", binding);
 	}
 
-	torture = talloc_zero(talloc_autofree_context(), struct torture_context);
 	if (!strcmp(ui_ops_name, "simple")) {
-		torture->ui_ops = &std_ui_ops;
+		ui_ops = &std_ui_ops;
 	} else if (!strcmp(ui_ops_name, "subunit")) {
-		torture->ui_ops = &subunit_ui_ops;
+		ui_ops = &subunit_ui_ops;
 	} else if (!strcmp(ui_ops_name, "harness")) {
-		torture->ui_ops = &harness_ui_ops;
+		ui_ops = &harness_ui_ops;
 	} else if (!strcmp(ui_ops_name, "quiet")) {
-		torture->ui_ops = &quiet_ui_ops;
+		ui_ops = &quiet_ui_ops;
 	} else {
 		printf("Unknown output format '%s'\n", ui_ops_name);
 		exit(1);
 	}
 
+	torture = torture_context_init(talloc_autofree_context(), "KNOWN_FAILURES", 
+						 ui_ops);
+
 	if (argc_new == 0) {
 		printf("You must specify a test to run, or 'ALL'\n");
 	} else {
 		int total;
 		double rate;
+		int unexpected_failures;
 		for (i=2;i<argc_new;i++) {
 			if (!run_test(torture, argv_new[i])) {
 				correct = false;
 			}
 		}
 
-		total = torture->skipped+torture->success+torture->failed;
+
+		unexpected_failures = str_list_length(torture->results.unexpected_failures);
+
+		total = torture->results.skipped+torture->results.success+torture->results.failed+torture->results.errors;
 		if (total == 0) {
 			printf("No tests run.\n");
 		} else {
-			rate = ((total - torture->failed) * (100.0 / total));
-			printf("Tests: %d, Errors: %d, Skipped: %d. Success rate: %.2f%%\n",
-			   total, torture->failed, torture->skipped,
-			   rate);
+			rate = ((total - unexpected_failures - torture->results.errors) * (100.0 / total));
+		
+			printf("Tests: %d, Failures: %d", total, torture->results.failed);
+			if (torture->results.failed - unexpected_failures) {
+				printf(" (%d expected)", torture->results.failed - unexpected_failures);
+			}
+			printf(", Errors: %d, Skipped: %d. Success rate: %.2f%%\n",
+			   torture->results.errors, torture->results.skipped, rate);
+		}
+
+		if (unexpected_failures) {
+			printf("The following tests failed:\n");
+			for (i = 0; torture->results.unexpected_failures[i]; i++) {
+				printf("  %s\n", torture->results.unexpected_failures[i]);
+			}
+			printf("\n");
+		}
+
+		if (str_list_length(torture->results.unexpected_errors)) {
+			printf("Errors occurred while running the following tests:\n");
+			for (i = 0; torture->results.unexpected_errors[i]; i++) {
+				printf("  %s\n", torture->results.unexpected_errors[i]);
+			}
+			printf("\n");
+		}
+
+		if (str_list_length(torture->results.unexpected_successes)) {
+			printf("The following tests were expected to fail but succeeded:\n");
+			for (i = 0; torture->results.unexpected_successes[i]; i++) {
+				printf("  %s\n", torture->results.unexpected_successes[i]);
+			}
+			printf("\n");
 		}
 	}
 
diff --git a/source4/torture/subunit.c b/source4/torture/subunit.c
index 3c4790b706..bed1972079 100644
--- a/source4/torture/subunit.c
+++ b/source4/torture/subunit.c
@@ -190,12 +190,12 @@ bool torture_subunit_run_suite(struct torture_context *context,
 		return false;
 
 	if (waitpid(pid, &status, 0) == -1) {
-		torture_comment(context, "waitpid(%d) failed\n", pid);
+		torture_result(context, TORTURE_ERROR, "waitpid(%d) failed\n", pid);
 		return false;
 	}
 
 	if (WEXITSTATUS(status) != 0) {
-		torture_comment(context, "failed with status %d\n", WEXITSTATUS(status));
+		torture_result(context, TORTURE_ERROR, "failed with status %d\n", WEXITSTATUS(status));
 		return false;
 	}
 
@@ -258,7 +258,7 @@ bool torture_subunit_run_suite(struct torture_context *context,
 	}
 
 	if (name != NULL) {
-		torture_comment(context, "Interrupted during %s\n", name);
+		torture_result(context, TORTURE_ERROR, "Interrupted during %s\n", name);
 		return false;
 	}
 
diff --git a/source4/torture/torture.c b/source4/torture/torture.c
index f83a7330c2..e6629eb214 100644
--- a/source4/torture/torture.c
+++ b/source4/torture/torture.c
@@ -43,6 +43,21 @@ bool torture_register_suite(struct torture_suite *suite)
 	return torture_suite_add_suite(torture_root, suite);
 }
 
+struct torture_context *torture_context_init(TALLOC_CTX *mem_ctx, 
+											 const char *known_failures_file, 
+											 const struct torture_ui_ops *ui_ops)
+{
+	int numlines;
+	struct torture_context *torture = talloc_zero(mem_ctx, 
+												  struct torture_context);
+	torture->expected_failures = file_lines_load(known_failures_file, &numlines,
+												 torture);
+	torture->ui_ops = ui_ops;
+
+	return torture;
+}
+
+
 int torture_init(void)
 {
 	init_module_fn static_init[] = STATIC_torture_MODULES;
diff --git a/source4/torture/ui.c b/source4/torture/ui.c
index 670b151686..1e404d55ad 100644
--- a/source4/torture/ui.c
+++ b/source4/torture/ui.c
@@ -128,11 +128,19 @@ BOOL torture_run_suite(struct torture_context *context,
 	BOOL ret = True;
 	struct torture_tcase *tcase;
 	struct torture_suite *tsuite;
+	char *old_testname;
 
 	context->level++;
 	if (context->ui_ops->suite_start)
 		context->ui_ops->suite_start(context, suite);
 
+	old_testname = context->active_testname;
+	if (context->active_testname)
+		context->active_testname = talloc_asprintf(context, "%s-%s", 
+											   old_testname, suite->name);
+	else
+		context->active_testname = talloc_strdup(context, suite->name);
+
 	if (suite->path)
 		torture_subunit_run_suite(context, suite);
 
@@ -144,6 +152,9 @@ BOOL torture_run_suite(struct torture_context *context,
 		ret &= torture_run_suite(context, tsuite);
 	}
 
+	talloc_free(context->active_testname);
+	context->active_testname = old_testname;
+
 	if (context->ui_ops->suite_finish)
 		context->ui_ops->suite_finish(context, suite);
 
@@ -160,6 +171,19 @@ void torture_ui_test_start(struct torture_context *context,
 		context->ui_ops->test_start(context, tcase, test);
 }
 
+int str_list_match(const char *name, char **list)
+{
+	int i, ret = 0;
+	if (list == NULL)
+		return 0;
+
+	for (i = 0; list[i]; i++) {
+		if (gen_fnmatch(list[i], name) == 0)
+			ret++;
+	}
+	return ret;
+}
+
 void torture_ui_test_result(struct torture_context *context,
 								enum torture_result result,
 								const char *comment)
@@ -167,13 +191,29 @@ void torture_ui_test_result(struct torture_context *context,
 	if (context->ui_ops->test_result)
 		context->ui_ops->test_result(context, result, comment);
 
-	/* FIXME: */
-
-	switch (result) {
-		case TORTURE_SKIP: context->skipped++; break;
-		case TORTURE_ERROR: context->errors++; break;
-		case TORTURE_FAIL: context->failed++; break;
-		case TORTURE_OK: context->success++; break;
+	if (result == TORTURE_SKIP) {
+		context->results.skipped++;
+	} else if (result == TORTURE_OK) {
+		if (str_list_match(context->active_testname, 
+						   context->expected_failures)) {
+			context->results.unexpected_successes = str_list_add(
+					context->results.unexpected_successes, 
+					talloc_reference(context, context->active_testname));
+		} 
+		context->results.success++;
+	} else if (result == TORTURE_ERROR) {
+		context->results.unexpected_errors = str_list_add(
+					context->results.unexpected_errors, 
+					talloc_reference(context, context->active_testname));
+		context->results.errors++;
+	} else if (result == TORTURE_FAIL) {
+		if (0 == str_list_match(context->active_testname, 
+						   context->expected_failures)) {
+			context->results.unexpected_failures = str_list_add(
+					context->results.unexpected_failures, 
+					talloc_reference(context, context->active_testname));
+		} 
+		context->results.failed++;
 	}
 }
 
@@ -183,6 +223,7 @@ static BOOL internal_torture_run_test(struct torture_context *context,
 					  BOOL already_setup)
 {
 	BOOL ret;
+	char *old_testname;
 
 	if (test->dangerous && !torture_setting_bool(context, "dangerous", False)) {
 		torture_result(context, TORTURE_SKIP,
@@ -194,16 +235,22 @@ static BOOL internal_torture_run_test(struct torture_context *context,
 		!tcase->setup(context, &(tcase->data)))
 		return False;
 
+	if (tcase == NULL || strcmp(test->name, tcase->name) != 0) { 
+		old_testname = context->active_testname;
+		context->active_testname = talloc_asprintf(context, "%s-%s", 
+											   old_testname, test->name);
+	}
 	context->active_tcase = tcase;
 	context->active_test = test;
 
 	torture_ui_test_start(context, tcase, test);
 
+
 	context->last_reason = NULL;
 	context->last_result = TORTURE_OK;
 
 	ret = test->run(context, tcase, test);
-	if (!ret) {
+	if (!ret && context->last_result == TORTURE_OK) {
 		if (context->last_reason == NULL)
 			context->last_reason = talloc_strdup(context, "Unknown error/failure");
 		context->last_result = TORTURE_ERROR;
@@ -213,6 +260,10 @@ static BOOL internal_torture_run_test(struct torture_context *context,
 	
 	talloc_free(context->last_reason);
 
+	if (tcase == NULL || strcmp(test->name, tcase->name) != 0) { 
+		talloc_free(context->active_testname);
+		context->active_testname = old_testname;
+	}
 	context->active_test = NULL;
 	context->active_tcase = NULL;
 
@@ -226,6 +277,7 @@ BOOL torture_run_tcase(struct torture_context *context,
 					   struct torture_tcase *tcase)
 {
 	BOOL ret = True;
+	char *old_testname;
 	struct torture_test *test;
 
 	context->level++;
@@ -240,10 +292,15 @@ BOOL torture_run_tcase(struct torture_context *context,
 		goto done;
 	}
 
+	old_testname = context->active_testname;
+	context->active_testname = talloc_asprintf(context, "%s-%s", 
+											   old_testname, tcase->name);
 	for (test = tcase->tests; test; test = test->next) {
 		ret &= internal_torture_run_test(context, tcase, test, 
 				tcase->fixture_persistent);
 	}
+	talloc_free(context->active_testname);
+	context->active_testname = old_testname;
 
 	if (tcase->fixture_persistent && tcase->teardown &&
 		!tcase->teardown(context, tcase->data))
diff --git a/source4/torture/ui.h b/source4/torture/ui.h
index f0628b7758..23988d9bf4 100644
--- a/source4/torture/ui.h
+++ b/source4/torture/ui.h
@@ -75,14 +75,23 @@ struct torture_context
 	const struct torture_ui_ops *ui_ops;
 	void *ui_data;
 
+	char *active_testname;
 	struct torture_test *active_test;
 	struct torture_tcase *active_tcase;
 
-	int skipped;
-	int todo;
-	int success;
-	int failed;
-	int errors;
+	char **expected_failures;
+
+	struct torture_results {
+		int skipped;
+		int todo;
+		int success;
+		int failed;
+		int errors;
+
+		const char **unexpected_failures;
+		const char **unexpected_successes;
+		const char **unexpected_errors;
+	} results;
 
 	bool quiet; /* Whether tests should avoid writing output to stdout */