From 8773e743c518578584d07d35ffdafdd598af88b0 Mon Sep 17 00:00:00 2001 From: Jelmer Vernooij Date: Mon, 16 Oct 2006 13:06:41 +0000 Subject: r19339: Merge my 4.0-unittest branch. This adds an API for more fine-grained output in the testsuite rather than just True or False for a set of tests. The aim is to use this for: * known failure lists (run all tests and detect tests that started working or started failing). This would allow us to get rid of the RPC-SAMBA3-* tests * nicer torture output * simplification of the testsuite system * compatibility with other unit testing systems * easier usage of smbtorture (being able to run one test and automatically set up the environment for that) This is still a work-in-progress; expect more updates over the next couple of days. (This used to be commit 0eb6097305776325c75081356309115f445a7218) --- source4/torture/ui.h | 230 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 167 insertions(+), 63 deletions(-) (limited to 'source4/torture/ui.h') diff --git a/source4/torture/ui.h b/source4/torture/ui.h index 006e2b99ef..36457e6a84 100644 --- a/source4/torture/ui.h +++ b/source4/torture/ui.h @@ -34,6 +34,10 @@ enum torture_result { TORTURE_SKIP=3 }; +/* + * These callbacks should be implemented by any backend that wishes + * to listen to reports from the torture tests. + */ struct torture_ui_ops { void (*comment) (struct torture_context *, const char *); @@ -44,10 +48,20 @@ struct torture_ui_ops void (*test_start) (struct torture_context *, struct torture_tcase *, struct torture_test *); - void (*test_result) (struct torture_context *, enum torture_result, - const char *reason); + void (*test_result) (struct torture_context *, + enum torture_result, const char *reason); }; +/* + * Holds information about a specific run of the testsuite. + * The data in this structure should be considered private to + * the torture tests and should only be used directly by the torture + * code and the ui backends. + * + * Torture tests should instead call the torture_*() macros and functions + * specified below. + */ + struct torture_context { const struct torture_ui_ops *ui_ops; @@ -61,124 +75,214 @@ struct torture_context int success; int failed; + bool quiet; /* Whether tests should avoid writing output to stdout */ + enum torture_result last_result; char *last_reason; char *outputdir; + int level; +}; + +/* + * Describes a particular torture test + */ +struct torture_test { + const char *name; + const char *description; + bool dangerous; + /* Function to call to run this test */ + bool (*run) (struct torture_context *torture_ctx, + struct torture_tcase *tcase, + struct torture_test *test); + + struct torture_test *prev, *next; + + /* Pointer to the actual test function. This is run by the + * run() function above. */ + void *fn; + const void *data; +}; + +/* + * Describes a particular test case. + */ +struct torture_tcase { + const char *name; + const char *description; + bool (*setup) (struct torture_context *tcase, void **data); + bool (*teardown) (struct torture_context *tcase, void *data); + bool fixture_persistent; + void *data; + struct torture_test *tests; + struct torture_tcase *prev, *next; }; struct torture_suite { const char *name; const char *description; - struct torture_tcase { - const char *name; - const char *description; - BOOL (*setup) (struct torture_context *tcase, void **data); - BOOL (*teardown) (struct torture_context *tcase, void *data); - BOOL fixture_persistent; - const void *data; - struct torture_test { - const char *name; - const char *description; - const void *data; - BOOL dangerous; - BOOL (*run) (struct torture_context *test, - const void *tcase_data, - const void *test_data); - struct torture_test *prev, *next; - } *tests; - struct torture_tcase *prev, *next; - } *testcases; + struct torture_tcase *testcases; + struct torture_suite *children; + + /* Pointers to siblings of this torture suite */ + struct torture_suite *prev, *next; }; -struct torture_suite *torture_suite_create(TALLOC_CTX *ctx, const char *name); +/** Create a new torture suite */ +struct torture_suite *torture_suite_create(TALLOC_CTX *mem_ctx, + const char *name); + +/** Change the setup and teardown functions for a testcase */ void torture_tcase_set_fixture(struct torture_tcase *tcase, - BOOL (*setup) (struct torture_context *, void **), - BOOL (*teardown) (struct torture_context *, void *)); + bool (*setup) (struct torture_context *, void **), + bool (*teardown) (struct torture_context *, void *)); + +/* Add another test to run for a particular testcase */ struct torture_test *torture_tcase_add_test(struct torture_tcase *tcase, const char *name, - BOOL (*run) (struct torture_context *test, const void *tcase_data, + bool (*run) (struct torture_context *test, const void *tcase_data, const void *test_data), const void *test_data); + +/* Add a testcase to a testsuite */ struct torture_tcase *torture_suite_add_tcase(struct torture_suite *suite, const char *name); + +/* Convenience wrapper that adds a testcase against only one + * test will be run */ struct torture_tcase *torture_suite_add_simple_tcase( struct torture_suite *suite, const char *name, - BOOL (*run) (struct torture_context *test, const void *test_data), + bool (*run) (struct torture_context *test, const void *test_data), const void *data); -BOOL torture_run_suite(struct torture_context *context, +/* Convenience wrapper that adds a test that doesn't need any + * testcase data */ +struct torture_tcase *torture_suite_add_simple_test( + struct torture_suite *suite, + const char *name, + bool (*run) (struct torture_context *test)); + +/* Add a child testsuite to an existing testsuite */ +bool torture_suite_add_suite(struct torture_suite *suite, + struct torture_suite *child); + +/* Run the specified testsuite recursively */ +bool torture_run_suite(struct torture_context *context, struct torture_suite *suite); -BOOL torture_run_tcase(struct torture_context *context, +/* Run the specified testcase */ +bool torture_run_tcase(struct torture_context *context, struct torture_tcase *tcase); -BOOL torture_run_test(struct torture_context *context, +/* Run the specified test */ +bool torture_run_test(struct torture_context *context, struct torture_tcase *tcase, struct torture_test *test); -#define torture_assert(ctx,expr,string) \ +void _torture_fail_ext(struct torture_context *test, const char *reason, ...) PRINTF_ATTRIBUTE(2,3); +void torture_comment(struct torture_context *test, const char *comment, ...) PRINTF_ATTRIBUTE(2,3); +void _torture_skip_ext(struct torture_context *test, const char *reason, ...) PRINTF_ATTRIBUTE(2,3); + +#define torture_assert(torture_ctx,expr,cmt) \ if (!(expr)) { \ - torture_fail(ctx, "%s:%d (%s): %s", __FILE__, __LINE__, string, \ - __STRING(expr)); \ - return False; \ + torture_comment(torture_ctx, __location__": Expression `%s' failed\n", __STRING(expr)); \ + _torture_fail_ext(torture_ctx, __location__": %s", cmt); \ + return false; \ } -#define torture_assert_werr_equal(ctx,got,expected,string) \ +#define torture_assert_werr_equal(torture_ctx, got, expected, cmt) \ do { WERROR __got = got, __expected = expected; \ if (!W_ERROR_EQUAL(__got, __expected)) { \ - torture_fail(ctx, "%s:%d (%s): got %s, expected %s", __FILE__, \ - __LINE__, string, win_errstr(__got), win_errstr(__expected)); \ - return False; \ + torture_comment(torture_ctx, __location__": "#got" was %s, expected %s\n", \ + win_errstr(__got), win_errstr(__expected)); \ + _torture_fail_ext(torture_ctx, __location__": %s", cmt); \ + return false; \ } \ } while (0) -#define torture_assert_ntstatus_equal(ctx,got,expected,string) \ +#define torture_assert_ntstatus_equal(torture_ctx,got,expected,cmt) \ do { NTSTATUS __got = got, __expected = expected; \ if (!NT_STATUS_EQUAL(__got, __expected)) { \ - torture_fail(ctx, "%s:%d (%s): got %s, expected %s", __FILE__, \ - __LINE__, string, nt_errstr(__got), nt_errstr(__expected)); \ - return False; \ + torture_comment(torture_ctx, __location__": "#got" was %s, expected %s\n", \ + nt_errstr(__got), nt_errstr(__expected)); \ + _torture_fail_ext(torture_ctx, __location__": %s", cmt); \ + return false; \ }\ } while(0) -#define torture_assert_casestr_equal(ctx,got,expected,string) \ - do { const char *__got = got, *__expected = expected; \ - if (strcasecmp(__got, __expected) != 0) { \ - torture_fail(ctx, "%s:%d (%s): got %s, expected %s", __FILE__, \ - __LINE__, string, got, expected); \ - return False; \ +#define torture_assert_casestr_equal(torture_ctx,got,expected,cmt) \ + do { const char *__got = (got), *__expected = (expected); \ + if (!strequal(__got, __expected)) { \ + torture_comment(torture_ctx, __location__": "#got" was %s, expected %s\n", __got, __expected); \ + _torture_fail_ext(torture_ctx, __location__": %s", cmt); \ + return false; \ } \ } while(0) -#define torture_assert_str_equal(ctx,got,expected,string) \ - do { const char *__got = got, *__expected = expected; \ - if (strcmp(__got, __expected) != 0) { \ - torture_fail(ctx, "%s:%d (%s): got %s, expected %s", __FILE__, \ - __LINE__, string, __got, __expected); \ - return False; \ +#define torture_assert_str_equal(torture_ctx,got,expected,cmt)\ + do { const char *__got = (got), *__expected = (expected); \ + if (strcmp_safe(__got, __expected) != 0) { \ + torture_comment(torture_ctx, __location__": "#got" was %s, expected %s\n", __got, __expected); \ + _torture_fail_ext(torture_ctx, __location__": %s", cmt); \ + return false; \ } \ } while(0) +#define torture_assert_int_equal(torture_ctx,got,expected,cmt)\ + do { int __got = (got), __expected = (expected); \ + if (__got != __expected) { \ + torture_comment(torture_ctx, __location__": "#got" was %d, expected %d\n", __got, __expected); \ + _torture_fail_ext(torture_ctx, __location__": %s", cmt); \ + return false; \ + } \ + } while(0) + +#define torture_assert_errno_equal(torture_ctx,expected,cmt)\ + do { int __expected = (expected); \ + if (errno != __expected) { \ + torture_comment(torture_ctx, __location__": errno was %d, expected %s\n", errno, strerror(__expected)); \ + _torture_fail_ext(torture_ctx, __location__": %s", cmt); \ + return false; \ + } \ + } while(0) + + + +#define torture_skip(torture_ctx,cmt) do {\ + _torture_skip_ext(torture_ctx, __location__": %s", cmt);\ + return true; \ + } while(0) +#define torture_fail(torture_ctx,cmt) do {\ + _torture_fail_ext(torture_ctx, __location__": %s", cmt);\ + return false; \ + } while (0) + +#define torture_out stderr /* Convenience macros */ +#define torture_assert_ntstatus_ok(torture_ctx,expr,cmt) \ + torture_assert_ntstatus_equal(torture_ctx,expr,NT_STATUS_OK,cmt) -#define torture_assert_ntstatus_ok(ctx,expr,string) \ - torture_assert_ntstatus_equal(ctx,expr,NT_STATUS_OK,string) +#define torture_assert_werr_ok(torture_ctx,expr,cmt) \ + torture_assert_werr_equal(torture_ctx,expr,WERR_OK,cmt) -#define torture_assert_werr_ok(ctx,expr,string) \ - torture_assert_werr_equal(ctx,expr,WERR_OK,string) +/* Getting settings */ +const char *torture_setting_string(struct torture_context *test, \ + const char *name, + const char *default_value); -void torture_comment(struct torture_context *test, const char *comment, ...) PRINTF_ATTRIBUTE(2,3); -void torture_fail(struct torture_context *test, const char *reason, ...) PRINTF_ATTRIBUTE(2,3); -void torture_skip(struct torture_context *test, const char *reason, ...) PRINTF_ATTRIBUTE(2,3); -const char *torture_setting(struct torture_context *test, const char *name, - const char *default_value); +int torture_setting_int(struct torture_context *test, + const char *name, + int default_value); + +bool torture_setting_bool(struct torture_context *test, + const char *name, + bool default_value); /* Helper function commonly used */ -BOOL torture_teardown_free(struct torture_context *torture, void *data); +bool torture_teardown_free(struct torture_context *torture, void *data); #endif /* __TORTURE_UI_H__ */ -- cgit