From 2028ada8ca6e1b5840c4f358dc649578759cb36b Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan.bosch@open-xchange.com>
Date: Thu, 30 Jul 2020 23:33:04 +0200
Subject: [PATCH] testsuite: Add support for test cases with expected failure.
---
src/testsuite/testsuite-common.c | 14 +++++++++++++-
src/testsuite/testsuite-common.h | 2 +-
src/testsuite/testsuite.c | 11 +++++++----
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/src/testsuite/testsuite-common.c b/src/testsuite/testsuite-common.c
index f40e34dbd..33434afca 100644
--- a/src/testsuite/testsuite-common.c
+++ b/src/testsuite/testsuite-common.c
@@ -239,8 +239,20 @@ static void testsuite_test_context_deinit(void)
str_free(&test_name);
}
-bool testsuite_testcase_result(void)
+bool testsuite_testcase_result(bool expect_failure)
{
+ if (expect_failure) {
+ if (test_failures < test_index) {
+ printf("\nFAIL: Only %d of %d tests failed "
+ "(all expected to fail).\n\n",
+ test_failures, test_index);
+ return FALSE;
+ }
+
+ printf("\nPASS: %d tests failed.\n\n", test_index);
+ return TRUE;
+ }
+
if (test_failures > 0) {
printf("\nFAIL: %d of %d tests failed.\n\n",
test_failures, test_index);
diff --git a/src/testsuite/testsuite-common.h b/src/testsuite/testsuite-common.h
index b4b3cfd3e..644ce5bbc 100644
--- a/src/testsuite/testsuite-common.h
+++ b/src/testsuite/testsuite-common.h
@@ -169,7 +169,7 @@ void testsuite_test_fail_cstr(const char *reason);
void testsuite_test_succeed(string_t *reason);
void testsuite_testcase_fail(const char *reason);
-bool testsuite_testcase_result(void);
+bool testsuite_testcase_result(bool expect_failure);
/*
* Testsuite temporary directory
diff --git a/src/testsuite/testsuite.c b/src/testsuite/testsuite.c
index d96bd19bd..2be6d2d3b 100644
--- a/src/testsuite/testsuite.c
+++ b/src/testsuite/testsuite.c
@@ -47,7 +47,7 @@ const struct sieve_script_env *testsuite_scriptenv;
static void print_help(void)
{
printf(
-"Usage: testsuite [-D] [-E] [-d <dump-filename>]\n"
+"Usage: testsuite [-D] [-E] [-F] [-d <dump-filename>]\n"
" [-t <trace-filename>] [-T <trace-option>]\n"
" [-P <plugin>] [-x <extensions>]\n"
" <scriptfile>\n"
@@ -86,11 +86,11 @@ int main(int argc, char **argv)
struct sieve_trace_config trace_config;
struct sieve_binary *sbin;
const char *sieve_dir, *cwd, *error;
- bool log_stdout = FALSE;
+ bool log_stdout = FALSE, expect_failure = FALSE;
int ret, c;
sieve_tool = sieve_tool_init("testsuite", &argc, &argv,
- "d:t:T:EDP:", TRUE);
+ "d:t:T:EFDP:", TRUE);
/* Parse arguments */
dumpfile = tracefile = NULL;
@@ -112,6 +112,9 @@ int main(int argc, char **argv)
case 'E':
log_stdout = TRUE;
break;
+ case 'F':
+ expect_failure = TRUE;
+ break;
default:
print_help();
i_fatal_status(EX_USAGE, "Unknown argument: %c", c);
@@ -244,7 +247,7 @@ int main(int argc, char **argv)
sieve_tool_deinit(&sieve_tool);
- if (!testsuite_testcase_result())
+ if (!testsuite_testcase_result(expect_failure))
return EXIT_FAILURE;
return EXIT_SUCCESS;
--
GitLab