Add a way to skip a test (#1012)

* add a way to skip ctest * update fail condition * update the skip macro definition * more comments and clang-format
awslabs · Apr 14, 2023 · ffb14f9 · ffb14f9
1 parent cfaa904
commit ffb14f9
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 2 deletions.
diff --git a/cmake/AwsTestHarness.cmake b/cmake/AwsTestHarness.cmake
@@ -10,6 +10,10 @@ option(ENABLE_NET_TESTS "Run tests requiring an internet connection." ON)
 define_property(GLOBAL PROPERTY AWS_TEST_CASES BRIEF_DOCS "Test Cases" FULL_DOCS "Test Cases")
 set(AWS_TEST_CASES "" CACHE INTERNAL "Test cases valid for this configuration")
 
+# The return value for the skipped test cases. Refer to the return code defined in aws_test_harness.h:
+# #define SKIP (1)
+set(SKIP_RETURN_CODE_VALUE 1)
+
 # Registers a test case by name (the first argument to the AWS_TEST_CASE macro in aws_test_harness.h)
 macro(add_test_case name)
     list(APPEND TEST_CASES "${name}")
@@ -87,6 +91,7 @@ function(generate_cpp_test_driver driver_exe_name)
 
     foreach(name IN LISTS TEST_CASES)
         add_test(${name} ${driver_exe_name} "${name}")
+        set_tests_properties("${name}" PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE_VALUE})
     endforeach()
 
     # Clear test cases in case another driver needsto be generated

diff --git a/include/aws/testing/aws_test_harness.h b/include/aws/testing/aws_test_harness.h
@@ -16,6 +16,11 @@
 #include <stdlib.h>
 #include <string.h>
 
+/**
+ * The return code for skipped tests. Use the return code if the test should be skipped.
+ */
+#define AWS_OP_SKIP (-2)
+
 #ifndef AWS_UNSTABLE_TESTING_API
 #    error The AWS Test Fixture is designed only for use by AWS owned libraries for the AWS C99 SDK. You are welcome to use it,   \
 but you should be aware we make no promises on the stability of this API.  To enable use of the aws test fixtures, set \
@@ -62,9 +67,26 @@ static int s_cunit_failure_message0(
     s_cunit_failure_message0(FAIL_PREFIX, func, file, line, format, #__VA_ARGS__)
 
 static int total_failures;
+static int total_skip;
 
 #define SUCCESS (0)
 #define FAILURE (-1)
+/* The skip code returned to ctest to indicate the test is skipped. Refer to cmake doc:
+ * https://cmake.org/cmake/help/latest/prop_test/SKIP_RETURN_CODE.html */
+#define SKIP (1)
+
+#define POSTSKIP_INTERNAL()                                                                                            \
+    do {                                                                                                               \
+        total_skip++;                                                                                                  \
+        return SKIP;                                                                                                   \
+    } while (0)
+
+#define RETURN_SKIP(format, ...)                                                                                       \
+    do {                                                                                                               \
+        printf(format, ##__VA_ARGS__);                                                                                 \
+        printf("\n");                                                                                                  \
+        POSTSKIP_INTERNAL();                                                                                           \
+    } while (0)
 
 #define RETURN_SUCCESS(format, ...)                                                                                    \
     do {                                                                                                               \
@@ -434,7 +456,7 @@ static inline int s_aws_run_test_case(struct aws_test_harness *harness) {
         test_res |= harness->on_after(allocator, setup_res, harness->ctx);
     }
 
-    if (test_res != AWS_OP_SUCCESS) {
+    if (test_res != AWS_OP_SUCCESS && test_res != AWS_OP_SKIP) {
         goto fail;
     }
 
@@ -456,7 +478,12 @@ static inline int s_aws_run_test_case(struct aws_test_harness *harness) {
     aws_logger_set(NULL);
     aws_logger_clean_up(&err_logger);
 
-    RETURN_SUCCESS("%s [ \033[32mOK\033[0m ]", harness->test_name);
+    if (test_res == AWS_OP_SUCCESS) {
+        RETURN_SUCCESS("%s [ \033[32mOK\033[0m ]", harness->test_name);
+    } else if (test_res == AWS_OP_SKIP) {
+        RETURN_SKIP("%s [ \033[32mSKIP\033[0m ]", harness->test_name);
+    }
+
 fail:
     FAIL("%s [ \033[31mFAILED\033[0m ]", harness->test_name);
 }