GD-242: Add runtime argument to skip the entire test-suite or test-ca…

…se (#243)

# Why
We want to skip a test-case or entire test-suite

# What
Add new test arguments
* do_skip as runtime expression
* skip_reason as String

```gdscript
func test_case1(do_skip=true, skip_reason="do not run this"):
	pass
```
* fix cmd_tool to report skipped test-suites
* remove `skip` function from TestSuite class
* handle skipped test-shuite on executor
* extends test-suite scanner to handle the new skip arguments

addons/gdUnit4/bin/GdUnitCmdTool.gd

@@ -314,6 +314,7 @@ class CLIRunner extends Node:
 					event.is_failed(),
 					event.is_warning(),
 					event.is_skipped(),
+					event.skipped_count(),
 					event.failed_count(),
 					event.orphan_nodes(),
 					event.reports())

addons/gdUnit4/src/GdUnitTestSuite.gd

@@ -17,6 +17,11 @@ extends Node
 
 const NO_ARG = GdUnitConstants.NO_ARG
 
+### internal runtime variables that must not be overwritten!!!
+var __is_skipped := false
+var __skip_reason :String = "Unknow."
+
+
 ## This function is called before a test suite starts[br]
 ## You can overwrite to prepare test data or initalizize necessary variables
 func before() -> void:
@@ -41,19 +46,10 @@ func after_test() -> void:
 	pass
 
 
-## Skip the test-suite from execution, it will be ignored
-func skip(skipped :bool) -> void:
-	set_meta("gd_skipped", skipped)
-
-
 func is_failure(_expected_failure :String = NO_ARG) -> bool:
 	return Engine.get_meta("GD_TEST_FAILURE") if Engine.has_meta("GD_TEST_FAILURE") else false
 
 
-func is_skipped() -> bool:
-	return get_meta("gd_skipped") if has_meta("gd_skipped") else false
-
-
 var __active_test_case :String
 func set_active_test_case(test_case :String) -> void:
 	__active_test_case = test_case

addons/gdUnit4/src/asserts/GdAssertMessages.gd

@@ -156,8 +156,21 @@ static func test_timeout(timeout :int) -> String:
 	return "%s\n %s" % [_error("Timeout !"), _colored_value("Test timed out after %s" %  LocalTime.elapsed(timeout))]
 
 
+static func test_suite_skipped(hint :String, skip_count) -> String:
+	return """
+		%s
+		  Tests skipped: %s
+		  Reason: %s
+		""".dedent().trim_prefix("\n")\
+		% [_error("Entire test-suite is skipped!"), _colored_value(skip_count), _colored_value(hint)]
+
+
 static func test_skipped(hint :String) -> String:
-	return "%s\n %s" % [_error("This test is skipped!"), "Reason: %s" % _colored_value(hint)]
+	return """
+		%s
+		  Reason: %s
+		""".dedent().trim_prefix("\n")\
+		% [_error("This test is skipped!"), _colored_value(hint)]
 
 
 static func error_not_implemented() -> String:

addons/gdUnit4/src/core/GdUnitExecutor.gd

@@ -55,6 +55,7 @@ func fire_event(event :GdUnitEvent) -> void:
 
 
 func fire_test_skipped(test_suite :GdUnitTestSuite, test_case :_TestCase):
+	set_stage(STAGE_TEST_CASE_BEFORE)
 	fire_event(GdUnitEvent.new()\
 		.test_before(test_suite.get_script().resource_path, test_suite.get_name(), test_case.get_name()))
 	var statistics = {
@@ -68,32 +69,53 @@ func fire_test_skipped(test_suite :GdUnitTestSuite, test_case :_TestCase):
 		GdUnitEvent.SKIPPED: true,
 		GdUnitEvent.SKIPPED_COUNT: 1,
 	}
+	set_stage(STAGE_TEST_CASE_AFTER)
 	var report := GdUnitReport.new().create(GdUnitReport.SKIPPED, test_case.line_number(), GdAssertMessages.test_skipped(test_case.skip_info()))
 	fire_event(GdUnitEvent.new()\
 		.test_after(test_suite.get_script().resource_path, test_suite.get_name(), test_case.get_name(), statistics, [report]))
 
 
-func suite_before(test_suite :GdUnitTestSuite, total_count :int):
+func fire_test_suite_skipped(test_suite :GdUnitTestSuite):
+	var skip_count := test_suite.get_child_count()
 	set_stage(STAGE_TEST_SUITE_BEFORE)
 	fire_event(GdUnitEvent.new()\
-		.suite_before(test_suite.get_script().resource_path, test_suite.get_name(), total_count))
+		.suite_before(test_suite.get_script().resource_path, test_suite.get_name(), skip_count))
+	var statistics = {
+		GdUnitEvent.ORPHAN_NODES: 0,
+		GdUnitEvent.ELAPSED_TIME: 0,
+		GdUnitEvent.WARNINGS: false,
+		GdUnitEvent.ERRORS: false,
+		GdUnitEvent.ERROR_COUNT: 0,
+		GdUnitEvent.FAILED: false,
+		GdUnitEvent.FAILED_COUNT: 0,
+		GdUnitEvent.SKIPPED_COUNT: skip_count,
+		GdUnitEvent.SKIPPED: true
+	}
+	set_stage(STAGE_TEST_SUITE_AFTER)
+	var report := GdUnitReport.new().create(GdUnitReport.SKIPPED, -1, GdAssertMessages.test_suite_skipped(test_suite.__skip_reason, skip_count))
+	fire_event(GdUnitEvent.new().suite_after(test_suite.get_script().resource_path, test_suite.get_name(), statistics, [report]))
+
+
+func suite_before(test_suite :GdUnitTestSuite):
+	set_stage(STAGE_TEST_SUITE_BEFORE)
+	fire_event(GdUnitEvent.new()\
+		.suite_before(test_suite.get_script().resource_path, test_suite.get_name(), test_suite.get_child_count()))
 	_testsuite_timer = LocalTime.now()
 	_total_test_errors = 0
 	_total_test_failed = 0
 	_total_test_warnings = 0
-	if not test_suite.is_skipped():
-		_memory_pool.set_pool(test_suite, GdUnitMemoryPool.POOL.TESTSUITE, true)
-		@warning_ignore("redundant_await")
-		await test_suite.before()
-		_memory_pool.monitor_stop()
+	_memory_pool.set_pool(test_suite, GdUnitMemoryPool.POOL.TESTSUITE, true)
+	@warning_ignore("redundant_await")
+	await test_suite.before()
+	_memory_pool.monitor_stop()
 
 
 func suite_after(test_suite :GdUnitTestSuite):
 	set_stage(STAGE_TEST_SUITE_AFTER)
 	GdUnitTools.clear_tmp()
 
 	var is_warning := _total_test_warnings != 0
-	var is_skipped := test_suite.is_skipped()
+	var is_skipped := test_suite.__is_skipped
 	var skip_count := test_suite.get_child_count()
 	var orphan_nodes := 0
 	var reports := _report_collector.get_reports(STAGE_TEST_SUITE_BEFORE)
@@ -295,9 +317,9 @@ func Execute(test_suite :GdUnitTestSuite) -> void:
 		ExecutionCompleted.emit()
 		return
 	var ts := test_suite
-	await suite_before(ts, ts.get_child_count())
-
-	if not ts.is_skipped() and ts.get_child_count() != 0:
+	if not ts.__is_skipped and ts.get_child_count() != 0:
+		await suite_before(ts)
+		
 		for test_case_index in ts.get_child_count():
 			var test_case := ts.get_child(test_case_index) as _TestCase
 			# only iterate over test case, we need to filter because of possible adding other child types checked before() or before_test()
@@ -322,10 +344,12 @@ func Execute(test_suite :GdUnitTestSuite) -> void:
 				# we delete the current test suite where is execute the current test case to kill the function state
 				# and replace it by a clone without function state
 				ts = await clone_test_suite(ts)
+
+		await suite_after(ts)
 	else:
+		fire_test_suite_skipped(ts)
 		# needs at least one yielding otherwise the waiting function is blocked
 		await get_tree().process_frame
-	await suite_after(ts)
 	ts.free()
 	context.clear()
 	ExecutionCompleted.emit()

addons/gdUnit4/src/core/GdUnitTestSuiteScanner.gd

@@ -10,6 +10,7 @@ func test_${func_name}() -> void:
 
 var _script_parser := GdScriptParser.new()
 var _extends_test_suite_classes := Array()
+var regex_replace_class_name := GdUnitTools.to_regex("(?m)^class_name .*$")
 
 
 func scan_testsuite_classes() -> void:
@@ -114,50 +115,96 @@ static func parse_test_suite_name(script :Script) -> String:
 	return script.resource_path.get_file().replace(".gd", "")
 
 
+func _handle_test_suite_arguments(test_suite, script :GDScript, fd :GdFunctionDescriptor):
+	for arg in fd.args():
+		match arg.name():
+			_TestCase.ARGUMENT_SKIP:
+				var result = _run_expression(script, arg.value_as_string())
+				if result is bool:
+					test_suite.__is_skipped = result
+				else:
+					push_error("Test expression '%s' cannot be evaluated because it is not of type bool!" % arg.value_as_string())
+			_TestCase.ARGUMENT_SKIP_REASON:
+				test_suite.__skip_reason = arg.value_as_string()
+			_:
+				push_error("Unsuported argument `%s` found on before() at '%s'!" % [arg.name(), script.resource_path])
+
+
+func _handle_test_case_arguments(test_suite, script :GDScript, fd :GdFunctionDescriptor):
+	var timeout := _TestCase.DEFAULT_TIMEOUT
+	var iterations := Fuzzer.ITERATION_DEFAULT_COUNT
+	var seed_value := -1
+	var is_skipped := false
+	var skip_reason := "Unknown."
+	var fuzzers :Array[GdFunctionArgument] = []
+	var test := _TestCase.new()
+
+	for arg in fd.args():
+		# verify argument is allowed
+		# is test using fuzzers?
+		if arg.type() == GdObjects.TYPE_FUZZER:
+			fuzzers.append(arg)
+		elif arg.has_default():
+			match arg.name():
+				_TestCase.ARGUMENT_TIMEOUT:
+					timeout = arg.default()
+				_TestCase.ARGUMENT_SKIP:
+					var result = _run_expression(script, arg.value_as_string())
+					if result is bool:
+						is_skipped = result
+					else:
+						push_error("Test expression '%s' cannot be evaluated because it is not of type bool!" % arg.value_as_string())
+				_TestCase.ARGUMENT_SKIP_REASON:
+					skip_reason = arg.value_as_string()
+				Fuzzer.ARGUMENT_ITERATIONS:
+					iterations = arg.default()
+				Fuzzer.ARGUMENT_SEED:
+					seed_value = arg.default()
+	# create new test
+	test.configure(fd.name(), fd.line_number(), script.resource_path, timeout, fuzzers, iterations, seed_value)
+	test.skip(is_skipped, skip_reason)
+	_validate_argument(fd, test)
+	test_suite.add_child(test)
+	# is parameterized test?
+	if fd.is_parameterized():
+		var test_paramaters := GdTestParameterSet.extract_test_parameters(test_suite.get_script(), fd)
+		var error := GdTestParameterSet.validate(fd.args(), test_paramaters)
+		if not error.is_empty():
+			test.skip(true, error)
+		test.set_test_parameters(test_paramaters)
+
+
 func _parse_and_add_test_cases(test_suite, script :GDScript, test_case_names :PackedStringArray):
 	var test_cases_to_find = Array(test_case_names)
+	var functions_to_scan := test_case_names
+	functions_to_scan.append("before")
 	var source := _script_parser.load_source_code(script, [script.resource_path])
-	var functions = _script_parser.parse_functions(source, "", [script.resource_path], test_case_names)
-	for function in functions:
-		var fd :GdFunctionDescriptor = function
+	var function_descriptors := _script_parser.parse_functions(source, "", [script.resource_path], functions_to_scan)
+	for fd in function_descriptors:
+		if fd.name() == "before":
+			_handle_test_suite_arguments(test_suite, script, fd)
 		if test_cases_to_find.has(fd.name()):
-			var timeout := _TestCase.DEFAULT_TIMEOUT
-			var iterations := Fuzzer.ITERATION_DEFAULT_COUNT
-			var seed_value := -1
-			var fuzzers :Array[GdFunctionArgument] = []
-			var test := _TestCase.new()
-
-			_validate_argument(fd, test)
-			for arg in fd.args():
-				var fa := arg as GdFunctionArgument
-				# verify argument is allowed
-				# is test using fuzzers?
-				if fa.type() == GdObjects.TYPE_FUZZER:
-					fuzzers.append(fa)
-				elif fa.has_default():
-					match fa.name():
-						_TestCase.ARGUMENT_TIMEOUT:
-							timeout = fa.default()
-						Fuzzer.ARGUMENT_ITERATIONS:
-							iterations = fa.default()
-						Fuzzer.ARGUMENT_SEED:
-							seed_value = fa.default()
-						#_:
-						#	push_error("Invalid test case arguemnt found. ", fa)
-					continue
-			# create new test
-			test.configure(fd.name(), fd.line_number(), script.resource_path, timeout, fuzzers, iterations, seed_value)
-			test_suite.add_child(test)
-			# is parameterized test?
-			if fd.is_parameterized():
-				var test_paramaters := GdTestParameterSet.extract_test_parameters(test_suite.get_script(), fd)
-				var error := GdTestParameterSet.validate(fd.args(), test_paramaters)
-				if not error.is_empty():
-					test.skip(true, error)
-				test.set_test_parameters(test_paramaters)
-
-
-const TEST_CASE_ARGUMENTS = [_TestCase.ARGUMENT_TIMEOUT, Fuzzer.ARGUMENT_ITERATIONS, Fuzzer.ARGUMENT_SEED]
+			_handle_test_case_arguments(test_suite, script, fd)
+
+
+func _run_expression(src_script :GDScript, expression :String) -> Variant:
+	var script := GDScript.new()
+	script.source_code = _remove_class_name(src_script.source_code)
+	script.source_code += """
+		func __run_expression() -> Variant:
+			return $expression
+		""".dedent().replace("$expression", expression)
+	script.reload(false)
+	var runner := script.new()
+	runner.queue_free()
+	return runner.__run_expression()
+
+
+func _remove_class_name(source_code :String) -> String:
+	return regex_replace_class_name.sub(source_code, "")
+
+
+const TEST_CASE_ARGUMENTS = [_TestCase.ARGUMENT_TIMEOUT, _TestCase.ARGUMENT_SKIP, _TestCase.ARGUMENT_SKIP_REASON, Fuzzer.ARGUMENT_ITERATIONS, Fuzzer.ARGUMENT_SEED]
 
 func _validate_argument(fd :GdFunctionDescriptor, test_case :_TestCase) -> void:
 	if fd.is_parameterized():

addons/gdUnit4/src/core/_TestCase.gd

@@ -6,6 +6,8 @@ signal completed()
 # default timeout 5min
 const DEFAULT_TIMEOUT := -1
 const ARGUMENT_TIMEOUT := "timeout"
+const ARGUMENT_SKIP := "do_skip"
+const ARGUMENT_SKIP_REASON := "skip_reason"
 
 var _iterations: int = 1
 var _current_iteration :int = -1
@@ -16,7 +18,7 @@ var _test_param_index := -1
 var _line_number: int = -1
 var _script_path: String
 var _skipped := false
-var _skip_info := ""
+var _skip_reason := ""
 var _expect_to_interupt := false
 var _timer : Timer
 var _interupted :bool = false
@@ -157,7 +159,8 @@ func report() -> GdUnitReport:
 
 
 func skip_info() -> String:
-	return _skip_info
+	return _skip_reason
+
 
 func line_number() -> int:
 	return _line_number
@@ -196,9 +199,9 @@ func generate_seed() -> void:
 		seed(_seed)
 
 
-func skip(skipped :bool, error :String = "") -> void:
+func skip(skipped :bool, reason :String = "") -> void:
 	_skipped = skipped
-	_skip_info = error
+	_skip_reason = reason
 
 
 func set_test_parameters(p_test_parameters :Array) -> void:

addons/gdUnit4/src/core/parse/GdScriptParser.gd

@@ -565,7 +565,7 @@ func _parse_end_function(input: String, remove_trailing_char := false) -> String
 			"(": bracket_count += 1
 			")":
 				bracket_count -= 1
-				if bracket_count <= 0 and in_array <= 0:
+				if bracket_count < 0 and in_array <= 0:
 					end_of_func = true
 			",":
 				if bracket_count == 0 and in_array == 0:
@@ -664,8 +664,8 @@ func parse_func_name(row :String) -> String:
 	return token._token
 
 
-func parse_functions(rows :PackedStringArray, clazz_name :String, clazz_path :PackedStringArray, included_functions :PackedStringArray = PackedStringArray()) -> Array:
-	var func_descriptors := Array()
+func parse_functions(rows :PackedStringArray, clazz_name :String, clazz_path :PackedStringArray, included_functions := PackedStringArray()) -> Array[GdFunctionDescriptor]:
+	var func_descriptors :Array[GdFunctionDescriptor] = []
 	for rowIndex in rows.size():
 		var row = rows[rowIndex]
 		# step over inner class functions

addons/gdUnit4/src/report/GdUnitHtmlReport.gd

@@ -10,6 +10,7 @@ var _iteration :int
 func _init(path :String):
 	_iteration = GdUnitTools.find_last_path_index(path, REPORT_DIR_PREFIX) + 1
 	_report_path = "%s/%s%d" % [path, REPORT_DIR_PREFIX, _iteration]
+	DirAccess.make_dir_recursive_absolute(_report_path)
 
 
 func add_testsuite_report(suite_report :GdUnitTestSuiteReport):
@@ -29,6 +30,7 @@ func update_test_suite_report(
 	is_failed: bool,
 	is_warning :bool,
 	is_skipped :bool,
+	skipped_count :int,
 	failed_count :int,
 	orphan_count :int,
 	reports :Array = []) -> void:
@@ -39,6 +41,8 @@ func update_test_suite_report(
 			report.set_failed(is_failed, failed_count)
 			report.set_orphans(orphan_count)
 			report.set_reports(reports)
+	if is_skipped:
+		_skipped_count = skipped_count
 
 
 func update_testcase_report(resource_path :String, test_report :GdUnitTestCaseReport):