From 0a0ab71de49890a9380956f8547a5c936bdfdb45 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 7 Nov 2024 12:19:17 +0000 Subject: [PATCH] Add spec links to test failures --- nmostesting/IS12Utils.py | 2 +- nmostesting/suites/MS0501Test.py | 25 +++++++++++++++----- nmostesting/suites/MS0502Test.py | 40 ++++++++++++++++---------------- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 1b6a0346..c3548f26 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -176,7 +176,7 @@ def send_command(self, test, command_json): context=self.message_type_to_schema_name(parsed_message["messageType"]) + ": ") else: raise NMOSTestException(test.FAIL( - f"Unrecognised message type: {parsed_message.get("messageType")}", + f"Unrecognised message type: {parsed_message.get('messageType')}", f"https://specs.amwa.tv/is-12/branches/{self.apis[CONTROL_API_KEY]['spec_branch']}" "/docs/Protocol_messaging.html#command-message-type")) diff --git a/nmostesting/suites/MS0501Test.py b/nmostesting/suites/MS0501Test.py index 8c07c2ea..f50b4d8e 100644 --- a/nmostesting/suites/MS0501Test.py +++ b/nmostesting/suites/MS0501Test.py @@ -1072,7 +1072,10 @@ def test_ms05_21(self, test): self.do_validate_runtime_constraints_test(test, device_model, class_manager, test_metadata) if test_metadata.error: - return test.FAIL(test_metadata.error_msg) + return test.FAIL(test_metadata.error_msg, + "https://specs.amwa.tv/ms-05-02/branches/" + f"{self.apis[MS05_API_KEY]['spec_branch']}" + "/docs/Constraints.html") if not test_metadata.checked: return test.UNCLEAR("No runtime constraints found.") @@ -1109,7 +1112,10 @@ def test_ms05_22(self, test): self.do_validate_property_constraints_test(test, device_model, class_manager, test_metadata) if test_metadata.error: - return test.FAIL(test_metadata.error_msg) + return test.FAIL(test_metadata.error_msg, + "https://specs.amwa.tv/ms-05-02/branches/" + f"{self.apis[MS05_API_KEY]['spec_branch']}" + "/docs/Constraints.html") if not test_metadata.checked: return test.UNCLEAR("No property constraints found.") @@ -1142,7 +1148,10 @@ def test_ms05_23(self, test): self.do_validate_datatype_constraints_test(test, datatype, test_metadata) if test_metadata.error: - return test.FAIL(test_metadata.error_msg) + return test.FAIL(test_metadata.error_msg, + "https://specs.amwa.tv/ms-05-02/branches/" + f"{self.apis[MS05_API_KEY]['spec_branch']}" + "/docs/Constraints.html") if not test_metadata.checked: return test.UNCLEAR("No datatype constraints found.") @@ -1293,7 +1302,9 @@ def test_ms05_24(self, test): if test_metadata.error: return test.FAIL(test_metadata.error_msg, - "https://specs.amwa.tv/ms-05-02/branches/v1.0.x/docs/Constraints.html") + "https://specs.amwa.tv/ms-05-02/branches/" + f"{self.apis[MS05_API_KEY]['spec_branch']}" + "/docs/Constraints.html") if not test_metadata.checked: return test.UNCLEAR("No constraints hierarchy found.") @@ -1311,7 +1322,8 @@ def test_ms05_25(self, test): if not isinstance(method_result, NcMethodResultError): return test.FAIL("Read only properties error expected.", - f"https://specs.amwa.tv/ms-05-02/branches/{self.apis[MS05_API_KEY]['spec_branch']}" + "https://specs.amwa.tv/ms-05-02/branches/" + f"{self.apis[MS05_API_KEY]['spec_branch']}" "/docs/Framework.html#ncmethodresult") if method_result.status != NcMethodStatus.Readonly.value: @@ -1348,7 +1360,8 @@ def test_ms05_26(self, test): if not isinstance(method_result, NcMethodResultError): return test.FAIL("Sequence out of bounds error expected.", - f"https://specs.amwa.tv/ms-05-02/branches/{self.apis[MS05_API_KEY]['spec_branch']}" + "https://specs.amwa.tv/ms-05-02/branches/" + f"{self.apis[MS05_API_KEY]['spec_branch']}" "/docs/Framework.html#ncmethodresult") if method_result.status != NcMethodStatus.IndexOutOfBounds: diff --git a/nmostesting/suites/MS0502Test.py b/nmostesting/suites/MS0502Test.py index b17969f2..5d8b09cd 100644 --- a/nmostesting/suites/MS0502Test.py +++ b/nmostesting/suites/MS0502Test.py @@ -70,7 +70,7 @@ def __init__(self, apis, utils, **kwargs): def set_up_tests(self): self.ms05_utils.reset() - self.constraints_validation_metadata = MS0502Test.TestMetadata() + self.check_property_metadata = MS0502Test.TestMetadata() self.set_sequence_item_metadata = MS0502Test.TestMetadata() self.add_sequence_item_metadata = MS0502Test.TestMetadata() self.remove_sequence_item_metadata = MS0502Test.TestMetadata() @@ -212,16 +212,16 @@ def _do_check(check_function): # Expecting a parameter constraint violation if not (expect_error ^ isinstance(method_result, NcMethodResultError)): if expect_error: # only set checked if constraints have been violated/tested - self.constraints_validation_metadata.checked = True + self.check_property_metadata.checked = True else: - self.constraints_validation_metadata.error = True + self.check_property_metadata.error = True if expect_error: - self.constraints_validation_metadata.error_msg += \ + self.check_property_metadata.error_msg += \ f"Constraints not enforced for {constrained_property.name}: " \ f"Value: {value} " \ f"Constraints: {constrained_property.constraints}; " else: - self.constraints_validation_metadata.error_msg += \ + self.check_property_metadata.error_msg += \ f"Constraints incorrectly applied for {constrained_property.name}; " def _do_set_sequence(): @@ -315,7 +315,7 @@ def _check_parameter_constraints_string(self, test, constrained_property): self._check_constrained_parameter(test, constrained_property, new_value) def _check_sequence_datatype_type(self, test, property_under_test, original_value): - self.constraints_validation_metadata.checked = True + self.check_property_metadata.checked = True modified_value = list(reversed(original_value)) @@ -327,8 +327,8 @@ def _check_sequence_datatype_type(self, test, property_under_test, original_valu role_path=property_under_test.role_path) if isinstance(method_result, NcMethodResultError): - self.constraints_validation_metadata.error = True - self.constraints_validation_metadata.error_msg += \ + self.check_property_metadata.error = True + self.check_property_metadata.error_msg += \ f"{self.ms05_utils.create_role_path_string(property_under_test.role_path)}: " \ f"Unable to set property {str(property_under_test.descriptor.id)}: " \ f"{str(method_result.errorMessage)} " @@ -362,7 +362,7 @@ def _do_check_property_test(self, test, question, get_constraints=False, get_seq # If non-interactive then test all methods selected_properties = [p["resource"] for p in possible_properties] - self.constraints_validation_metadata = MS0502Test.TestMetadata() + self.check_property_metadata = MS0502Test.TestMetadata() for constrained_property in selected_properties: @@ -373,11 +373,9 @@ def _do_check_property_test(self, test, question, get_constraints=False, get_seq oid=constrained_property.oid, role_path=constrained_property.role_path) if isinstance(method_result, NcMethodResultError): - self.constraints_validation_metadata.error = True - self.constraints_validation_metadata.error_msg += \ - f"{constrained_property.name}: error getting property: " \ - f"{str(method_result.errorMessage)}: constraints {str(constraints)} " - continue + return test.FAIL(f"{constrained_property.name}: error getting value of property: " + f"{str(constrained_property.descriptor.id)}: {str(method_result.errorMessage)}: " + f"constraints {str(constraints)}") original_value = method_result.value try: @@ -399,14 +397,16 @@ def _do_check_property_test(self, test, question, get_constraints=False, get_seq oid=constrained_property.oid, role_path=constrained_property.role_path) if isinstance(method_result, NcMethodResultError): - return test.FAIL(f"{constrained_property.name}: error restoring original value of property: " - f"{str(method_result.errorMessage)} original value: {str(original_value)}" - f": constraints {str(constraints)}") + return test.FAIL(f"{constrained_property.name}: error setting value of property: " + f"{str(constrained_property.descriptor.id)}: {str(method_result.errorMessage)}: " + f"original value: {str(original_value)}: " + f"constraints {str(constraints)}") - if self.constraints_validation_metadata.error: - return test.FAIL(self.constraints_validation_metadata.error_msg) + if self.check_property_metadata.error: + # JRT add link to constraints spec + return test.FAIL(self.check_property_metadata.error_msg) - if self.constraints_validation_metadata.checked: + if get_constraints and self.check_property_metadata.checked: return test.PASS() return test.UNCLEAR("No properties of this type checked")