Skip to content

Commit

Permalink
Add spec links to test failures
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonathan-r-thorpe
jonathan-r-thorpe committed Nov 7, 2024
1 parent d989b4b commit 0a0ab71
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 27 deletions.
2 changes: 1 addition & 1 deletion nmostesting/IS12Utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def send_command(self, test, command_json):
context=self.message_type_to_schema_name(parsed_message["messageType"]) + ": ")
else:
raise NMOSTestException(test.FAIL(
f"Unrecognised message type: {parsed_message.get("messageType")}",
f"Unrecognised message type: {parsed_message.get('messageType')}",
f"https://specs.amwa.tv/is-12/branches/{self.apis[CONTROL_API_KEY]['spec_branch']}"
"/docs/Protocol_messaging.html#command-message-type"))

Expand Down
25 changes: 19 additions & 6 deletions nmostesting/suites/MS0501Test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1072,7 +1072,10 @@ def test_ms05_21(self, test):
self.do_validate_runtime_constraints_test(test, device_model, class_manager, test_metadata)

if test_metadata.error:
return test.FAIL(test_metadata.error_msg)
return test.FAIL(test_metadata.error_msg,
"https://specs.amwa.tv/ms-05-02/branches/"
f"{self.apis[MS05_API_KEY]['spec_branch']}"
"/docs/Constraints.html")

if not test_metadata.checked:
return test.UNCLEAR("No runtime constraints found.")
Expand Down Expand Up @@ -1109,7 +1112,10 @@ def test_ms05_22(self, test):
self.do_validate_property_constraints_test(test, device_model, class_manager, test_metadata)

if test_metadata.error:
return test.FAIL(test_metadata.error_msg)
return test.FAIL(test_metadata.error_msg,
"https://specs.amwa.tv/ms-05-02/branches/"
f"{self.apis[MS05_API_KEY]['spec_branch']}"
"/docs/Constraints.html")

if not test_metadata.checked:
return test.UNCLEAR("No property constraints found.")
Expand Down Expand Up @@ -1142,7 +1148,10 @@ def test_ms05_23(self, test):
self.do_validate_datatype_constraints_test(test, datatype, test_metadata)

if test_metadata.error:
return test.FAIL(test_metadata.error_msg)
return test.FAIL(test_metadata.error_msg,
"https://specs.amwa.tv/ms-05-02/branches/"
f"{self.apis[MS05_API_KEY]['spec_branch']}"
"/docs/Constraints.html")

if not test_metadata.checked:
return test.UNCLEAR("No datatype constraints found.")
Expand Down Expand Up @@ -1293,7 +1302,9 @@ def test_ms05_24(self, test):

if test_metadata.error:
return test.FAIL(test_metadata.error_msg,
"https://specs.amwa.tv/ms-05-02/branches/v1.0.x/docs/Constraints.html")
"https://specs.amwa.tv/ms-05-02/branches/"
f"{self.apis[MS05_API_KEY]['spec_branch']}"
"/docs/Constraints.html")

if not test_metadata.checked:
return test.UNCLEAR("No constraints hierarchy found.")
Expand All @@ -1311,7 +1322,8 @@ def test_ms05_25(self, test):

if not isinstance(method_result, NcMethodResultError):
return test.FAIL("Read only properties error expected.",
f"https://specs.amwa.tv/ms-05-02/branches/{self.apis[MS05_API_KEY]['spec_branch']}"
"https://specs.amwa.tv/ms-05-02/branches/"
f"{self.apis[MS05_API_KEY]['spec_branch']}"
"/docs/Framework.html#ncmethodresult")

if method_result.status != NcMethodStatus.Readonly.value:
Expand Down Expand Up @@ -1348,7 +1360,8 @@ def test_ms05_26(self, test):

if not isinstance(method_result, NcMethodResultError):
return test.FAIL("Sequence out of bounds error expected.",
f"https://specs.amwa.tv/ms-05-02/branches/{self.apis[MS05_API_KEY]['spec_branch']}"
"https://specs.amwa.tv/ms-05-02/branches/"
f"{self.apis[MS05_API_KEY]['spec_branch']}"
"/docs/Framework.html#ncmethodresult")

if method_result.status != NcMethodStatus.IndexOutOfBounds:
Expand Down
40 changes: 20 additions & 20 deletions nmostesting/suites/MS0502Test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def __init__(self, apis, utils, **kwargs):

def set_up_tests(self):
self.ms05_utils.reset()
self.constraints_validation_metadata = MS0502Test.TestMetadata()
self.check_property_metadata = MS0502Test.TestMetadata()
self.set_sequence_item_metadata = MS0502Test.TestMetadata()
self.add_sequence_item_metadata = MS0502Test.TestMetadata()
self.remove_sequence_item_metadata = MS0502Test.TestMetadata()
Expand Down Expand Up @@ -212,16 +212,16 @@ def _do_check(check_function):
# Expecting a parameter constraint violation
if not (expect_error ^ isinstance(method_result, NcMethodResultError)):
if expect_error: # only set checked if constraints have been violated/tested
self.constraints_validation_metadata.checked = True
self.check_property_metadata.checked = True
else:
self.constraints_validation_metadata.error = True
self.check_property_metadata.error = True
if expect_error:
self.constraints_validation_metadata.error_msg += \
self.check_property_metadata.error_msg += \
f"Constraints not enforced for {constrained_property.name}: " \
f"Value: {value} " \
f"Constraints: {constrained_property.constraints}; "
else:
self.constraints_validation_metadata.error_msg += \
self.check_property_metadata.error_msg += \
f"Constraints incorrectly applied for {constrained_property.name}; "

def _do_set_sequence():
Expand Down Expand Up @@ -315,7 +315,7 @@ def _check_parameter_constraints_string(self, test, constrained_property):
self._check_constrained_parameter(test, constrained_property, new_value)

def _check_sequence_datatype_type(self, test, property_under_test, original_value):
self.constraints_validation_metadata.checked = True
self.check_property_metadata.checked = True

modified_value = list(reversed(original_value))

Expand All @@ -327,8 +327,8 @@ def _check_sequence_datatype_type(self, test, property_under_test, original_valu
role_path=property_under_test.role_path)

if isinstance(method_result, NcMethodResultError):
self.constraints_validation_metadata.error = True
self.constraints_validation_metadata.error_msg += \
self.check_property_metadata.error = True
self.check_property_metadata.error_msg += \
f"{self.ms05_utils.create_role_path_string(property_under_test.role_path)}: " \
f"Unable to set property {str(property_under_test.descriptor.id)}: " \
f"{str(method_result.errorMessage)} "
Expand Down Expand Up @@ -362,7 +362,7 @@ def _do_check_property_test(self, test, question, get_constraints=False, get_seq
# If non-interactive then test all methods
selected_properties = [p["resource"] for p in possible_properties]

self.constraints_validation_metadata = MS0502Test.TestMetadata()
self.check_property_metadata = MS0502Test.TestMetadata()

for constrained_property in selected_properties:

Expand All @@ -373,11 +373,9 @@ def _do_check_property_test(self, test, question, get_constraints=False, get_seq
oid=constrained_property.oid,
role_path=constrained_property.role_path)
if isinstance(method_result, NcMethodResultError):
self.constraints_validation_metadata.error = True
self.constraints_validation_metadata.error_msg += \
f"{constrained_property.name}: error getting property: " \
f"{str(method_result.errorMessage)}: constraints {str(constraints)} "
continue
return test.FAIL(f"{constrained_property.name}: error getting value of property: "
f"{str(constrained_property.descriptor.id)}: {str(method_result.errorMessage)}: "
f"constraints {str(constraints)}")

original_value = method_result.value
try:
Expand All @@ -399,14 +397,16 @@ def _do_check_property_test(self, test, question, get_constraints=False, get_seq
oid=constrained_property.oid,
role_path=constrained_property.role_path)
if isinstance(method_result, NcMethodResultError):
return test.FAIL(f"{constrained_property.name}: error restoring original value of property: "
f"{str(method_result.errorMessage)} original value: {str(original_value)}"
f": constraints {str(constraints)}")
return test.FAIL(f"{constrained_property.name}: error setting value of property: "
f"{str(constrained_property.descriptor.id)}: {str(method_result.errorMessage)}: "
f"original value: {str(original_value)}: "
f"constraints {str(constraints)}")

if self.constraints_validation_metadata.error:
return test.FAIL(self.constraints_validation_metadata.error_msg)
if self.check_property_metadata.error:
# JRT add link to constraints spec
return test.FAIL(self.check_property_metadata.error_msg)

if self.constraints_validation_metadata.checked:
if get_constraints and self.check_property_metadata.checked:
return test.PASS()

return test.UNCLEAR("No properties of this type checked")
Expand Down

0 comments on commit 0a0ab71

Please sign in to comment.