From 190287659cb92126b7687af666cbae15693de599 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Tue, 28 Nov 2023 15:49:55 -0500 Subject: [PATCH 01/15] IS-13: add basic API tests Successfully tested against: https://github.com/garethsb/nmos-cpp/commit/04ac4fd0893828a3d197b1c6c12f7c70e8990dda https://github.com/AMWA-TV/is-13/commit/8655e0ce019341a21fc607c9d91ccea9b8a0db7b --- README.md | 1 + nmostesting/Config.py | 11 ++++++++++ nmostesting/NMOSTesting.py | 9 ++++++++ nmostesting/suites/IS1301Test.py | 35 ++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 nmostesting/suites/IS1301Test.py diff --git a/README.md b/README.md index 01bed2c7..febf3e81 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ The following test suites are currently supported. | IS-09-01 | IS-09 System API | | (X) | | System Parameters Server | | IS-09-02 | IS-09 System API Discovery | X | | | | | IS-10-01 | IS-10 Authorization API | | | | Authorization Server | +| IS-13-01 | IS-13 Annotation API | X | | | | | - | BCP-002-01 Natural Grouping | X | | | Included in IS-04 Node API suite | | - | BCP-002-02 Asset Distinguishing Information | X | | | Included in IS-04 Node API suite | | BCP-003-01 | BCP-003-01 Secure Communication | X | X | | See [Testing TLS](docs/2.2.%20Usage%20-%20Testing%20BCP-003-01%20TLS.md) | diff --git a/nmostesting/Config.py b/nmostesting/Config.py index c663ec91..0a2240b3 100644 --- a/nmostesting/Config.py +++ b/nmostesting/Config.py @@ -280,6 +280,17 @@ } } }, + "is-13": { + "repo": "is-13", + "versions": ["v1.0"], + "default_version": "v1.0", + "apis": { + "annotation": { + "name": "Annotation API", + "raml": "AnnotationAPI.raml" + } + } + }, "bcp-002-01": { "repo": "bcp-002-01", "versions": ["v1.0"], diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index e4860827..b5dee403 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -82,6 +82,7 @@ from .suites import IS0901Test from .suites import IS0902Test # from .suites import IS1001Test +from .suites import IS1301Test from .suites import BCP00301Test from .suites import BCP0060101Test from .suites import BCP0060102Test @@ -340,6 +341,14 @@ # }], # "class": IS1001Test.IS1001Test # }, + "IS-13-01": { + "name": "IS-13 Annotation API", + "specs": [{ + "spec_key": "is-13", + "api_key": "annotation" + }], + "class": IS1301Test.IS1301Test, + }, "BCP-003-01": { "name": "BCP-003-01 Secure Communication", "specs": [{ diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py new file mode 100644 index 00000000..e66756b8 --- /dev/null +++ b/nmostesting/suites/IS1301Test.py @@ -0,0 +1,35 @@ +# Copyright (C) 2023 Advanced Media Workflow Association +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..GenericTest import GenericTest, NMOSTestException +from ..TestHelper import compare_json + +ANNOTATION_API_KEY = "annotation" + + +class IS1301Test(GenericTest): + """ + Runs IS-13-Test + """ + def __init__(self, apis, **kwargs): + GenericTest.__init__(self, apis, **kwargs) + self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] + + def test_01(self, test): + """ 1st annotation test """ + + if compare_json({}, {}): + return test.PASS() + else: + return test.FAIL("IO Resource does not correctly reflect the API resources") From 8a4c1bf2150c4ab32310bda33ee95af4d91cd369 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Wed, 29 Nov 2023 10:05:08 -0500 Subject: [PATCH 02/15] Output more explicit error messages When possible, TestResult.detail should be displayed in stderr since the raised exception prevents from the report creation. --- nmostesting/GenericTest.py | 2 +- nmostesting/NMOSTesting.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nmostesting/GenericTest.py b/nmostesting/GenericTest.py index 951eb1fa..ecb73131 100644 --- a/nmostesting/GenericTest.py +++ b/nmostesting/GenericTest.py @@ -671,7 +671,7 @@ def check_api_resource(self, test, resource, response_code, api, path): schema = self.get_schema(api, resource[1]["method"], resource[0], response.status_code) if not schema: - raise NMOSTestException(test.MANUAL("Test suite unable to locate schema")) + raise NMOSTestException(test.MANUAL(f"Test suite unable to locate schema for resource:{resource}")) return self.check_response(schema, resource[1]["method"], response) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index b5dee403..39906492 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -49,7 +49,7 @@ from .DNS import DNS from .GenericTest import NMOSInitException from . import ControllerTest -from .TestResult import TestStates +from .TestResult import TestStates, TestResult from .TestHelper import get_default_ip from .NMOSUtils import DEFAULT_ARGS from .CRL import CRL, CRL_API @@ -632,7 +632,7 @@ def run_tests(test, endpoints, test_selection=["all"]): try: result = test_obj.run_tests(test_selection) except Exception as ex: - print(" * ERROR: {}".format(ex)) + print(" * ERROR while running {}: {}".format(test_selection, ex)) raise ex finally: core_app.config['TEST_ACTIVE'] = False @@ -970,7 +970,8 @@ def run_noninteractive_tests(args): else: exit_code = print_test_results(results, endpoints, args) except Exception as e: - print(" * ERROR: {}".format(str(e))) + print(" * ERROR raw: {}".format(e.args)) + print(" * ERROR in non-interactive tests: {}".format(str(e) if not isinstance(e.args[0], TestResult) else e.args[0].detail)) exit_code = ExitCodes.ERROR return exit_code From 9d9241235b937a043609bd8423041ce5271f94b2 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 22 Jan 2024 17:41:57 -0500 Subject: [PATCH 03/15] IS-13: add tests for self & devices resources For each resource&objects: - Read initial value and store - Reset default value, check timestamp and store - Write max-length and check value+timestamp - Write >max-length and check value+timestamp - Reset default value and compare - Restore initial value Apply to objects: label, description, tags TODO: check IS04 --- nmostesting/suites/IS1301Test.py | 184 ++++++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 5 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index e66756b8..d8478662 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -17,6 +17,27 @@ ANNOTATION_API_KEY = "annotation" +from ..import TestHelper +import re +import copy + +ANNOTATION_API_KEY = "annotation" + +RESOURCES = ["self", "devices", "senders", "receivers"] +OBJECTS = ["label", "description", "tags"] + +# const for label&description-related tests +STRING_OVER_MAX_VALUE = ''.join(['X' for i in range(100)]) +STRING_MAX_VALUE = STRING_OVER_MAX_VALUE[:64] # this is the max length tolerated + +# const for tags-related tests +TAGS_OVER_MAX_VALUE = {'location': ['underground'], 'studio': ['42'], 'tech': ['John', 'Mike']} +TAGS_MAX_VALUE = TAGS_OVER_MAX_VALUE.copy() +TAGS_OVER_MAX_VALUE.pop('tech') # must have a max of 5 + +def get_ts_from_version(version): + """ Convert the 'version' object (string) into float """ + return float(re.sub(':', '.', version)) class IS1301Test(GenericTest): """ @@ -26,10 +47,163 @@ def __init__(self, apis, **kwargs): GenericTest.__init__(self, apis, **kwargs) self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] - def test_01(self, test): - """ 1st annotation test """ + def set_up_tests(self): + """ + FAKE_ORIG = { + 'description': 'fake_orig_desc', + 'label': 'fake_orig_label', + 'tags': { + 'location': ['fake_location'] + } + } + for resource in RESOURCES: + url = "{}{}{}".format(self.annotation_url, 'node/', resource) + TestHelper.do_request("PATCH", url, json=FAKE_ORIG) + """ + pass + + def get_resource(self, url): + """ Get a resource """ + valid, r = self.do_request("GET", url) + if valid and r.status_code == 200: + try: + return True, r.json() + except Exception as e: + return False, e.msg + else: + return False, "GET Resquest FAIL" + + def set_resource(self, url, new, prev): + """ Patch a resource with one ore several object values """ + object = list(new.keys())[0] + + valid, r = TestHelper.do_request("PATCH", url, json=new) + if valid and r.status_code == 200: + try: + resp = r.json() + except Exception as e: + return False, e.msg + else: + return False, "PATCH max Resquest FAIL" + + if get_ts_from_version(prev['version']) >= get_ts_from_version(resp['version']): + return False, "new version FAIL" + + if new[object] is not None: # NOT a reset + if object == "tags" and not TestHelper.compare_json(resp[object], new[object]): + return False, f"new {object} FAIL" + elif resp[object] != new[object]: + return False, f"new {object} FAIL" + + # TODO this is reflected in IS04 + + return True, resp + + def log(self, msg): + print(msg) + + def do_test(self, test, resource, object): + """ + Perform the test sequence for a resource: + + - Read initial value and store + - Reset default value, check timestamp and store + - Write max-length and check value+timestamp + - Write >max-length and check value+timestamp + - Reset default value and compare + - Restore initial value + """ - if compare_json({}, {}): - return test.PASS() + url = "{}{}{}".format(self.annotation_url, 'node/', resource) + if resource != "self": # get first of the list of devices, receivers, senders + valid, r = self.get_resource(url) + if valid: + print(f" Possible endpoint: {r}") + index = r[0] + url = "{}{}{}".format(url, '/', index) + else: + return test.FAIL(f"Can't find any {resource}") + + msg = "save initial" + valid, r = self.get_resource(url) + if valid: + prev = r + initial = copy.copy(r) + initial.pop('id') + initial.pop('version') + self.log(f" {msg}: {r}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "reset to default and save" + valid, r = self.set_resource(url, {object: None}, prev) + if valid: + default = prev = r + self.log(f" {msg}: {r}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "set max value and expected complete response" + value = TAGS_MAX_VALUE if object == "tags" else STRING_MAX_VALUE + valid, r = self.set_resource(url, {object: value}, prev) + if valid: + prev = r + self.log(f" {msg}: {r}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "set >max value and expect truncated response" + value = TAGS_OVER_MAX_VALUE if object == "tags" else STRING_OVER_MAX_VALUE + valid, r = self.set_resource(url, {object: value}, prev) + if valid: + prev = r + self.log(f" {msg}: {r}") + if object == "tags" and not TestHelper.compare_json(r[object], TAGS_MAX_VALUE): + return test.FAIL(f"Can't {msg} {resource}/{object}") + elif r[object] != STRING_MAX_VALUE: + return test.FAIL(f"Can't {msg} {resource}/{object}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "reset again and compare" + valid, r = self.set_resource(url, {object: None}, prev) + if valid: + self.log(f" {msg}: {r}") + if object == "tags" and not TestHelper.compare_json(default[object], r[object]): + return test.FAIL("Second reset give a different default.") + elif default[object] != r[object]: + return test.FAIL("Second reset give a different default.") + prev = r else: - return test.FAIL("IO Resource does not correctly reflect the API resources") + return test.FAIL(f"Can't {msg} {resource}/{object}") + + # restore initial for courtesy + self.set_resource(url, initial, prev) + + return test.PASS() + + def test_01_01(self, test): + """ Annotation test: self/label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "self", "label") + + def test_01_02(self, test): + """ Annotation test: self/description (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "self", "description") + + def test_01_03(self, test): + """ Annotation test: self/tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "self", "tags") + + def test_02_01(self, test): + """ Annotation test: devices/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "devices", "label") + + def test_02_02(self, test): + """Annotation test: devices/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "devices", "description") + + def test_02_03(self, test): + """Annotation test: devices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "devices", "tags") + +# TODO add receivers + senders From 9c3f07886792a0f1440cb5a3f0254064c4d703a7 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 4 Mar 2024 17:28:31 -0500 Subject: [PATCH 04/15] IS-13: test consistency with IS-04 --- nmostesting/NMOSTesting.py | 4 ++ nmostesting/suites/IS1301Test.py | 88 ++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index 39906492..bce92215 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -346,6 +346,10 @@ "specs": [{ "spec_key": "is-13", "api_key": "annotation" + }, { + "spec_key": "is-04", + "api_key": "node", + "disable_fields": ["host", "port"] }], "class": IS1301Test.IS1301Test, }, diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index d8478662..fd2e6313 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -15,13 +15,12 @@ from ..GenericTest import GenericTest, NMOSTestException from ..TestHelper import compare_json -ANNOTATION_API_KEY = "annotation" - from ..import TestHelper import re import copy ANNOTATION_API_KEY = "annotation" +NODE_API_KEY = "node" RESOURCES = ["self", "devices", "senders", "receivers"] OBJECTS = ["label", "description", "tags"] @@ -46,6 +45,7 @@ class IS1301Test(GenericTest): def __init__(self, apis, **kwargs): GenericTest.__init__(self, apis, **kwargs) self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] + self.node_url = f"{self.apis[ANNOTATION_API_KEY]['base_url']}/x-nmos/node/{self.apis[NODE_API_KEY]['version']}/" def set_up_tests(self): """ @@ -73,35 +73,61 @@ def get_resource(self, url): else: return False, "GET Resquest FAIL" - def set_resource(self, url, new, prev): + def set_resource(self, url, node_url, new, prev): """ Patch a resource with one ore several object values """ object = list(new.keys())[0] - valid, r = TestHelper.do_request("PATCH", url, json=new) - if valid and r.status_code == 200: - try: - resp = r.json() - except Exception as e: - return False, e.msg - else: - return False, "PATCH max Resquest FAIL" + valid, resp = TestHelper.do_request("PATCH", url, json=new) + if not valid: + return False, "PATCH Resquest FAIL" + + valid, resp = self.get_resource(url) + if not valid: + return False, "Get Resquest FAIL" + # check that the version (timestamp) has increased if get_ts_from_version(prev['version']) >= get_ts_from_version(resp['version']): return False, "new version FAIL" - + # check PATCH == GET if new[object] is not None: # NOT a reset if object == "tags" and not TestHelper.compare_json(resp[object], new[object]): return False, f"new {object} FAIL" elif resp[object] != new[object]: return False, f"new {object} FAIL" - # TODO this is reflected in IS04 + # validate that it is reflected in IS04 + valid, node_resp = self.get_resource(node_url) + if not valid: + return False, "GET node FAIL" + if new[object] is not None: # NOT a reset + if object == "tags" and not TestHelper.compare_json(node_resp[object], new[object]): + return False, f"new node/.../{object} FAIL" + elif node_resp[object] != new[object]: + return False, f"new node/.../{object} FAIL" + if get_ts_from_version(node_resp['version']) != get_ts_from_version(resp['version']): + return False, "new node version FAIL" return True, resp def log(self, msg): print(msg) + def get_url(self, base_url, resource): + url = f"{base_url}{resource}" + if resource != "self": # get first of the list of devices, receivers, senders + valid, r = self.get_resource(url) + if valid: + if isinstance(r[0], str): # in annotation api + index = r[0] + elif isinstance(r[0], dict): # in node api + index = r[0]['id'] + else: + return None + url = f"{url}/{index}" + else: + return None + return url + def do_test(self, test, resource, object): """ Perform the test sequence for a resource: @@ -114,17 +140,15 @@ def do_test(self, test, resource, object): - Restore initial value """ - url = "{}{}{}".format(self.annotation_url, 'node/', resource) - if resource != "self": # get first of the list of devices, receivers, senders - valid, r = self.get_resource(url) - if valid: - print(f" Possible endpoint: {r}") - index = r[0] - url = "{}{}{}".format(url, '/', index) - else: - return test.FAIL(f"Can't find any {resource}") + url = self.get_url(f"{self.annotation_url}node/", resource) + if not url: + return test.FAIL(f"Can't get annotation url for {resource}") + + node_url = self.get_url(self.node_url, resource) + if not url: + return test.FAIL(f"Can't get node url for {resource}") - msg = "save initial" + msg = "SAVE initial" valid, r = self.get_resource(url) if valid: prev = r @@ -135,26 +159,26 @@ def do_test(self, test, resource, object): else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "reset to default and save" - valid, r = self.set_resource(url, {object: None}, prev) + msg = "RESET to default and save" + valid, r = self.set_resource(url, node_url, {object: None}, prev) if valid: default = prev = r self.log(f" {msg}: {r}") else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "set max value and expected complete response" + msg = "SET MAX value and expected complete response" value = TAGS_MAX_VALUE if object == "tags" else STRING_MAX_VALUE - valid, r = self.set_resource(url, {object: value}, prev) + valid, r = self.set_resource(url, node_url, {object: value}, prev) if valid: prev = r self.log(f" {msg}: {r}") else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "set >max value and expect truncated response" + msg = "SET >MAX value and expect truncated response" value = TAGS_OVER_MAX_VALUE if object == "tags" else STRING_OVER_MAX_VALUE - valid, r = self.set_resource(url, {object: value}, prev) + valid, r = self.set_resource(url, node_url, {object: value}, prev) if valid: prev = r self.log(f" {msg}: {r}") @@ -165,8 +189,8 @@ def do_test(self, test, resource, object): else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "reset again and compare" - valid, r = self.set_resource(url, {object: None}, prev) + msg = "RESET again and compare" + valid, r = self.set_resource(url, node_url, {object: None}, prev) if valid: self.log(f" {msg}: {r}") if object == "tags" and not TestHelper.compare_json(default[object], r[object]): @@ -178,7 +202,7 @@ def do_test(self, test, resource, object): return test.FAIL(f"Can't {msg} {resource}/{object}") # restore initial for courtesy - self.set_resource(url, initial, prev) + self.set_resource(url, node_url, initial, prev) return test.PASS() From 35a2c78b5d1fd8715a3d74c5c2ecbc6c4821172e Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Tue, 5 Mar 2024 10:29:38 -0500 Subject: [PATCH 05/15] is-13: cover senders and receivers --- nmostesting/suites/IS1301Test.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index fd2e6313..f8818392 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -230,4 +230,26 @@ def test_02_03(self, test): """Annotation test: devices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" return self.do_test(test, "devices", "tags") -# TODO add receivers + senders + def test_03_01(self, test): + """ Annotation test: senders/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "senders", "label") + + def test_03_02(self, test): + """Annotation test: senders/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "senders", "description") + + def test_03_03(self, test): + """Annotation test: sender/sevices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "senders", "tags") + + def test_04_01(self, test): + """ Annotation test: receivers/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "receivers", "label") + + def test_04_02(self, test): + """Annotation test: receivers/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "receivers", "description") + + def test_04_03(self, test): + """Annotation test: receivers/sevices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "receivers", "tags") From 1f3b948a6ed99ccc215e2aeb20bf94a69a98abb7 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 15:33:27 -0400 Subject: [PATCH 06/15] IS-13: strip grouphint tag which pollutes the test --- nmostesting/suites/IS1301Test.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index f8818392..113b68f4 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -33,15 +33,25 @@ TAGS_OVER_MAX_VALUE = {'location': ['underground'], 'studio': ['42'], 'tech': ['John', 'Mike']} TAGS_MAX_VALUE = TAGS_OVER_MAX_VALUE.copy() TAGS_OVER_MAX_VALUE.pop('tech') # must have a max of 5 +TAGS_TO_BE_SKIPPED = 'urn:x-nmos:tag:grouphint/v1.0' + def get_ts_from_version(version): """ Convert the 'version' object (string) into float """ return float(re.sub(':', '.', version)) + +def strip_tags(tags): + if TAGS_TO_BE_SKIPPED in list(tags.keys()): + tags.pop(TAGS_TO_BE_SKIPPED) + return tags + + class IS1301Test(GenericTest): """ Runs IS-13-Test """ + def __init__(self, apis, **kwargs): GenericTest.__init__(self, apis, **kwargs) self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] @@ -90,8 +100,11 @@ def set_resource(self, url, node_url, new, prev): return False, "new version FAIL" # check PATCH == GET if new[object] is not None: # NOT a reset - if object == "tags" and not TestHelper.compare_json(resp[object], new[object]): - return False, f"new {object} FAIL" + if object == "tags": + if TAGS_TO_BE_SKIPPED in list(resp[object].keys()): + resp[object].pop(TAGS_TO_BE_SKIPPED) + if not TestHelper.compare_json(resp[object], new[object]): + return False, f"new {object} FAIL" elif resp[object] != new[object]: return False, f"new {object} FAIL" @@ -117,9 +130,9 @@ def get_url(self, base_url, resource): if resource != "self": # get first of the list of devices, receivers, senders valid, r = self.get_resource(url) if valid: - if isinstance(r[0], str): # in annotation api + if isinstance(r[0], str): # in annotation api index = r[0] - elif isinstance(r[0], dict): # in node api + elif isinstance(r[0], dict): # in node api index = r[0]['id'] else: return None @@ -155,6 +168,7 @@ def do_test(self, test, resource, object): initial = copy.copy(r) initial.pop('id') initial.pop('version') + initial['tags'] = strip_tags(initial['tags']) self.log(f" {msg}: {r}") else: return test.FAIL(f"Can't {msg} {resource}/{object}") From 3c8bbf4de3406d2d667bbc9cbf40cb3a900504cf Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 15:44:01 -0400 Subject: [PATCH 07/15] IS-13: improve error msg --- nmostesting/suites/IS1301Test.py | 116 +++++++++++++++++++------------ 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 113b68f4..1ee50c42 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -74,6 +74,7 @@ def set_up_tests(self): def get_resource(self, url): """ Get a resource """ + valid, r = self.do_request("GET", url) if valid and r.status_code == 200: try: @@ -83,6 +84,22 @@ def get_resource(self, url): else: return False, "GET Resquest FAIL" + def compare_resource(self, object, new, resp): + """ Compare string values (or "tags" dict) """ + + if new[object] is None: # this is a reset, new is null, skip + return True, "" + + if object == "tags": # tags needs to be stripped + resp[object] = strip_tags(resp[object]) + if not TestHelper.compare_json(resp[object], new[object]): + return False, f"{object} value FAIL" + + elif resp[object] != new[object]: + return False, f"{object} value FAIL" + + return True, "" + def set_resource(self, url, node_url, new, prev): """ Patch a resource with one ore several object values """ object = list(new.keys())[0] @@ -91,34 +108,29 @@ def set_resource(self, url, node_url, new, prev): if not valid: return False, "PATCH Resquest FAIL" + # re-GET valid, resp = self.get_resource(url) if not valid: return False, "Get Resquest FAIL" - + # check PATCH == GET + valid, msg = self.compare_resource(object, new, resp) + if not valid: + return False, f"new {msg}" # check that the version (timestamp) has increased if get_ts_from_version(prev['version']) >= get_ts_from_version(resp['version']): - return False, "new version FAIL" - # check PATCH == GET - if new[object] is not None: # NOT a reset - if object == "tags": - if TAGS_TO_BE_SKIPPED in list(resp[object].keys()): - resp[object].pop(TAGS_TO_BE_SKIPPED) - if not TestHelper.compare_json(resp[object], new[object]): - return False, f"new {object} FAIL" - elif resp[object] != new[object]: - return False, f"new {object} FAIL" + return False, "new version (timestamp) FAIL" # validate that it is reflected in IS04 valid, node_resp = self.get_resource(node_url) if not valid: - return False, "GET node FAIL" - if new[object] is not None: # NOT a reset - if object == "tags" and not TestHelper.compare_json(node_resp[object], new[object]): - return False, f"new node/.../{object} FAIL" - elif node_resp[object] != new[object]: - return False, f"new node/.../{object} FAIL" + return False, "GET IS-04 Node FAIL" + # check PATCH == GET + valid, msg = self.compare_resource(object, new, resp) + if not valid: + return False, f"new IS-04/node/.../ {msg}" + # check that the version (timestamp) has increased if get_ts_from_version(node_resp['version']) != get_ts_from_version(resp['version']): - return False, "new node version FAIL" + return False, "new IS-04/node/.../version (timestamp) FAIL" return True, resp @@ -126,8 +138,13 @@ def log(self, msg): print(msg) def get_url(self, base_url, resource): + """ + Build the url for both annotation and node APIs which behaves differently. + For iterables resources (devices, senders, receivers), return the 1st element. + """ + url = f"{base_url}{resource}" - if resource != "self": # get first of the list of devices, receivers, senders + if resource != "self": valid, r = self.get_resource(url) if valid: if isinstance(r[0], str): # in annotation api @@ -139,6 +156,7 @@ def get_url(self, base_url, resource): url = f"{url}/{index}" else: return None + return url def do_test(self, test, resource, object): @@ -149,75 +167,83 @@ def do_test(self, test, resource, object): - Reset default value, check timestamp and store - Write max-length and check value+timestamp - Write >max-length and check value+timestamp - - Reset default value and compare + - Reset default value again and compare - Restore initial value """ url = self.get_url(f"{self.annotation_url}node/", resource) if not url: - return test.FAIL(f"Can't get annotation url for {resource}") + msg = f"Can't get annotation url for {resource}" + self.log(f" FAIL {msg}") + return test.FAIL(msg) node_url = self.get_url(self.node_url, resource) if not url: - return test.FAIL(f"Can't get node url for {resource}") + msg = f"Can't get node url for {resource}" + self.log(f" FAIL {msg}") + return test.FAIL(msg) - msg = "SAVE initial" valid, r = self.get_resource(url) + msg = f"SAVE initial: {r}" + self.log(f" {msg}") if valid: prev = r initial = copy.copy(r) initial.pop('id') initial.pop('version') initial['tags'] = strip_tags(initial['tags']) - self.log(f" {msg}: {r}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "RESET to default and save" valid, r = self.set_resource(url, node_url, {object: None}, prev) + msg = f"RESET to default and save: {r}" + self.log(f" {msg}") if valid: default = prev = r - self.log(f" {msg}: {r}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "SET MAX value and expected complete response" value = TAGS_MAX_VALUE if object == "tags" else STRING_MAX_VALUE valid, r = self.set_resource(url, node_url, {object: value}, prev) + msg = f"SET MAX value and expected complete response: {r}" + self.log(f" {msg}") if valid: prev = r - self.log(f" {msg}: {r}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "SET >MAX value and expect truncated response" value = TAGS_OVER_MAX_VALUE if object == "tags" else STRING_OVER_MAX_VALUE valid, r = self.set_resource(url, node_url, {object: value}, prev) + msg = f"SET >MAX value and expect truncated response: {r}" + self.log(f" {msg}") if valid: prev = r - self.log(f" {msg}: {r}") - if object == "tags" and not TestHelper.compare_json(r[object], TAGS_MAX_VALUE): - return test.FAIL(f"Can't {msg} {resource}/{object}") - elif r[object] != STRING_MAX_VALUE: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(f" {msg}") + if object == "tags" and not TestHelper.compare_json(r[object], TAGS_MAX_VALUE) or r[object] != STRING_MAX_VALUE: + return test.FAIL(f"Can't {msg}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "RESET again and compare" valid, r = self.set_resource(url, node_url, {object: None}, prev) + msg = f"RESET again and compare: {r}" + self.log(f" {msg}") if valid: - self.log(f" {msg}: {r}") - if object == "tags" and not TestHelper.compare_json(default[object], r[object]): - return test.FAIL("Second reset give a different default.") - elif default[object] != r[object]: - return test.FAIL("Second reset give a different default.") + if object == "tags" and not TestHelper.compare_json(default[object], r[object]) or default[object] != r[object]: + self.log(" FAIL") + return test.FAIL("Second reset gives a different default value.") prev = r else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") # restore initial for courtesy self.set_resource(url, node_url, initial, prev) + self.log(" PASS") return test.PASS() def test_01_01(self, test): From 66642768835da979d14ef6813a02e9a11f7370d0 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 16:40:03 -0400 Subject: [PATCH 08/15] IS-13: add a pause to accomodate the update propagation This seems to impact the version(timestamp) test. --- nmostesting/suites/IS1301Test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 1ee50c42..59ca04b7 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -18,6 +18,7 @@ from ..import TestHelper import re import copy +import time ANNOTATION_API_KEY = "annotation" NODE_API_KEY = "node" @@ -102,12 +103,15 @@ def compare_resource(self, object, new, resp): def set_resource(self, url, node_url, new, prev): """ Patch a resource with one ore several object values """ - object = list(new.keys())[0] + object = list(new.keys())[0] valid, resp = TestHelper.do_request("PATCH", url, json=new) if not valid: return False, "PATCH Resquest FAIL" + # pause to accomodate update propagation + time.sleep(0.1) + # re-GET valid, resp = self.get_resource(url) if not valid: From 5fafebb1529c0e62b44a1c381b5c3c7872f08f1a Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 17:36:05 -0400 Subject: [PATCH 09/15] is-13: add test suite description --- nmostesting/suites/IS1301Test.py | 33 ++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 59ca04b7..eddf161d 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -12,6 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. + +""" +The script implements the IS-13 test suite as specified by the nmos-resource-labelling workgroup. +At the end of the test, the initial state of the tested unit is supposed to be restored but this +cannot be garanteed. + +In addition to the basic annotation API tests, this suite includes the test sequence: +For each resource type (self, devices, senders, receivers): + For each annotable object type (label, description, tags): + - Read initial value + - store + - Reset default value by sending null + - check value + timestamp + is-14/value + is-04 timestamp + - store + - Write max-length + - check value + timestamp + is-14/value + is-04 timestamp + - Write >max-length + - check value + timestamp + is-14/value + is-04 timestamp + - Reset default value again + - check value + timestamp + is-14/value + is-04 timestamp + - compare with 1st reset + - Restore initial value +""" + from ..GenericTest import GenericTest, NMOSTestException from ..TestHelper import compare_json @@ -165,14 +189,7 @@ def get_url(self, base_url, resource): def do_test(self, test, resource, object): """ - Perform the test sequence for a resource: - - - Read initial value and store - - Reset default value, check timestamp and store - - Write max-length and check value+timestamp - - Write >max-length and check value+timestamp - - Reset default value again and compare - - Restore initial value + Perform the test sequence as documented in the file header """ url = self.get_url(f"{self.annotation_url}node/", resource) From e5d6cbe16ad76263146224497738ae37f6a34dcb Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 29 Apr 2024 14:49:25 -0400 Subject: [PATCH 10/15] IS-13: rename functions, variables and constants --- nmostesting/suites/IS1301Test.py | 190 +++++++++++++------------------ 1 file changed, 81 insertions(+), 109 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index eddf161d..1b5d81ab 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -14,26 +14,13 @@ """ -The script implements the IS-13 test suite as specified by the nmos-resource-labelling workgroup. -At the end of the test, the initial state of the tested unit is supposed to be restored but this -cannot be garanteed. - -In addition to the basic annotation API tests, this suite includes the test sequence: -For each resource type (self, devices, senders, receivers): - For each annotable object type (label, description, tags): - - Read initial value - - store - - Reset default value by sending null - - check value + timestamp + is-14/value + is-04 timestamp - - store - - Write max-length - - check value + timestamp + is-14/value + is-04 timestamp - - Write >max-length - - check value + timestamp + is-14/value + is-04 timestamp - - Reset default value again - - check value + timestamp + is-14/value + is-04 timestamp - - compare with 1st reset - - Restore initial value +The script implements the IS-13 test suite according to the AMWA IS-13 NMOS Annotation +Specification (https://specs.amwa.tv/is-13/). At the end of the test, the initial state +of the tested unit is supposed to be restored but this cannot be garanteed. + +Terminology: +* `resource` refers to self, devices, senders or receivers endpoints +* `annotation_property` refers to annotable objects: label, description or tags """ from ..GenericTest import GenericTest, NMOSTestException @@ -47,18 +34,15 @@ ANNOTATION_API_KEY = "annotation" NODE_API_KEY = "node" -RESOURCES = ["self", "devices", "senders", "receivers"] -OBJECTS = ["label", "description", "tags"] - -# const for label&description-related tests -STRING_OVER_MAX_VALUE = ''.join(['X' for i in range(100)]) -STRING_MAX_VALUE = STRING_OVER_MAX_VALUE[:64] # this is the max length tolerated +# Constants for label and description related tests +STRING_LENGTH_OVER_MAX_VALUE = ''.join(['X' for i in range(100)]) +STRING_LENGTH_MAX_VALUE = STRING_LENGTH_OVER_MAX_VALUE[:64] # this is the max length tolerated -# const for tags-related tests -TAGS_OVER_MAX_VALUE = {'location': ['underground'], 'studio': ['42'], 'tech': ['John', 'Mike']} -TAGS_MAX_VALUE = TAGS_OVER_MAX_VALUE.copy() -TAGS_OVER_MAX_VALUE.pop('tech') # must have a max of 5 -TAGS_TO_BE_SKIPPED = 'urn:x-nmos:tag:grouphint/v1.0' +# Constants for tags related tests +TAGS_LENGTH_OVER_MAX_VALUE = {'location': ['underground'], 'studio': ['42'], 'tech': ['John', 'Mike']} +TAGS_LENGTH_MAX_VALUE = TAGS_LENGTH_OVER_MAX_VALUE.copy() +TAGS_LENGTH_MAX_VALUE.pop('tech') # must have a max of 5 +TAGS_TO_BE_STRIPPED = 'urn:x-nmos:tag:grouphint/v1.0' def get_ts_from_version(version): @@ -67,8 +51,8 @@ def get_ts_from_version(version): def strip_tags(tags): - if TAGS_TO_BE_SKIPPED in list(tags.keys()): - tags.pop(TAGS_TO_BE_SKIPPED) + if TAGS_TO_BE_STRIPPED in list(tags.keys()): + tags.pop(TAGS_TO_BE_STRIPPED) return tags @@ -82,21 +66,6 @@ def __init__(self, apis, **kwargs): self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] self.node_url = f"{self.apis[ANNOTATION_API_KEY]['base_url']}/x-nmos/node/{self.apis[NODE_API_KEY]['version']}/" - def set_up_tests(self): - """ - FAKE_ORIG = { - 'description': 'fake_orig_desc', - 'label': 'fake_orig_label', - 'tags': { - 'location': ['fake_location'] - } - } - for resource in RESOURCES: - url = "{}{}{}".format(self.annotation_url, 'node/', resource) - TestHelper.do_request("PATCH", url, json=FAKE_ORIG) - """ - pass - def get_resource(self, url): """ Get a resource """ @@ -107,31 +76,35 @@ def get_resource(self, url): except Exception as e: return False, e.msg else: - return False, "GET Resquest FAIL" + return False, "GET Request FAIL" - def compare_resource(self, object, new, resp): - """ Compare string values (or "tags" dict) """ + def compare_resource(self, annotation_property, value1, value2): + """ Compare strings (or dict for 'tags') """ - if new[object] is None: # this is a reset, new is null, skip + if value1[annotation_property] is None: # this is a reset, value1 is null, skip return True, "" - if object == "tags": # tags needs to be stripped - resp[object] = strip_tags(resp[object]) - if not TestHelper.compare_json(resp[object], new[object]): - return False, f"{object} value FAIL" + if annotation_property == "tags": # tags needs to be stripped + value2[annotation_property] = strip_tags(value2[annotation_property]) + if not TestHelper.compare_json(value2[annotation_property], value1[annotation_property]): + return False, f"{annotation_property} value FAIL" - elif resp[object] != new[object]: - return False, f"{object} value FAIL" + elif value2[annotation_property] != value1[annotation_property]: + return False, f"{annotation_property} value FAIL" return True, "" - def set_resource(self, url, node_url, new, prev): + def set_resource(self, url, node_url, new, prev, msg): """ Patch a resource with one ore several object values """ + global prev + + self.log(f" {msg}") - object = list(new.keys())[0] - valid, resp = TestHelper.do_request("PATCH", url, json=new) + annotation_property = list(new.keys())[0] + valid, resp = self.do_request("PATCH", url, json=new) if not valid: - return False, "PATCH Resquest FAIL" + # raise NMOSTestException(test.WARNING("501 'Not Implemented' status code is not supported below API " "version v1.3", NMOS_WIKI_URL + "/IS-04#nodes-basic-connection-management")) + return False, "PATCH Request FAIL" # pause to accomodate update propagation time.sleep(0.1) @@ -139,9 +112,9 @@ def set_resource(self, url, node_url, new, prev): # re-GET valid, resp = self.get_resource(url) if not valid: - return False, "Get Resquest FAIL" + return False, "Get Request FAIL" # check PATCH == GET - valid, msg = self.compare_resource(object, new, resp) + valid, msg = self.compare_resource(annotation_property, new, resp) if not valid: return False, f"new {msg}" # check that the version (timestamp) has increased @@ -153,7 +126,7 @@ def set_resource(self, url, node_url, new, prev): if not valid: return False, "GET IS-04 Node FAIL" # check PATCH == GET - valid, msg = self.compare_resource(object, new, resp) + valid, msg = self.compare_resource(annotation_property, new, resp) if not valid: return False, f"new IS-04/node/.../ {msg}" # check that the version (timestamp) has increased @@ -165,7 +138,7 @@ def set_resource(self, url, node_url, new, prev): def log(self, msg): print(msg) - def get_url(self, base_url, resource): + def create_url(self, base_url, resource): """ Build the url for both annotation and node APIs which behaves differently. For iterables resources (devices, senders, receivers), return the 1st element. @@ -187,18 +160,34 @@ def get_url(self, base_url, resource): return url - def do_test(self, test, resource, object): + + def do_test_sequence(self, test, resource, annotation_property): """ - Perform the test sequence as documented in the file header + In addition to the basic annotation API tests, this suite includes the test sequence: + For each resource: + For each annotation_property: + - Read initial value + - store + - Reset default value by sending null + - check value + timestamp + is-14/value + is-04 timestamp + - store + - Write max-length + - check value + timestamp + is-14/value + is-04 timestamp + - Write >max-length + - check value + timestamp + is-14/value + is-04 timestamp + - Reset default value again + - check value + timestamp + is-14/value + is-04 timestamp + - compare with 1st reset + - Restore initial value """ - url = self.get_url(f"{self.annotation_url}node/", resource) + url = self.create_url(f"{self.annotation_url}node/", resource) if not url: msg = f"Can't get annotation url for {resource}" self.log(f" FAIL {msg}") return test.FAIL(msg) - node_url = self.get_url(self.node_url, resource) + node_url = self.create_url(self.node_url, resource) if not url: msg = f"Can't get node url for {resource}" self.log(f" FAIL {msg}") @@ -217,8 +206,8 @@ def do_test(self, test, resource, object): self.log(" FAIL") return test.FAIL(f"Can't {msg}") - valid, r = self.set_resource(url, node_url, {object: None}, prev) msg = f"RESET to default and save: {r}" + valid, r = self.set_resource(url, node_url, {annotation_property: None}, prev, msg) self.log(f" {msg}") if valid: default = prev = r @@ -226,91 +215,74 @@ def do_test(self, test, resource, object): self.log(" FAIL") return test.FAIL(f"Can't {msg}") - value = TAGS_MAX_VALUE if object == "tags" else STRING_MAX_VALUE - valid, r = self.set_resource(url, node_url, {object: value}, prev) msg = f"SET MAX value and expected complete response: {r}" - self.log(f" {msg}") - if valid: - prev = r - else: - self.log(" FAIL") - return test.FAIL(f"Can't {msg}") + value = TAGS_LENGTH_MAX_VALUE if annotation_property == "tags" else STRING_LENGTH_MAX_VALUE + valid, r = self.set_resource(url, node_url, {annotation_property: value}, prev, msg) - value = TAGS_OVER_MAX_VALUE if object == "tags" else STRING_OVER_MAX_VALUE - valid, r = self.set_resource(url, node_url, {object: value}, prev) msg = f"SET >MAX value and expect truncated response: {r}" - self.log(f" {msg}") + value = TAGS_LENGTH_OVER_MAX_VALUE if annotation_property == "tags" else STRING_LENGTH_OVER_MAX_VALUE + valid, r = self.set_resource(url, node_url, {annotation_property: value}, prev, msg) if valid: - prev = r - self.log(f" {msg}") - if object == "tags" and not TestHelper.compare_json(r[object], TAGS_MAX_VALUE) or r[object] != STRING_MAX_VALUE: + if annotation_property == "tags" and not TestHelper.compare_json(r[annotation_property], TAGS_LENGTH_MAX_VALUE) or r[annotation_property] != STRING_LENGTH_MAX_VALUE: return test.FAIL(f"Can't {msg}") - else: - self.log(" FAIL") - return test.FAIL(f"Can't {msg}") - valid, r = self.set_resource(url, node_url, {object: None}, prev) msg = f"RESET again and compare: {r}" - self.log(f" {msg}") + valid, r = self.set_resource(url, node_url, {annotation_property: None}, prev, msg) if valid: - if object == "tags" and not TestHelper.compare_json(default[object], r[object]) or default[object] != r[object]: + if annotation_property == "tags" and not TestHelper.compare_json(default[annotation_property], r[annotation_property]) or default[annotation_property] != r[annotation_property]: self.log(" FAIL") return test.FAIL("Second reset gives a different default value.") - prev = r - else: - self.log(" FAIL") - return test.FAIL(f"Can't {msg}") - # restore initial for courtesy - self.set_resource(url, node_url, initial, prev) + msg = f"RESTORE initial values" + self.set_resource(url, node_url, initial, prev, msg) self.log(" PASS") return test.PASS() def test_01_01(self, test): """ Annotation test: self/label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" - return self.do_test(test, "self", "label") + return self.do_test_sequence(test, "self", "label") def test_01_02(self, test): """ Annotation test: self/description (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" - return self.do_test(test, "self", "description") + return self.do_test_sequence(test, "self", "description") def test_01_03(self, test): """ Annotation test: self/tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" - return self.do_test(test, "self", "tags") + return self.do_test_sequence(test, "self", "tags") def test_02_01(self, test): """ Annotation test: devices/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" - return self.do_test(test, "devices", "label") + return self.do_test_sequence(test, "devices", "label") def test_02_02(self, test): """Annotation test: devices/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" - return self.do_test(test, "devices", "description") + return self.do_test_sequence(test, "devices", "description") def test_02_03(self, test): """Annotation test: devices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" - return self.do_test(test, "devices", "tags") + return self.do_test_sequence(test, "devices", "tags") def test_03_01(self, test): """ Annotation test: senders/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" - return self.do_test(test, "senders", "label") + return self.do_test_sequence(test, "senders", "label") def test_03_02(self, test): """Annotation test: senders/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" - return self.do_test(test, "senders", "description") + return self.do_test_sequence(test, "senders", "description") def test_03_03(self, test): """Annotation test: sender/sevices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" - return self.do_test(test, "senders", "tags") + return self.do_test_sequence(test, "senders", "tags") def test_04_01(self, test): """ Annotation test: receivers/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" - return self.do_test(test, "receivers", "label") + return self.do_test_sequence(test, "receivers", "label") def test_04_02(self, test): """Annotation test: receivers/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" - return self.do_test(test, "receivers", "description") + return self.do_test_sequence(test, "receivers", "description") def test_04_03(self, test): """Annotation test: receivers/sevices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" - return self.do_test(test, "receivers", "tags") + return self.do_test_sequence(test, "receivers", "tags") From 6080619ce46937cf59b56502ffa726e1624a3050 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Thu, 17 Oct 2024 14:07:19 -0400 Subject: [PATCH 11/15] IS-13: remove logger --- nmostesting/suites/IS1301Test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 1b5d81ab..20f8cfbc 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -136,7 +136,11 @@ def set_resource(self, url, node_url, new, prev, msg): return True, resp def log(self, msg): - print(msg) + """ + Enable for quick debug only + """ + # print(msg) + return def create_url(self, base_url, resource): """ From 1886026a3ab222535afa195af0165f20cf640a33 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 21 Oct 2024 15:01:18 -0400 Subject: [PATCH 12/15] IS-13: refactor results and errors Raise exceptions instead of returning tuples. Add reference to the specification when possible. --- nmostesting/suites/IS1301Test.py | 166 ++++++++++++++----------------- 1 file changed, 76 insertions(+), 90 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 20f8cfbc..d3268e22 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -24,13 +24,15 @@ """ from ..GenericTest import GenericTest, NMOSTestException -from ..TestHelper import compare_json from ..import TestHelper import re import copy import time +IS13_SPEC_VERSION = "v1.0-dev" +IS13_SPEC_URL = f"https://specs.amwa.tv/is-13/branches/{IS13_SPEC_VERSION}/docs" + ANNOTATION_API_KEY = "annotation" NODE_API_KEY = "node" @@ -72,68 +74,65 @@ def get_resource(self, url): valid, r = self.do_request("GET", url) if valid and r.status_code == 200: try: - return True, r.json() + return r.json() except Exception as e: - return False, e.msg + raise(f"Can't parse response as json {e.msg}") else: - return False, "GET Request FAIL" + return None def compare_resource(self, annotation_property, value1, value2): """ Compare strings (or dict for 'tags') """ - if value1[annotation_property] is None: # this is a reset, value1 is null, skip - return True, "" + if value2[annotation_property] is None: # this is a reset, value2 is null, skip + return True if annotation_property == "tags": # tags needs to be stripped value2[annotation_property] = strip_tags(value2[annotation_property]) - if not TestHelper.compare_json(value2[annotation_property], value1[annotation_property]): - return False, f"{annotation_property} value FAIL" - - elif value2[annotation_property] != value1[annotation_property]: - return False, f"{annotation_property} value FAIL" + return TestHelper.compare_json(value2[annotation_property], value1[annotation_property]) - return True, "" + return value2[annotation_property] == value1[annotation_property] - def set_resource(self, url, node_url, new, prev, msg): + def set_resource(self, test, url, node_url, value, prev, expected, msg, link): """ Patch a resource with one ore several object values """ - global prev self.log(f" {msg}") - annotation_property = list(new.keys())[0] - valid, resp = self.do_request("PATCH", url, json=new) + annotation_property = list(value.keys())[0] + valid, resp = self.do_request("PATCH", url, json=value) if not valid: - # raise NMOSTestException(test.WARNING("501 'Not Implemented' status code is not supported below API " "version v1.3", NMOS_WIKI_URL + "/IS-04#nodes-basic-connection-management")) - return False, "PATCH Request FAIL" + raise NMOSTestException(test.FAIL("PATCH Request FAIL", link=f"{IS13_SPEC_URL}/Behaviour.html#setting-values")) + # TODO: if put_response.status_code == 500: # pause to accomodate update propagation time.sleep(0.1) # re-GET - valid, resp = self.get_resource(url) - if not valid: - return False, "Get Request FAIL" + resp = self.get_resource(url) + if not resp: + raise NMOSTestException(test.FAIL(f"GET /{ANNOTATION_API_KEY} FAIL")) # check PATCH == GET - valid, msg = self.compare_resource(annotation_property, new, resp) - if not valid: - return False, f"new {msg}" + if not self.compare_resource(annotation_property, resp, expected): + raise NMOSTestException(test.FAIL(f"Compare req vs expect FAIL - {msg}", link=link)) # check that the version (timestamp) has increased if get_ts_from_version(prev['version']) >= get_ts_from_version(resp['version']): - return False, "new version (timestamp) FAIL" + raise NMOSTestException(test.FAIL(f"Version update FAIL \ + ({get_ts_from_version(prev['version'])} !>= {get_ts_from_version(resp['version'])})", \ + link=f"{IS13_SPEC_URL}/Behaviour.html#successful-response>")) # validate that it is reflected in IS04 - valid, node_resp = self.get_resource(node_url) - if not valid: - return False, "GET IS-04 Node FAIL" + node_resp = self.get_resource(node_url) + if not node_resp: + raise NMOSTestException(test.FAIL(f"GET /{NODE_API_KEY} FAIL")) # check PATCH == GET - valid, msg = self.compare_resource(annotation_property, new, resp) - if not valid: - return False, f"new IS-04/node/.../ {msg}" + if not self.compare_resource(annotation_property, resp, expected): + raise NMOSTestException(test.FAIL(f"Compare /annotation vs /node FAIL {msg}", + link=f"{IS13_SPEC_URL}/Interoperability_-_IS-04.html#consistent-resources")) # check that the version (timestamp) has increased if get_ts_from_version(node_resp['version']) != get_ts_from_version(resp['version']): - return False, "new IS-04/node/.../version (timestamp) FAIL" + raise NMOSTestException(test.FAIL(f"Compare /annotation Version vs /node FAIL {msg}", + link=f"{IS13_SPEC_URL}/Interoperability_-_IS-04.html#version-increments")) - return True, resp + return resp def log(self, msg): """ @@ -142,28 +141,37 @@ def log(self, msg): # print(msg) return - def create_url(self, base_url, resource): + def create_url(self, test, base_url, resource): """ Build the url for both annotation and node APIs which behaves differently. For iterables resources (devices, senders, receivers), return the 1st element. """ url = f"{base_url}{resource}" - if resource != "self": - valid, r = self.get_resource(url) - if valid: + r = self.get_resource(url) + if r: + if resource != "self": if isinstance(r[0], str): # in annotation api index = r[0] elif isinstance(r[0], dict): # in node api index = r[0]['id'] else: - return None + raise NMOSTestException(test.FAIL(f"Unexpected resource found @ {url}")) url = f"{url}/{index}" - else: - return None + else: + raise NMOSTestException(test.FAIL(f"No resource found @ {url}")) return url + def copy_resource(self, resource): + """ Strip and copy resource """ + + r = copy.copy(resource) + r.pop('id') + r.pop('version') + r['tags'] = strip_tags(r['tags']) + + return r def do_test_sequence(self, test, resource, annotation_property): """ @@ -185,62 +193,40 @@ def do_test_sequence(self, test, resource, annotation_property): - Restore initial value """ - url = self.create_url(f"{self.annotation_url}node/", resource) - if not url: - msg = f"Can't get annotation url for {resource}" - self.log(f" FAIL {msg}") - return test.FAIL(msg) + url = self.create_url(test, f"{self.annotation_url}node/", resource) + node_url = self.create_url(test, self.node_url, resource) - node_url = self.create_url(self.node_url, resource) - if not url: - msg = f"Can't get node url for {resource}" - self.log(f" FAIL {msg}") - return test.FAIL(msg) + # Save initial resource value + resp = self.get_resource(url) + initial = self.copy_resource(resp) - valid, r = self.get_resource(url) - msg = f"SAVE initial: {r}" - self.log(f" {msg}") - if valid: - prev = r - initial = copy.copy(r) - initial.pop('id') - initial.pop('version') - initial['tags'] = strip_tags(initial['tags']) - else: - self.log(" FAIL") - return test.FAIL(f"Can't {msg}") - - msg = f"RESET to default and save: {r}" - valid, r = self.set_resource(url, node_url, {annotation_property: None}, prev, msg) - self.log(f" {msg}") - if valid: - default = prev = r - else: - self.log(" FAIL") - return test.FAIL(f"Can't {msg}") + msg = "Reset to default and save." + link = f"{IS13_SPEC_URL}/Behaviour.html#resetting-values" + default = resp = self.set_resource(test, url, node_url, {annotation_property: None}, resp, + {annotation_property: None}, msg, link) - msg = f"SET MAX value and expected complete response: {r}" + msg = "Set max-length value and return complete response." + link = f"{IS13_SPEC_URL}/Behaviour.html#setting-values" value = TAGS_LENGTH_MAX_VALUE if annotation_property == "tags" else STRING_LENGTH_MAX_VALUE - valid, r = self.set_resource(url, node_url, {annotation_property: value}, prev, msg) + resp = self.set_resource(test, url, node_url, {annotation_property: value}, resp, + {annotation_property: value}, msg, link) - msg = f"SET >MAX value and expect truncated response: {r}" + msg = "Exceed max-length value and return truncated response" + link = f"{IS13_SPEC_URL}/Behaviour.html#additional-limitations" value = TAGS_LENGTH_OVER_MAX_VALUE if annotation_property == "tags" else STRING_LENGTH_OVER_MAX_VALUE - valid, r = self.set_resource(url, node_url, {annotation_property: value}, prev, msg) - if valid: - if annotation_property == "tags" and not TestHelper.compare_json(r[annotation_property], TAGS_LENGTH_MAX_VALUE) or r[annotation_property] != STRING_LENGTH_MAX_VALUE: - return test.FAIL(f"Can't {msg}") - - msg = f"RESET again and compare: {r}" - valid, r = self.set_resource(url, node_url, {annotation_property: None}, prev, msg) - if valid: - if annotation_property == "tags" and not TestHelper.compare_json(default[annotation_property], r[annotation_property]) or default[annotation_property] != r[annotation_property]: - self.log(" FAIL") - return test.FAIL("Second reset gives a different default value.") - - msg = f"RESTORE initial values" - self.set_resource(url, node_url, initial, prev, msg) - - self.log(" PASS") + expected = TAGS_LENGTH_MAX_VALUE if annotation_property == "tags" else STRING_LENGTH_MAX_VALUE + resp = self.set_resource(test, url, node_url, {annotation_property: value}, resp, + {annotation_property: expected}, msg, link) + + msg = "Reset again and compare with default." + link = f"{IS13_SPEC_URL}/Behaviour.html#resetting-values" + resp = self.set_resource(test, url, node_url, {annotation_property: None}, resp, default, + msg, link) + + msg = "Restore initial values." + link = "{IS13_SPEC_URL}/Behaviour.html#setting-values" + self.set_resource(test, url, node_url, initial, resp, msg, link) + return test.PASS() def test_01_01(self, test): From 0a2f6fddd171f62e7f7e9d68c87d79cf9dd5c71f Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 21 Oct 2024 17:32:27 -0400 Subject: [PATCH 13/15] IS-13: add service announcement test --- nmostesting/suites/IS1301Test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index d3268e22..dde64f0c 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -229,6 +229,19 @@ def do_test_sequence(self, test, resource, annotation_property): return test.PASS() + def test_00_01(self, test): + """ Annotation service must be announced """ + r = self. get_resource(f"{self.node_url}self/") + print(self.annotation_url) + try: + for s in r['services']: + if 'urn:x-nmos:service:annotation' in s['type'] and s['href'] + '/' == self.annotation_url: + return test.PASS() + except Exception as e: + return test.FAIL(f"Could't parse services in '/node/self' {str(e)}") + return test.FAIL("Could't found 'annotation' as a service in '/node/self'", + link=f"{IS13_SPEC_URL}/Interoperability_-_IS-04.html#discovery") + def test_01_01(self, test): """ Annotation test: self/label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" return self.do_test_sequence(test, "self", "label") From 24a981749b9194875a467cfffd1dfdd56356a0a7 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Tue, 22 Oct 2024 15:07:13 -0400 Subject: [PATCH 14/15] IS-13: list uncovered tests --- nmostesting/suites/IS1301Test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index dde64f0c..8ce57fa0 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -18,6 +18,11 @@ Specification (https://specs.amwa.tv/is-13/). At the end of the test, the initial state of the tested unit is supposed to be restored but this cannot be garanteed. +Not covered yet: +* Persistency: https://specs.amwa.tv/is-13/branches/v1.0-dev/docs/Behaviour.html#persistence-of-updates +* Read only Tags: https://specs.amwa.tv/is-13/branches/v1.0-dev/docs/Behaviour.html#read-only-tags +* 500 reponses are ignored + Terminology: * `resource` refers to self, devices, senders or receivers endpoints * `annotation_property` refers to annotable objects: label, description or tags From 8269c654ab667a882ff30854fada56c10c9e3768 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Tue, 22 Oct 2024 15:53:46 -0400 Subject: [PATCH 15/15] IS-13: make the propagation delay configurable Default value (1s) increases the duration of the test suite to approx 1min. --- nmostesting/suites/IS1301Test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 8ce57fa0..7eec8467 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -21,6 +21,7 @@ Not covered yet: * Persistency: https://specs.amwa.tv/is-13/branches/v1.0-dev/docs/Behaviour.html#persistence-of-updates * Read only Tags: https://specs.amwa.tv/is-13/branches/v1.0-dev/docs/Behaviour.html#read-only-tags +* Individual Tag reset * 500 reponses are ignored Terminology: @@ -30,6 +31,7 @@ from ..GenericTest import GenericTest, NMOSTestException +from ..import Config as CONFIG from ..import TestHelper import re import copy @@ -109,7 +111,7 @@ def set_resource(self, test, url, node_url, value, prev, expected, msg, link): # TODO: if put_response.status_code == 500: # pause to accomodate update propagation - time.sleep(0.1) + time.sleep(CONFIG.API_PROCESSING_TIMEOUT) # re-GET resp = self.get_resource(url)