Is-13 add basic API tests

README.md

@@ -32,6 +32,7 @@ The following test suites are currently supported.
 | IS-09-01 | IS-09 System API | | (X) | | System Parameters Server |
 | IS-09-02 | IS-09 System API Discovery | X | | | |
 | IS-10-01 | IS-10 Authorization API | | | | Authorization Server |
+| IS-13-01 | IS-13 Annotation API | X | | | |
 | - | BCP-002-01 Natural Grouping | X | | | Included in IS-04 Node API suite |
 | - | BCP-002-02 Asset Distinguishing Information | X | | | Included in IS-04 Node API suite |
 | BCP-003-01 | BCP-003-01 Secure Communication | X | X | | See [Testing TLS](docs/2.2.%20Usage%20-%20Testing%20BCP-003-01%20TLS.md) |

nmostesting/Config.py

@@ -280,6 +280,17 @@
             }
         }
     },
+    "is-13": {
+        "repo": "is-13",
+        "versions": ["v1.0"],
+        "default_version": "v1.0",
+        "apis": {
+            "annotation": {
+                "name": "Annotation API",
+                "raml": "AnnotationAPI.raml"
+            }
+        }
+    },
     "bcp-002-01": {
         "repo": "bcp-002-01",
         "versions": ["v1.0"],

nmostesting/GenericTest.py

@@ -671,7 +671,7 @@ def check_api_resource(self, test, resource, response_code, api, path):
         schema = self.get_schema(api, resource[1]["method"], resource[0], response.status_code)
 
         if not schema:
-            raise NMOSTestException(test.MANUAL("Test suite unable to locate schema"))
+            raise NMOSTestException(test.MANUAL(f"Test suite unable to locate schema for resource:{resource}"))
 
         return self.check_response(schema, resource[1]["method"], response)
 

nmostesting/NMOSTesting.py

@@ -49,7 +49,7 @@
 from .DNS import DNS
 from .GenericTest import NMOSInitException
 from . import ControllerTest
-from .TestResult import TestStates
+from .TestResult import TestStates, TestResult
 from .TestHelper import get_default_ip
 from .NMOSUtils import DEFAULT_ARGS
 from .CRL import CRL, CRL_API
@@ -82,6 +82,7 @@
 from .suites import IS0901Test
 from .suites import IS0902Test
 # from .suites import IS1001Test
+from .suites import IS1301Test
 from .suites import BCP00301Test
 from .suites import BCP0060101Test
 from .suites import BCP0060102Test
@@ -340,6 +341,14 @@
     #     }],
     #     "class": IS1001Test.IS1001Test
     # },
+    "IS-13-01": {
+        "name": "IS-13 Annotation API",
+        "specs": [{
+            "spec_key": "is-13",
+            "api_key": "annotation"
+        }],
+        "class": IS1301Test.IS1301Test,
+    },
     "BCP-003-01": {
         "name": "BCP-003-01 Secure Communication",
         "specs": [{
@@ -623,7 +632,7 @@ def run_tests(test, endpoints, test_selection=["all"]):
         try:
             result = test_obj.run_tests(test_selection)
         except Exception as ex:
-            print(" * ERROR: {}".format(ex))
+            print(" * ERROR while running {}: {}".format(test_selection, ex))
             raise ex
         finally:
             core_app.config['TEST_ACTIVE'] = False
@@ -961,7 +970,8 @@ def run_noninteractive_tests(args):
         else:
             exit_code = print_test_results(results, endpoints, args)
     except Exception as e:
-        print(" * ERROR: {}".format(str(e)))
+        print(" * ERROR raw: {}".format(e.args))
+        print(" * ERROR in non-interactive tests: {}".format(str(e) if not isinstance(e.args[0], TestResult) else e.args[0].detail))
         exit_code = ExitCodes.ERROR
     return exit_code
 

nmostesting/suites/IS1301Test.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2023 Advanced Media Workflow Association
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..GenericTest import GenericTest, NMOSTestException
+from ..TestHelper import compare_json
+
+ANNOTATION_API_KEY = "annotation"
+
+
+class IS1301Test(GenericTest):
+    """
+    Runs IS-13-Test
+    """
+    def __init__(self, apis, **kwargs):
+        GenericTest.__init__(self, apis, **kwargs)
+        self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"]
+
+    def test_01(self, test):
+        """ 1st annotation test  """
+
+        if compare_json({}, {}):
+            return test.PASS()
+        else:
+            return test.FAIL("IO Resource does not correctly reflect the API resources")