[IS-11] Make test cases independent

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:bionic
+FROM ubuntu:focal
 
 WORKDIR /home/nmos-testing
 ADD . .
@@ -7,9 +7,9 @@ ADD .git .git
 RUN apt-get update \
     && export DEBIAN_FRONTEND=noninteractive \
     && apt-get install -y wget \
-    && wget https://deb.nodesource.com/setup_14.x \
-    && chmod 755 setup_14.x \
-    && /home/nmos-testing/setup_14.x \
+    && wget https://deb.nodesource.com/setup_16.x \
+    && chmod 755 setup_16.x \
+    && /home/nmos-testing/setup_16.x \
     && apt-get install -y --no-install-recommends \
     gcc openssl libssl-dev wget ca-certificates avahi-daemon avahi-utils libnss-mdns libavahi-compat-libdnssd-dev \
     python3 python3-pip python3-dev nodejs \
@@ -26,7 +26,7 @@ RUN apt-get update \
     && rm v3.0.7.tar.gz \
     && npm config set unsafe-perm true \
     && npm install -g AMWA-TV/sdpoker#v0.3.0 \
-    && rm /home/nmos-testing/setup_14.x \
+    && rm /home/nmos-testing/setup_16.x \
     && apt-get remove -y wget \
     && apt-get clean -y --no-install-recommends \
     && apt-get autoclean -y --no-install-recommends \

docs/1.1. Installation - Local.md

@@ -4,7 +4,7 @@
 
 Please ensure that the following dependencies are installed on your system first.
 
-- Python 3.6 or higher, including the 'pip' package manager
+- Python 3.8 or higher, including the 'pip' package manager
 - Git
 - [testssl.sh](https://testssl.sh) (required for BCP-003-01 testing, see our [README](../testssl/README.md) for instructions)
 - [OpenSSL](https://www.openssl.org/) (required for BCP-003-01 OCSP testing)

nmostesting/Config.py

@@ -19,6 +19,9 @@
 # Please consult the documentation for instructions on how to adjust these values for common testing setups including
 # unicast DNS-SD and HTTPS testing.
 
+# Number of seconds to wait after starting the mock DNS server, authorization server, etc. before running tests.
+# This gives the API or client under test a chance to use these services before any test case is run.
+MOCK_SERVICES_WARM_UP_DELAY = 0
 
 # Enable or disable DNS-SD advertisements. Browsing is always permitted.
 # The IS-04 Node tests create a mock registry on the network unless the `ENABLE_DNS_SD` parameter is set to `False`.

nmostesting/IS11Utils.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from enum import Enum
-
 from . import TestHelper
 from . import Config as CONFIG
 from .IS04Utils import IS04Utils
@@ -22,7 +20,6 @@
 from .GenericTest import GenericTest
 NODE_API_KEY = "node"
 CONN_API_KEY = "connection"
-SND_RCV_SUBSET = Enum('SndRcvSubset', ['ALL', 'WITH_I_O', 'WITHOUT_I_O'])
 
 
 class IS11Utils(NMOSUtils, GenericTest):
@@ -41,47 +38,27 @@ def __init__(self, url, apis):
             self.reference_is05_utils = IS05Utils(CONFIG.IS11_REFERENCE_SENDER_CONNECTION_API_URL)
 
     # TODO: Remove the duplication (IS05Utils)
-    def get_senders(self, filter=SND_RCV_SUBSET.ALL):
+    def get_senders(self):
         """Gets a list of the available senders on the API"""
         toReturn = []
         valid, r = TestHelper.do_request("GET", self.url + "senders/")
         if valid and r.status_code == 200:
             try:
                 for value in r.json():
-                    if filter == SND_RCV_SUBSET.ALL:
-                        toReturn.append(value[:-1])
-                    else:
-                        valid_io, r_io = TestHelper.do_request("GET", self.url + "senders/" + value + "inputs/")
-                        if valid_io and r_io.status_code == 200:
-                            try:
-                                if len(r_io.json()) > 0 and filter == SND_RCV_SUBSET.WITH_I_O or \
-                                   len(r_io.json()) == 0 and filter == SND_RCV_SUBSET.WITHOUT_I_O:
-                                    toReturn.append(value[:-1])
-                            except ValueError:
-                                pass
+                    toReturn.append(value[:-1])
             except ValueError:
                 pass
         return toReturn
 
     # TODO: Remove the duplication (IS05Utils)
-    def get_receivers(self, filter=SND_RCV_SUBSET.ALL):
+    def get_receivers(self):
         """Gets a list of the available receivers on the API"""
         toReturn = []
         valid, r = TestHelper.do_request("GET", self.url + "receivers/")
         if valid and r.status_code == 200:
             try:
                 for value in r.json():
-                    if filter == SND_RCV_SUBSET.ALL:
-                        toReturn.append(value[:-1])
-                    else:
-                        valid_io, r_io = TestHelper.do_request("GET", self.url + "receivers/" + value + "outputs/")
-                        if valid_io and r_io.status_code == 200:
-                            try:
-                                if len(r_io.json()) > 0 and filter == SND_RCV_SUBSET.WITH_I_O or \
-                                   len(r_io.json()) == 0 and filter == SND_RCV_SUBSET.WITHOUT_I_O:
-                                    toReturn.append(value[:-1])
-                            except ValueError:
-                                pass
+                    toReturn.append(value[:-1])
             except ValueError:
                 pass
         return toReturn

nmostesting/NMOSTesting.py

@@ -1163,6 +1163,13 @@ def main(args):
     print(" * Testing tool running on 'http://{}:{}'. Version '{}'"
           .format(get_default_ip(), core_app.config['PORT'], TOOL_VERSION))
 
+    # Give an API or client that is already running a chance to use the mock services
+    # before running any test cases
+    if CONFIG.MOCK_SERVICES_WARM_UP_DELAY:
+        print(" * Waiting for {} seconds to allow discovery of mock services"
+              .format(CONFIG.MOCK_SERVICES_WARM_UP_DELAY))
+        time.sleep(CONFIG.MOCK_SERVICES_WARM_UP_DELAY)
+
     exit_code = 0
     if "suite" not in vars(CMD_ARGS):
         # Interactive testing mode. Await user input.

nmostesting/suites/BCP00301Test.py

@@ -56,7 +56,9 @@ def perform_test_ssl(self, test, args=None):
                                       "--openssl-timeout",
                                       str(CONFIG.HTTP_TIMEOUT),
                                       "--add-ca",
-                                      CONFIG.CERT_TRUST_ROOT_CA
+                                      CONFIG.CERT_TRUST_ROOT_CA,
+                                      "--ip",
+                                      self.apis[SECURE_API_KEY]["ip"]
                                       ] + args + ["{}:{}".format(self.apis[SECURE_API_KEY]["hostname"],
                                                                  self.apis[SECURE_API_KEY]["port"])]
                                      )

nmostesting/suites/IS0401Test.py

@@ -32,10 +32,6 @@
 from ..IS04Utils import IS04Utils
 from ..TestHelper import get_default_ip, is_ip_address, load_resolved_schema, check_content_type
 
-# monkey patch zeroconf to allow us to advertise "_nmos-registration._tcp"
-from zeroconf import service_type_name
-service_type_name.__kwdefaults__['strict'] = False
-
 NODE_API_KEY = "node"
 RECEIVER_CAPS_KEY = "receiver-caps"
 CAPS_REGISTER_KEY = "caps-register"
@@ -94,6 +90,10 @@ def tear_down_tests(self):
         if self.dns_server:
             self.dns_server.reset()
 
+    def _strict_service_name(self, info):
+        # avoid zeroconf._exceptions.BadTypeInNameException: Service name (nmos-registration) must be <= 15 bytes
+        return len(info.type[1:info.type.find('.')]) <= 15
+
     def _mdns_info(self, port, service_type, txt={}, api_ver=None, api_proto=None, api_auth=None, ip=None):
         """Get an mDNS ServiceInfo object in order to create an advertisement"""
         if api_ver is None:
@@ -192,9 +192,9 @@ def do_registry_basics_prereqs(self):
         if CONFIG.DNS_SD_MODE == "multicast":
             # Advertise the primary registry and invalid ones at pri 0, and allow the Node to do a basic registration
             if self.is04_utils.compare_api_version(self.apis[NODE_API_KEY]["version"], "v1.0") != 0:
-                self.zc.register_service(registry_mdns[0])
-                self.zc.register_service(registry_mdns[1])
-            self.zc.register_service(registry_mdns[2])
+                self.zc.register_service(registry_mdns[0], strict=self._strict_service_name(registry_mdns[0]))
+                self.zc.register_service(registry_mdns[1], strict=self._strict_service_name(registry_mdns[1]))
+            self.zc.register_service(registry_mdns[2], strict=self._strict_service_name(registry_mdns[2]))
 
         # Wait for n seconds after advertising the service for the first POST from a Node
         start_time = time.time()
@@ -226,7 +226,7 @@ def do_registry_basics_prereqs(self):
 
             if CONFIG.DNS_SD_MODE == "multicast":
                 for info in registry_mdns[3:]:
-                    self.zc.register_service(info)
+                    self.zc.register_service(info, strict=self._strict_service_name(info))
 
             # Kill registries one by one to collect data around failover
             self.invalid_registry.disable()
@@ -1455,7 +1455,7 @@ def test_21(self, test):
 
         if CONFIG.DNS_SD_MODE == "multicast":
             # Advertise a registry at pri 0 and allow the Node to do a basic registration
-            self.zc.register_service(registry_info)
+            self.zc.register_service(registry_info, strict=self._strict_service_name(registry_info))
 
         # Wait for n seconds after advertising the service for the first POST and then DELETE from a Node
         self.primary_registry.wait_for_registration(CONFIG.DNS_SD_ADVERT_TIMEOUT)
@@ -2160,7 +2160,7 @@ def collect_mdns_announcements(self):
 
         if CONFIG.DNS_SD_MODE == "multicast":
             # Advertise a registry at pri 0 and allow the Node to do a basic registration
-            self.zc.register_service(registry_info)
+            self.zc.register_service(registry_info, strict=self._strict_service_name(registry_info))
 
         # Wait for n seconds after advertising the service for the first POST from a Node
         self.primary_registry.wait_for_registration(CONFIG.DNS_SD_ADVERT_TIMEOUT)