Add server-side unit tests (jupyter-server#54)

* add basic tests for traits

* fix traits to load dynamically

* initial proviser tests

* add initial provisioner tests

* add missing undefined trait

* more provisioner tests

* working provisioner tests

* add coverage to tests

* address coverage gaps

* add basic app tests

* more app tests and kernelspec tests

* refactor main app class to instantiate the NotebookServiceClient

* minor reformatting fixes

.coveragerc

@@ -0,0 +1,4 @@
+[run]
+omit =  data_studio_jupyter_extensions/tests/*
+        data_studio_jupyter_extensions/config/*
+        conftest.py

Makefile

@@ -16,7 +16,8 @@ watch:
 	jlpm run watch
 
 test:
-	pytest
+	pytest --cov=data_studio_jupyter_extensions
+	coverage report --fail-under=80
 
 run-remote:
 	. env.sh

conftest.py

@@ -1,4 +1,41 @@
 import pytest
+from attr import s
+
+from data_studio_jupyter_extensions import constants
+from data_studio_jupyter_extensions.tests.mock.utils import load_openapi_spec
 
 
 pytest_plugins = ["jupyter_server.pytest_plugin"]
+
+
+@pytest.fixture(scope="session")
+def openapi_spec():
+    return load_openapi_spec()
+
+
+@pytest.fixture
+def project_id():
+    return "testproject"
+
+
+@pytest.fixture
+def notebook_id():
+    return "testnotebook"
+
+
+@pytest.fixture
+def process_id():
+    return "testprocess"
+
+
+@pytest.fixture
+def app_base_url(http_server_port):
+    port = http_server_port[-1]
+    return f"http://127.0.0.1:{port}"
+
+
+@pytest.fixture
+def datastudio_env(monkeypatch, project_id, notebook_id, app_base_url):
+    monkeypatch.setenv(constants.DS_PROJECT_ID, project_id)
+    monkeypatch.setenv(constants.DS_NOTEBOOK_ID, notebook_id)
+    monkeypatch.setenv(constants.DS_API_URL, app_base_url)

data_studio_jupyter_extensions/__init__.py

@@ -9,11 +9,11 @@
     data = json.load(fid)
 
 
-def _jupyter_labextension_paths():
+def _jupyter_labextension_paths():  # pragma: no cover
     return [{"src": "labextension", "dest": data["name"]}]
 
 
-def _jupyter_server_extension_points():
+def _jupyter_server_extension_points():  # pragma: no cover
     from data_studio_jupyter_extensions.app import DataStudioJupyterExtensions
 
     return [

data_studio_jupyter_extensions/_version.py

@@ -1 +1 @@
-__version__ = "0.1"
+__version__ = "0.1"  # pragma: no cover

data_studio_jupyter_extensions/app.py

@@ -4,12 +4,14 @@
 from jupyter_server.config_manager import recursive_update
 from jupyter_server.extension.application import ExtensionApp
 from jupyter_server.extension.application import ExtensionAppJinjaMixin
-from traitlets import Instance
+from traitlets import default
 from traitlets import TraitError
+from traitlets import Type
 from traitlets import validate
 from traitlets.config.loader import PyFileConfigLoader
 
 from . import constants
+from .notebook_service import NotebookServiceClient
 from .telemetry.handlers import handlers
 from .telemetry.logger import TelemetryBus
 from .telemetry.utils import DEFAULT_STATIC_FILES_PATH
@@ -31,7 +33,6 @@ class DataStudioJupyterExtensions(ExtensionAppJinjaMixin, ExtensionApp):
     static_paths = [str(DEFAULT_STATIC_FILES_PATH)]
 
     handlers = handlers
-    telemetry_bus = Instance(TelemetryBus, allow_none=True)
 
     aliases = {"mode": "DataStudioJupyterExtensions.mode"}
 
@@ -43,13 +44,34 @@ class DataStudioJupyterExtensions(ExtensionAppJinjaMixin, ExtensionApp):
 
     mode_config_path = str(MODE_CONFIG_PATH)
 
+    # List configurables here.
+    nbservice_client_class = Type(
+        default_value=NotebookServiceClient,
+        config=True,
+    )
+
+    telemetry_bus_class = Type(default_value=TelemetryBus, config=True)
+
     def _jupyter_server_config(self):
-        config = super()._jupyter_server_config()
-        # Update config fro
+        # Type cast self.config to a dictionary
+        # due to some annoying habits of the Config object
+        # when doing a recursive update.
+        config = dict(self.config)
+
+        # Merge extension + its starter config.
+        starter_config = super()._jupyter_server_config()
+        recursive_update(config, starter_config)
+
+        # Load specific "mode" config from file.
         mode_config_fname = self.mode.replace("-", "_") + ".py"
         loader = PyFileConfigLoader(mode_config_fname, path=self.mode_config_path)
         mode_config = loader.load_config()
+
+        # Merge mode config with all other extension config
         recursive_update(config, mode_config)
+
+        # Update the extension's config object.
+        self.config.update(config)
         return config
 
     @validate("mode")
@@ -60,12 +82,26 @@ def _validate_mode(self, proposal):
         return proposal["value"]
 
     def initialize_settings(self):
+        # This method should probably be upstreamed to Jupyter Server.
+        self.initialize_configurables()
+
+        for schema_file in get_schema_files():
+            self.telemetry_bus.register_schema_file(schema_file)
+        self.settings.update(
+            {
+                "telemetry_bus": self.telemetry_bus,
+                "nbservice_client": self.nbservice_client,
+            }
+        )
+
+    def initialize_configurables(self):
+        self.nbservice_client = self.nbservice_client_class.instance(
+            parent=self,
+            log=self.log,
+        )
         self.telemetry_bus = TelemetryBus.instance(
             allowed_schemas=["event.datastudio.jupyter.com/kernel-message"]
         )
-        for schema_file in get_schema_files():
-            self.telemetry_bus.register_schema_file(schema_file)
-        self.settings.update({"telemetry_bus": self.telemetry_bus})
 
 
 launch_instance = DataStudioJupyterExtensions.launch_instance

data_studio_jupyter_extensions/kernelspecs.py

@@ -9,6 +9,8 @@
 from jupyter_server.utils import url_path_join
 from requests.exceptions import HTTPError
 from tornado.escape import json_decode
+from traitlets import default
+from traitlets import Instance
 
 from data_studio_jupyter_extensions import constants
 from data_studio_jupyter_extensions.notebook_service import NotebookServiceClient
@@ -33,17 +35,22 @@ class DSKernelSpecManager(KernelSpecManager):
     _cache = None
     _cache_time = 0
 
+    nbservice_client = Instance(NotebookServiceClient)
+
+    @default("nbservice_client")
+    def _default_nbservice_client(self):  # pragma: no cover
+        return NotebookServiceClient.instance(parent=self.parent)
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        import nest_asyncio
-
         # Apply nest_asyncio to allow nested async calls
         # in an event loop. This allows the kernelspec
         # manager to use the asynchronous methods of
         # the NotebookServiceClient while running
         # in a Tornado event loop.
+        import nest_asyncio
+
         nest_asyncio.apply()
-        self.nbservice_client = NotebookServiceClient.instance(parent=self.parent)
 
     def _fetch_kernel_specs(self):
         if self._cache and time.time() - self._cache_time < 10:
@@ -113,6 +120,6 @@ def get_all_kernel_specs(self):
         return res
 
 
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no cover
     manager = DSKernelSpecManager()
     print(manager.get_all_kernel_specs())

data_studio_jupyter_extensions/notebook_service.py

@@ -40,21 +40,19 @@ class NotebookServiceClient(SingletonConfigurable):
     api_token = UnicodeFromEnv(name="API_TOKEN").tag(config=True)
     client_id = UnicodeFromEnv(name="IAS_CLIENT_ID").tag(config=True)
     client_secret = UnicodeFromEnv(name="IAS_CLIENT_SECRET").tag(config=True)
-
-    ssl_cert_file = Unicode(allow_none=True).tag(config=True)
     request_token = Unicode(allow_none=True)
 
     @default("ssl_cert_file")
-    def _default_ssl_cert_file(self):
+    def _default_ssl_cert_file(self):  # pragma: no cover
         return get_ssl_cert()
 
     http_client = Instance(AsyncHTTPClient).tag(config=True)
 
     @default("http_client")
-    def _default_http_client(self):
+    def _default_http_client(self):  # pragma: no cover
         return AsyncHTTPClient()
 
-    async def fetch_token(self):
+    async def fetch_token(self):  # pragma: no cover
         # From DS-INT UI
         # These were all in data-platform-ui/config/dsp-ui-dev.json
         scope = "openid offline corpds:ds:dsid corpds:ds:firstName corpds:ds:lastName corpds:ds:email"
@@ -127,15 +125,12 @@ async def fetch(self, *parts, method="GET", data=None):
             self.request_token = ""
         elif not self.request_token:
             self.request_token = await self.fetch_token()
-        # # If a request token doesn't exist. Fetch one.
-        # if not self.request_token and self.local_mode:
-        #     self.request_token = await self.fetch_token()
 
         url = ujoin(self.base_url, *parts)
         try:
             request = self._get_request(url, method=method, data=data)
             response = await self.http_client.fetch(request)
-        except ssl.SSLCertVerificationError:
+        except ssl.SSLCertVerificationError:  # pragma: no cover
             # Refresh SSL Cert.
             self.ssl_cert_file = get_ssl_cert()
             request = self._get_request(url, method=method, data=data)
@@ -185,16 +180,18 @@ async def get_external_links_for_kernel(self, process_id):
         r = await self.fetch(f"/kernels/{process_id}/links", method="get")
         return json_decode(r.body)
 
-    async def initialize_namespace_for_spark_kernels(self, namespace_id):
+    async def initialize_namespace_for_spark_kernels(
+        self, namespace_id
+    ):  # pragma: no cover
         """Initialize namespace by Id for spark kernels."""
         raise NotImplementedError
 
-    async def get_profile_properties(self, kerneltype_id):
+    async def get_profile_properties(self, kerneltype_id):  # pragma: no cover
         """ Get profile properties by kernel type Id."""
         raise NotImplementedError
 
 
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no cover
     import asyncio
 
     nbservice = NotebookServiceClient()