Use jupyter kernel mechanism

README.md

@@ -53,38 +53,75 @@ conda install -n r_env r-irkernel
 For other languages, their [corresponding kernels](https://github.com/jupyter/jupyter/wiki/Jupyter-kernels)
 must be installed.
 
-### Limitations
+### Use with nbconvert, voila, papermill,...
 
-This extension works _only_ with Jupyter notebooks and
-JupyterLab. Unfortunately, it does not currently work with
-Jupyter Console, `nbconvert`, and other tools. This is because
-these tools were not designed to allow for the use of custom
-KernelSpecs.
+This extension works out of the box _only_ with Jupyter notebooks and
+JupyterLab.
 
-A new [kernel discovery system](https://jupyter-client.readthedocs.io/en/latest/kernel_providers.html)
-is being developed for Jupyter 6.0 that should enable the
+A new [kernel discovery system](https://github.com/jupyter/jupyter_server/pull/112)
+is being developed that should enable the 
 wider Jupyter ecosystem to take advantage of these external
 kernels. This package will require modification to
 function properly in this new system.
 
+But you can activate a workaround for it to work with 
+Jupyter Console, `nbconvert`, and other tools. As
+these tools were not designed to allow for the use of custom
+KernelSpecs, you can set the configuration parameter `kernelspec_path`
+to tell this extension to add dynamically the conda environment to
+the kernel list. To set it up:
+
+1. Create a configuration file for jupyter, like `jupyter_config.json`
+in the folder returned by `jupyter --config-dir`.
+2. Add the following configuration to install all kernel spec for the current user:
+```json
+{
+  "CondaKernelSpecManager": {
+    "kernelspec_path": "--user"
+}
+```
+3. Execute the command (or open the classical Notebook or JupyterLab UI):
+```sh
+python -m nb_conda_kernels list
+```
+4. Check that the conda environment kernels are discovered by `jupyter`:
+```sh
+jupyter kernelspec list
+```
+The previous command should list the same kernel than `nb_conda_kernels`.
+
+You are now all set. `nbconvert`, `voila`, `papermill`,... should find the 
+conda environment kernels.
+
 ## Configuration
 
 This package introduces two additional configuration options:
 
+- `conda_only`: Whether to include only the kernels not visible from Jupyter normally or not (default: False except if `kernelspec_path` is set)
 - `env_filter`: Regex to filter environment path matching it. Default: `None` (i.e. no filter)
+- `kernelspec_path`: Path to install conda kernel specs to if not `None`. Default: `None` (i.e. don't install the conda environment as kernel specs for other Jupyter tools)  
+Possible values are:
+  - `""` (empty string): Install for all users
+  - `--user`: Install for the current user instead of system-wide
+  - `--sys-prefix`: Install to Python's sys.prefix
+  - `PREFIX`: Specify an install prefix for the kernelspec. The kernel specs will be
+  written in `PREFIX/share/jupyter/kernels`. Be careful that the PREFIX
+  may not be discoverable by Jupyter; set JUPYTER_DATA_DIR to force it or run 
+  `jupyter --paths` to get the list of data directories.
+
 - `name_format`: String name format; `'{0}'` = Language, `'{1}'` = Kernel. Default: `'{0} [conda env:{1}]'`
 
 In order to pass a configuration option in the command line use ```python -m nb_conda_kernels list --CondaKernelSpecManager.env_filter="regex"``` where regex is the regular expression for filtering envs "this|that|and|that" works.
 To set it in jupyter config file, edit the jupyter configuration file (py or json) located in your ```jupyter --config-dir```
-- for `jupyter_notebook_config.py` - add a line "c.CondaKernelSpecManager.env_filter = 'regex'"
-- for `jupyter_notebook_config.json` - add a json key 
-```{
+- for `jupyter_config.py` - add a line "c.CondaKernelSpecManager.env_filter = 'regex'"
+- for `jupyter_config.json` - add a json key 
+
+```json
+{
   "CondaKernelSpecManager": {
     "env_filter": "regex"
-  ```
-
-- ```python -m nb_conda_kernels list``` does not seem to process jupyter config files
-* filter does not seem to filter out kernels installed with --user, local kernel of the jupyter env, or root kernels
+}
+```
 
 ## Development
 

nb_conda_kernels/__main__.py

@@ -1,5 +1,7 @@
 from jupyter_client import kernelspec
 from .manager import CondaKernelSpecManager
 kernelspec.KernelSpecManager = CondaKernelSpecManager
+
 from jupyter_client.kernelspecapp import KernelSpecApp  # noqa
+
 KernelSpecApp.launch_instance()

nb_conda_kernels/install.py

@@ -13,46 +13,12 @@
 
 
 log = logging.getLogger(__name__)
-log.addHandler(logging.StreamHandler())
-log.setLevel(logging.INFO)
-
-
-# Arguments for command line
-parser = argparse.ArgumentParser(
-    description="Installs the nb_conda_kernels notebook extension")
-group = parser.add_mutually_exclusive_group(required=True)
-group.add_argument(
-    "-s", "--status",
-    help="Print the current status of nb_conda_kernels installation",
-    action="store_true")
-group.add_argument(
-    "-e", "--enable",
-    help="Automatically load nb_conda_kernels on notebook launch",
-    action="store_true")
-group.add_argument(
-    "-d", "--disable",
-    help="Remove nb_conda_kernels from config on notebook launch",
-    action="store_true")
-group2 = parser.add_mutually_exclusive_group(required=False)
-group2.add_argument(
-    "-p", "--prefix",
-    help="Prefix where to load nb_conda_kernels config (default: sys.prefix)",
-    action="store")
-group2.add_argument(
-    "--path",
-    help="Absolute path to jupyter_notebook_config.json",
-    action="store")
-parser.add_argument(
-    "-v", "--verbose",
-    help="Show more output",
-    action="store_true"
-)
-
-
-NBA = "NotebookApp"
+
+
+NBA = "JupyterApp"
 CKSM = "nb_conda_kernels.CondaKernelSpecManager"
 KSMC = "kernel_spec_manager_class"
-JNC = "jupyter_notebook_config"
+JNC = "jupyter_config"
 JNCJ = JNC + ".json"
 JCKP = "jupyter_client.kernel_providers"
 NCKDCKP = "nb_conda_kernels.discovery:CondaKernelProvider"
@@ -191,4 +157,38 @@ def install(enable=False, disable=False, status=None, prefix=None, path=None, ve
 
 
 if __name__ == '__main__':
+    log.addHandler(logging.StreamHandler())
+    log.setLevel(logging.INFO)
+
+    # Arguments for command line
+    parser = argparse.ArgumentParser(
+        description="Installs the nb_conda_kernels notebook extension")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "-s", "--status",
+        help="Print the current status of nb_conda_kernels installation",
+        action="store_true")
+    group.add_argument(
+        "-e", "--enable",
+        help="Automatically load nb_conda_kernels on notebook launch",
+        action="store_true")
+    group.add_argument(
+        "-d", "--disable",
+        help="Remove nb_conda_kernels from config on notebook launch",
+        action="store_true")
+    group2 = parser.add_mutually_exclusive_group(required=False)
+    group2.add_argument(
+        "-p", "--prefix",
+        help="Prefix where to load nb_conda_kernels config (default: sys.prefix)",
+        action="store")
+    group2.add_argument(
+        "--path",
+        help="Absolute path to jupyter_notebook_config.json",
+        action="store")
+    parser.add_argument(
+        "-v", "--verbose",
+        help="Show more output",
+        action="store_true"
+    )
+
     exit(install(**parser.parse_args().__dict__))

nb_conda_kernels/manager.py

@@ -8,7 +8,7 @@
 
 import os
 from os.path import join, split, dirname, basename, abspath
-from traitlets import Unicode, Bool
+from traitlets import Bool, Unicode, TraitError, validate
 
 from jupyter_client.kernelspec import KernelSpecManager, KernelSpec, NoSuchKernel
 
@@ -23,12 +23,39 @@ class CondaKernelSpecManager(KernelSpecManager):
     """ A custom KernelSpecManager able to search for conda environments and
         create kernelspecs for them.
     """
-    conda_only = Bool(False,
-                      help="Include only the kernels not visible from Jupyter normally")
+    conda_only = Bool(False, config=True,
+                      help="Include only the kernels not visible from Jupyter normally (True if kernelspec_path is not None)")
 
     env_filter = Unicode(None, config=True, allow_none=True,
                          help="Do not list environment names that match this regex")
 
+    kernelspec_path = Unicode(None, config=True, allow_none=True,
+        help="""Path to install conda kernel specs to.
+
+        The acceptable values are:
+        - ``""`` (empty string): Install for all users
+        - ``--user``: Install for the current user instead of system-wide
+        - ``--sys-prefix``: Install to Python's sys.prefix
+        - ``PREFIX``: Specify an install prefix for the kernelspec. The kernel specs will be
+        written in ``PREFIX/share/jupyter/kernels``. Be careful that the PREFIX
+        may not be discoverable by Jupyter; set JUPYTER_DATA_DIR to force it or run 
+        ``jupyter --paths`` to get the list of data directories.
+
+        If None, the conda kernel specs will only be available dynamically on notebook editors.
+        """)
+
+    @validate("kernelspec_path")
+    def _validate_kernelspec_path(self, proposal):
+        new_value = proposal["value"]
+        if new_value is not None:
+            if new_value not in ("", "--user", "--sys-prefix"):
+                if not os.path.isdir(self.kernelspec_path):
+                    raise TraitError("CondaKernelSpecManager.kernelspec_path is not a directory.")
+            self.log.debug("[nb_conda_kernels] Force conda_only=True as kernelspec_path is not None.")
+            self.conda_only = True
+
+        return new_value
+
     name_format = Unicode('{0} [conda env:{1}]', config=True,
                           help="String name format; '{{0}}' = Language, '{{1}}' = Kernel")
 
@@ -44,8 +71,14 @@ def __init__(self, **kwargs):
         if self.env_filter is not None:
             self._env_filter_regex = re.compile(self.env_filter)
 
-        self.log.info("[nb_conda_kernels] enabled, %s kernels found",
-                      len(self._conda_kspecs))
+        self._kernel_user = self.kernelspec_path == "--user"
+        self._kernel_prefix = None
+        if not self._kernel_user:
+            self._kernel_prefix = sys.prefix if self.kernelspec_path == "--sys-prefix" else self.kernelspec_path
+
+        self.log.info(
+            "[nb_conda_kernels] enabled, %s kernels found", len(self._conda_kspecs)
+        )
 
     @staticmethod
     def clean_kernel_name(kname):
@@ -60,7 +93,7 @@ def clean_kernel_name(kname):
             nfkd_form = unicodedata.normalize('NFKD', kname)
             kname = u"".join([c for c in nfkd_form if not unicodedata.combining(c)])
         # Replace anything else, including spaces, with underscores
-        kname = re.sub('[^a-zA-Z0-9._\-]', '_', kname)
+        kname = re.sub(r'[^a-zA-Z0-9._\-]', '_', kname)
         return kname
 
     @property
@@ -172,6 +205,9 @@ def _all_specs(self):
                     continue
                 kernel_dir = dirname(spec_path).lower()
                 kernel_name = basename(kernel_dir)
+                if self.kernelspec_path is not None and kernel_name.startswith("conda-"):
+                    self.log.debug("[nb_conda_kernels] Skipping kernel spec %s", spec_path)
+                    continue  # Ensure to skip dynamically added kernel spec within the environment prefix
                 # We're doing a few of these adjustments here to ensure that
                 # the naming convention is as close as possible to the previous
                 # versions of this package; particularly so that the tests
@@ -184,6 +220,7 @@ def _all_specs(self):
                 kernel_name = u'conda-{}{}-{}'.format(kernel_prefix, env_name, kernel_name)
                 # Replace invalid characters with dashes
                 kernel_name = self.clean_kernel_name(kernel_name)
+
                 display_prefix = spec['display_name']
                 if display_prefix.startswith('Python'):
                     display_prefix = 'Python'
@@ -193,14 +230,31 @@ def _all_specs(self):
                 spec['display_name'] = display_name
                 if env_path != sys.prefix:
                     spec['argv'] = RUNNER_COMMAND + [conda_prefix, env_path] + spec['argv']
-                spec['resource_dir'] = abspath(kernel_dir)
                 metadata = spec.get('metadata', {})
                 metadata.update({
                     'conda_env_name': env_name,
                     'conda_env_path': env_path
                 })
                 spec['metadata'] = metadata
+
+                if self.kernelspec_path is not None:                    
+                    # Install the kernel spec
+                    destination = self.install_kernel_spec(
+                        kernel_dir,
+                        kernel_name=kernel_name,
+                        user=self._kernel_user,
+                        prefix=self._kernel_prefix
+                    )
+                    # Update the kernel spec
+                    kernel_spec = join(destination, "kernel.json")
+                    with open(kernel_spec, "w") as f:
+                        json.dump(spec, f)
+
+                # resource_dir is not part of the spec file, so it is added at the latest time
+                spec['resource_dir'] = abspath(kernel_dir)
+
                 all_specs[kernel_name] = spec
+
         return all_specs
 
     @property
@@ -220,9 +274,24 @@ def _conda_kspecs(self):
 
         self._conda_kernels_cache_expiry = time.time() + CACHE_TIMEOUT
         self._conda_kernels_cache = kspecs
+
+        # Remove non-existing conda environments
+        # This is done here to be able to use self.remove_kernel_spec() while avoiding
+        # recursive calls thanks to the cache as that method calls find_kernel_specs
+        if self.kernelspec_path is not None:
+            kernels_destination = self._get_destination_dir(
+                "",
+                user=self._kernel_user,
+                prefix=self._kernel_prefix
+            )
+            for folder in glob.glob(join(kernels_destination, "*", "kernel.json")):
+                kernel_dir = basename(dirname(folder))
+                if kernel_dir.startswith("conda-") and kernel_dir not in kspecs:
+                    self.remove_kernel_spec(kernel_dir)
+
         return kspecs
 
-    def find_kernel_specs(self, skip_base=False):
+    def find_kernel_specs(self):
         """ Returns a dict mapping kernel names to resource directories.
 
             The update process also adds the resource dir for the conda
@@ -246,7 +315,7 @@ def get_kernel_spec(self, kernel_name):
         """ Returns a :class:`KernelSpec` instance for the given kernel_name.
 
             Additionally, conda kernelspecs are generated on the fly
-            accordingly with the detected envitonments.
+            accordingly with the detected environments.
         """
 
         res = self._conda_kspecs.get(kernel_name)