diff --git a/.gitignore b/.gitignore index 8e3f39a69..e8630aa02 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ MANIFEST build dist -_build -docs/gh-pages *.py[co] __pycache__ *.egg-info @@ -16,3 +14,15 @@ __pycache__ .coverage .cache absolute.json + +# Sphinx documentation +_build +docs/_build/ +docs/gh-pages + +# PyBuilder +target/ + +# PyCharm +.idea/ +*.iml diff --git a/.travis.yml b/.travis.yml index 0a3a96915..88fd7e722 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,15 @@ language: python python: - "nightly" - - '3.6-dev' + - "3.7-dev" + - 3.6 - 3.5 - 3.4 - - 3.3 - 2.7 sudo: false install: - pip install --upgrade setuptools pip - - pip install --upgrade --pre -e .[test] pytest-cov pytest-warnings codecov + - pip install --upgrade --pre -e .[test] pytest-cov pytest-warnings codecov 'coverage<5' script: - py.test --cov jupyter_client jupyter_client after_success: @@ -17,3 +17,6 @@ after_success: matrix: allow_failures: - python: nightly +branches: + except: + - /^auto-backport-of-pr-[0-9]+$/ diff --git a/MANIFEST.in b/MANIFEST.in index 42edd273d..994648d70 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include README.md # Documentation graft docs exclude docs/\#* +exclude docs/_* # Examples graft examples diff --git a/docs/changelog.rst b/docs/changelog.rst index 35e21b5c6..11cbd9f1a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,82 @@ Changes in Jupyter Client ========================= +5.2.3 +===== + +`5.2.3 on GitHub `__ + +- Fix hang on close in :class:`.ThreadedKernelClient` (used in QtConsole) + when using tornado with asyncio + (default behavior of tornado 5, see :ghpull:`352`). +- Fix errors when using deprecated :attr:`.KernelManager.kernel_cmd` + (:ghpull:`343`, :ghpull:`344`). + +5.2.2 +===== + +`5.2.2 on GitHub `__ + +- Fix :meth:`.KernelSpecManager.get_all_specs` method in subclasses + that only override :meth:`.KernelSpecManager.find_kernel_specs` + and :meth:`.KernelSpecManager.get_kernel_spec`. + See :ghissue:`338` and :ghpull:`339`. +- Eliminate occasional error messages during process exit (:ghpull:`336`). +- Improve error message when attempting to bind on invalid address (:ghpull:`330`). +- Add missing direct dependency on tornado (:ghpull:`323`). + + +5.2.1 +===== + +`5.2.1 on GitHub `__ + +- Add parenthesis to conditional pytest requirement to work around a bug in the + ``wheel`` package, that generate a ``.whl`` which otherwise always depends on + ``pytest`` see :ghissue:`324` and :ghpull:`325`. + +5.2 +=== + +`5.2 on GitHub `__ + +- Define Jupyter protocol version 5.3: + + - Kernels can now opt to be interrupted by a message sent on the control channel + instead of a system signal. See :ref:`kernelspecs` and :ref:`msging_interrupt` + (:ghpull:`294`). + +- New ``jupyter kernel`` command to launch an installed kernel by name + (:ghpull:`240`). +- Kernelspecs where the command starts with e.g. ``python3`` or + ``python3.6``—matching the version ``jupyter_client`` is running on—are now + launched with the same Python executable as the launching process (:ghpull:`306`). + This extends the special handling of ``python`` added in 5.0. +- Command line arguments specified by a kernelspec can now include + ``{resource_dir}``, which will be substituted with the kernelspec resource + directory path when the kernel is launched (:ghpull:`289`). +- Kernelspecs now have an optional ``metadata`` field to hold arbitrary metadata + about kernels—see :ref:`kernelspecs` (:ghpull:`274`). +- Make the ``KernelRestarter`` class used by a ``KernelManager`` configurable + (:ghpull:`290`). +- When killing a kernel on Unix, kill its process group (:ghpull:`314`). +- If a kernel dies soon after starting, reassign random ports before restarting + it, in case one of the previously chosen ports has been bound by another + process (:ghpull:`279`). +- Avoid unnecessary filesystem operations when finding a kernelspec with + :meth:`.KernelSpecManager.get_kernel_spec` (:ghpull:`311`). +- :meth:`.KernelSpecManager.get_all_specs` will no longer raise an exception on + encountering an invalid ``kernel.json`` file. It will raise a warning and + continue (:ghpull:`310`). +- Check for non-contiguous buffers before trying to send them through ZMQ + (:ghpull:`258`). +- Compatibility with upcoming Tornado version 5.0 (:ghpull:`304`). +- Simplify setup code by always using setuptools (:ghpull:`284`). +- Soften warnings when setting the sticky bit on runtime files fails + (:ghpull:`286`). +- Various corrections and improvements to documentation. + + 5.1 === diff --git a/docs/conf.py b/docs/conf.py index 849d7a56e..c3de08efd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,6 +34,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', + 'sphinxcontrib_github_alt', ] # Add any paths that contain templates here, relative to this directory. @@ -55,6 +56,8 @@ copyright = '2015, Jupyter Development Team' author = 'Jupyter Development Team' +github_project_url = "https://github.com/jupyter/jupyter_client" + # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. diff --git a/docs/environment.yml b/docs/environment.yml index 3690c73b7..459e7ab3b 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -8,3 +8,5 @@ dependencies: - jupyter_core - sphinx>=1.3.6 - sphinx_rtd_theme +- pip: + - sphinxcontrib_github_alt diff --git a/docs/index.rst b/docs/index.rst index a0b8855cc..41e218ccc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ with Jupyter kernels. kernels wrapperkernels + kernel_providers .. toctree:: :maxdepth: 2 diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst new file mode 100644 index 000000000..2e1b7e295 --- /dev/null +++ b/docs/kernel_providers.rst @@ -0,0 +1,156 @@ +================ +Kernel providers +================ + +.. note:: + This is a new interface under development, and may still change. + Not all Jupyter applications use this yet. + See :ref:`kernelspecs` for the established way of discovering kernel types. + +Creating a kernel provider +========================== + +By writing a kernel provider, you can extend how Jupyter applications discover +and start kernels. For example, you could find kernels in an environment system +like conda, or kernels on remote systems which you can access. + +To write a kernel provider, subclass +:class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID +and overriding two methods. + +.. class:: MyKernelProvider + + .. attribute:: id + + A short string identifying this provider. Cannot contain forward slash + (``/``). + + .. method:: find_kernels() + + Get the available kernel types this provider knows about. + Return an iterable of 2-tuples: (name, attributes). + *name* is a short string identifying the kernel type. + *attributes* is a dictionary with information to allow selecting a kernel. + + .. method:: make_manager(name) + + Prepare and return a :class:`~jupyter_client.KernelManager` instance + ready to start a new kernel instance of the type identified by *name*. + The input will be one of the names given by :meth:`find_kernels`. + +For example, imagine we want to tell Jupyter about kernels for a new language +called *oblong*:: + + # oblong_provider.py + from jupyter_client.discovery import KernelProviderBase + from jupyter_client import KernelManager + from shutil import which + + class OblongKernelProvider(KernelProviderBase): + id = 'oblong' + + def find_kernels(self): + if not which('oblong-kernel'): + return # Check it's available + + # Two variants - for a real kernel, these could be something like + # different conda environments. + yield 'standard', { + 'display_name': 'Oblong (standard)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + yield 'rounded', { + 'display_name': 'Oblong (rounded)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + + def make_manager(self, name): + if name == 'standard': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '0'}) + elif name == 'rounded': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '1'}) + else: + raise ValueError("Unknown kernel %s" % name) + +You would then register this with an *entry point*. In your ``setup.py``, put +something like this:: + + setup(... + entry_points = { + 'jupyter_client.kernel_providers' : [ + # The name before the '=' should match the id attribute + 'oblong = oblong_provider:OblongKernelProvider', + ] + }) + +Finding kernel types +==================== + +To find and start kernels in client code, use +:class:`jupyter_client.discovery.KernelFinder`. This uses multiple kernel +providers to find available kernels. Like a kernel provider, it has methods +``find_kernels`` and ``make_manager``. The kernel names it works +with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example +above). + +:: + + from jupyter_client.discovery import KernelFinder + kf = KernelFinder.from_entrypoints() + + ## Find available kernel types + for name, attributes in kf.find_kernels(): + print(name, ':', attributes['display_name']) + # oblong/standard : Oblong (standard) + # oblong/rounded : Oblong(rounded) + # ... + + ## Start a kernel by name + manager = kf.make_manager('oblong/standard') + manager.start_kernel() + +.. module:: jupyter_client.discovery + +.. autoclass:: KernelFinder + + .. automethod:: from_entrypoints + + .. automethod:: find_kernels + + .. automethod:: make_manager + +Kernel providers included in ``jupyter_client`` +=============================================== + +``jupyter_client`` includes two kernel providers: + +.. autoclass:: KernelSpecProvider + + .. seealso:: :ref:`kernelspecs` + +.. autoclass:: IPykernelProvider + +Glossary +======== + +Kernel instance + A running kernel, a process which can accept ZMQ connections from frontends. + Its state includes a namespace and an execution counter. + +Kernel type + The software to run a kernel instance, along with the context in which a + kernel starts. One kernel type allows starting multiple, initially similar + kernel instances. For instance, one kernel type may be associated with one + conda environment containing ``ipykernel``. The same kernel software in + another environment would be a different kernel type. Another software package + for a kernel, such as ``IRkernel``, would also be a different kernel type. + +Kernel provider + A Python class to discover kernel types and allow a client to start instances + of those kernel types. For instance, one kernel provider might find conda + environments containing ``ipykernel`` and allow starting kernel instances in + these environments. diff --git a/docs/kernels.rst b/docs/kernels.rst index 3319dda31..5308c603f 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -6,7 +6,7 @@ Making kernels for Jupyter A 'kernel' is a program that runs and introspects the user's code. IPython includes a kernel for Python code, and people have written kernels for -`several other languages `_. +`several other languages `_. When Jupyter starts a kernel, it passes it a connection file. This specifies how to set up communications with the frontend. @@ -132,6 +132,13 @@ JSON serialised dictionary containing the following keys and values: is found, a kernel with a matching `language` will be used. This allows a notebook written on any Python or Julia kernel to be properly associated with the user's Python or Julia kernel, even if they aren't listed under the same name as the author's. +- **interrupt_mode** (optional): May be either ``signal`` or ``message`` and + specifies how a client is supposed to interrupt cell execution on this kernel, + either by sending an interrupt ``signal`` via the operating system's + signalling facilities (e.g. `SIGINT` on POSIX systems), or by sending an + ``interrupt_request`` message on the control channel (see + :ref:`msging_interrupt`). If this is not specified + the client will default to ``signal`` mode. - **env** (optional): A dictionary of environment variables to set for the kernel. These will be added to the current environment variables before the kernel is started. diff --git a/docs/messaging.rst b/docs/messaging.rst index 776dda681..7c533a7de 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -21,7 +21,7 @@ Versioning The Jupyter message specification is versioned independently of the packages that use it. -The current version of the specification is 5.2. +The current version of the specification is 5.3. .. note:: *New in* and *Changed in* messages in this document refer to versions of the @@ -959,6 +959,27 @@ Message type: ``shutdown_reply``:: socket, they simply send a forceful process termination signal, since a dead process is unlikely to respond in any useful way to messages. +.. _msging_interrupt: + +Kernel interrupt +---------------- + +In case a kernel can not catch operating system interrupt signals (e.g. the used +runtime handles signals and does not allow a user program to define a callback), +a kernel can choose to be notified using a message instead. For this to work, +the kernels kernelspec must set `interrupt_mode` to ``message``. An interruption +will then result in the following message on the `control` channel: + +Message type: ``interrupt_request``:: + + content = {} + +Message type: ``interrupt_reply``:: + + content = {} + +.. versionadded:: 5.3 + Messages on the IOPub (PUB/SUB) channel ======================================= diff --git a/jupyter_client/_version.py b/jupyter_client/_version.py index 90dd2e93e..53bf21fbd 100644 --- a/jupyter_client/_version.py +++ b/jupyter_client/_version.py @@ -1,5 +1,5 @@ -version_info = (5, 1, 0) +version_info = (6, 0, 0, 'dev') __version__ = '.'.join(map(str, version_info)) -protocol_version_info = (5, 2) +protocol_version_info = (5, 3) protocol_version = "%i.%i" % protocol_version_info diff --git a/jupyter_client/channels.py b/jupyter_client/channels.py index dd9906723..8c4ebf3c4 100644 --- a/jupyter_client/channels.py +++ b/jupyter_client/channels.py @@ -80,7 +80,10 @@ def __init__(self, context=None, session=None, address=None): @staticmethod @atexit.register def _notice_exit(): - HBChannel._exiting = True + # Class definitions can be torn down during interpreter shutdown. + # We only need to set _exiting flag if this hasn't happened. + if HBChannel is not None: + HBChannel._exiting = True def _create_socket(self): if self.socket is not None: @@ -139,7 +142,6 @@ def run(self): continue since_last_heartbeat = 0.0 - # io.rprint('Ping from HB channel') # dbg # no need to catch EFSM here, because the previous event was # either a recv or connect, which cannot be followed by EFSM self.socket.send(b'ping') diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 91efbc461..2cc89de35 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -35,9 +35,9 @@ def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, - control_port=0, ip='', key=b'', transport='tcp', - signature_scheme='hmac-sha256', kernel_name='' - ): + control_port=0, ip='', key=b'', transport='tcp', + signature_scheme='hmac-sha256', kernel_name='' + ): """Generates a JSON config file, including the selection of random ports. Parameters @@ -193,7 +193,7 @@ def find_connection_file(filename='kernel-*.json', path=None, profile=None): path = ['.', jupyter_runtime_dir()] if isinstance(path, string_types): path = [path] - + try: # first, try explicit name return filefind(filename, path) @@ -208,11 +208,11 @@ def find_connection_file(filename='kernel-*.json', path=None, profile=None): else: # accept any substring match pat = '*%s*' % filename - + matches = [] for p in path: matches.extend(glob.glob(os.path.join(p, pat))) - + matches = [ os.path.abspath(m) for m in matches ] if not matches: raise IOError("Could not find %r in %r" % (filename, path)) @@ -249,7 +249,7 @@ def tunnel_to_kernel(connection_info, sshserver, sshkey=None): (shell, iopub, stdin, hb) : ints The four ports on localhost that have been forwarded to the kernel. """ - from zmq.ssh import tunnel + from .ssh import tunnel if isinstance(connection_info, string_types): # it's a path, unpack it with open(connection_info) as f: @@ -289,11 +289,11 @@ def tunnel_to_kernel(connection_info, sshserver, sshkey=None): class ConnectionFileMixin(LoggingConfigurable): """Mixin for configurable classes that work with connection files""" - + data_dir = Unicode() def _data_dir_default(self): return jupyter_data_dir() - + # The addresses for the communication channels connection_file = Unicode('', config=True, help="""JSON file in which to store connection info [default: kernel-.json] @@ -480,7 +480,7 @@ def write_connection_file(self): def load_connection_file(self, connection_file=None): """Load connection info from JSON dict in self.connection_file. - + Parameters ---------- connection_file: unicode, optional @@ -496,10 +496,10 @@ def load_connection_file(self, connection_file=None): def load_connection_info(self, info): """Load connection info from a dict containing connection info. - + Typically this data comes from a connection file and is called by load_connection_file. - + Parameters ---------- info: dict diff --git a/jupyter_client/consoleapp.py b/jupyter_client/consoleapp.py index ce2ead429..8d5d329fa 100644 --- a/jupyter_client/consoleapp.py +++ b/jupyter_client/consoleapp.py @@ -18,7 +18,7 @@ from traitlets.config.application import boolean_flag from ipython_genutils.path import filefind from traitlets import ( - Dict, List, Unicode, CUnicode, CBool, Any + Dict, List, Unicode, CUnicode, CBool, Any, Type ) from jupyter_core.application import base_flags, base_aliases @@ -110,7 +110,11 @@ class JupyterConsoleApp(ConnectionFileMixin): classes = classes flags = Dict(flags) aliases = Dict(aliases) - kernel_manager_class = KernelManager + kernel_manager_class = Type( + default_value=KernelManager, + config=True, + help='The kernel manager class to use.' + ) kernel_client_class = BlockingKernelClient kernel_argv = List(Unicode()) @@ -281,10 +285,8 @@ def init_kernel_manager(self): self.exit(1) self.kernel_manager.client_factory = self.kernel_client_class - # FIXME: remove special treatment of IPython kernels kwargs = {} - if self.kernel_manager.ipykernel: - kwargs['extra_arguments'] = self.kernel_argv + kwargs['extra_arguments'] = self.kernel_argv self.kernel_manager.start_kernel(**kwargs) atexit.register(self.kernel_manager.cleanup_ipc_files) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py new file mode 100644 index 000000000..2bfe92b2a --- /dev/null +++ b/jupyter_client/discovery.py @@ -0,0 +1,131 @@ +from abc import ABCMeta, abstractmethod +import entrypoints +import logging +import six + +from .kernelspec import KernelSpecManager +from .manager import KernelManager + +log = logging.getLogger(__name__) + +class KernelProviderBase(six.with_metaclass(ABCMeta, object)): + id = None # Should be a short string identifying the provider class. + + @abstractmethod + def find_kernels(self): + """Return an iterator of (kernel_name, kernel_info_dict) tuples.""" + pass + + @abstractmethod + def make_manager(self, name): + """Make and return a KernelManager instance to start a specified kernel + + name will be one of the kernel names produced by find_kernels() + """ + pass + +class KernelSpecProvider(KernelProviderBase): + """Offers kernel types from installed kernelspec directories. + """ + id = 'spec' + + def __init__(self): + self.ksm = KernelSpecManager() + + def find_kernels(self): + for name, resdir in self.ksm.find_kernel_specs().items(): + spec = self.ksm._get_kernel_spec_by_name(name, resdir) + yield name, { + # TODO: get full language info + 'language': {'name': spec.language}, + 'display_name': spec.display_name, + 'argv': spec.argv, + } + + def make_manager(self, name): + spec = self.ksm.get_kernel_spec(name) + return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) + + +class IPykernelProvider(KernelProviderBase): + """Offers a kernel type using the Python interpreter it's running in. + + This checks if ipykernel is importable first. + """ + id = 'pyimport' + + def _check_for_kernel(self): + try: + from ipykernel.kernelspec import RESOURCES, get_kernel_dict + from ipykernel.ipkernel import IPythonKernel + except ImportError: + return None + else: + return { + 'spec': get_kernel_dict(), + 'language_info': IPythonKernel.language_info, + 'resources_dir': RESOURCES, + } + + def find_kernels(self): + info = self._check_for_kernel() + + if info: + yield 'kernel', { + 'language': info['language_info'], + 'display_name': info['spec']['display_name'], + 'argv': info['spec']['argv'], + } + + def make_manager(self, name): + info = self._check_for_kernel() + if info is None: + raise Exception("ipykernel is not importable") + return KernelManager(kernel_cmd=info['spec']['argv']) + + +class KernelFinder(object): + """Manages a collection of kernel providers to find available kernel types + + *providers* should be a list of kernel provider instances. + """ + def __init__(self, providers): + self.providers = providers + + @classmethod + def from_entrypoints(cls): + """Load all kernel providers advertised by entry points. + + Kernel providers should use the "jupyter_client.kernel_providers" + entry point group. + + Returns an instance of KernelFinder. + """ + providers = [] + for ep in entrypoints.get_group_all('jupyter_client.kernel_providers'): + try: + provider = ep.load()() # Load and instantiate + except Exception: + log.error('Error loading kernel provider', exc_info=True) + else: + providers.append(provider) + + return cls(providers) + + def find_kernels(self): + """Iterate over available kernel types. + + Yields 2-tuples of (prefixed_name, attributes) + """ + for provider in self.providers: + for kid, attributes in provider.find_kernels(): + id = provider.id + '/' + kid + yield id, attributes + + def make_manager(self, name): + """Make a KernelManager instance for a given kernel type. + """ + provider_id, kernel_id = name.split('/', 1) + for provider in self.providers: + if provider_id == provider.id: + return provider.make_manager(kernel_id) diff --git a/jupyter_client/ioloop/manager.py b/jupyter_client/ioloop/manager.py index 511a73f55..f6dee3641 100644 --- a/jupyter_client/ioloop/manager.py +++ b/jupyter_client/ioloop/manager.py @@ -1,15 +1,7 @@ """A kernel manager with a tornado IOLoop""" -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import @@ -24,10 +16,6 @@ from jupyter_client.manager import KernelManager from .restarter import IOLoopKernelRestarter -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - def as_zmqstream(f): def wrapped(self, *args, **kwargs): @@ -37,9 +25,9 @@ def wrapped(self, *args, **kwargs): class IOLoopKernelManager(KernelManager): - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + return ioloop.IOLoop.current() restarter_class = Type( default_value=IOLoopKernelRestarter, @@ -66,6 +54,7 @@ def stop_restarter(self): if self.autorestart: if self._restarter is not None: self._restarter.stop() + self._restarter = None connect_shell = as_zmqstream(KernelManager.connect_shell) connect_iopub = as_zmqstream(KernelManager.connect_iopub) diff --git a/jupyter_client/ioloop/restarter.py b/jupyter_client/ioloop/restarter.py index 6f531744c..69079eecf 100644 --- a/jupyter_client/ioloop/restarter.py +++ b/jupyter_client/ioloop/restarter.py @@ -4,37 +4,28 @@ restarts the kernel if it dies. """ -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import +import warnings from zmq.eventloop import ioloop - from jupyter_client.restarter import KernelRestarter from traitlets import ( Instance, ) -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - class IOLoopKernelRestarter(KernelRestarter): """Monitor and autorestart a kernel.""" - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + warnings.warn("IOLoopKernelRestarter.loop is deprecated in jupyter-client 5.2", + DeprecationWarning, stacklevel=4, + ) + return ioloop.IOLoop.current() _pcallback = None @@ -42,7 +33,7 @@ def start(self): """Start the polling of the kernel.""" if self._pcallback is None: self._pcallback = ioloop.PeriodicCallback( - self.poll, 1000*self.time_to_dead, self.loop + self.poll, 1000*self.time_to_dead, ) self._pcallback.start() diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py new file mode 100644 index 000000000..a2ab17812 --- /dev/null +++ b/jupyter_client/kernelapp.py @@ -0,0 +1,81 @@ +import os +import signal +import uuid + +from jupyter_core.application import JupyterApp, base_flags +from tornado.ioloop import IOLoop +from traitlets import Unicode + +from . import __version__ +from .kernelspec import KernelSpecManager, NATIVE_KERNEL_NAME +from .manager import KernelManager + +class KernelApp(JupyterApp): + """Launch a kernel by name in a local subprocess. + """ + version = __version__ + description = "Run a kernel locally in a subprocess" + + classes = [KernelManager, KernelSpecManager] + + aliases = { + 'kernel': 'KernelApp.kernel_name', + 'ip': 'KernelManager.ip', + } + flags = {'debug': base_flags['debug']} + + kernel_name = Unicode(NATIVE_KERNEL_NAME, + help = 'The name of a kernel type to start' + ).tag(config=True) + + def initialize(self, argv=None): + super(KernelApp, self).initialize(argv) + self.km = KernelManager(kernel_name=self.kernel_name, + config=self.config) + cf_basename = 'kernel-%s.json' % uuid.uuid4() + self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) + self.loop = IOLoop.current() + self.loop.add_callback(self._record_started) + + def setup_signals(self): + """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" + if os.name == 'nt': + return + + def shutdown_handler(signo, frame): + self.loop.add_callback_from_signal(self.shutdown, signo) + for sig in [signal.SIGTERM, signal.SIGINT]: + signal.signal(sig, shutdown_handler) + + def shutdown(self, signo): + self.log.info('Shutting down on signal %d' % signo) + self.km.shutdown_kernel() + self.loop.stop() + + def log_connection_info(self): + cf = self.km.connection_file + self.log.info('Connection file: %s', cf) + self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + + def _record_started(self): + """For tests, create a file to indicate that we've started + + Do not rely on this except in our own tests! + """ + fn = os.environ.get('JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE') + if fn is not None: + with open(fn, 'wb'): + pass + + def start(self): + self.log.info('Starting kernel %r', self.kernel_name) + try: + self.km.start_kernel() + self.log_connection_info() + self.setup_signals() + self.loop.start() + finally: + self.km.cleanup() + + +main = KernelApp.launch_instance diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 3465ac7a4..78a5b564c 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -3,6 +3,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import errno import io import json import os @@ -13,7 +14,9 @@ pjoin = os.path.join from ipython_genutils.py3compat import PY3 -from traitlets import HasTraits, List, Unicode, Dict, Set, Bool, Type +from traitlets import ( + HasTraits, List, Unicode, Dict, Set, Bool, Type, CaselessStrEnum +) from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH @@ -28,6 +31,9 @@ class KernelSpec(HasTraits): language = Unicode() env = Dict() resource_dir = Unicode() + interrupt_mode = CaselessStrEnum( + ['message', 'signal'], default_value='signal' + ) metadata = Dict() @classmethod @@ -46,6 +52,7 @@ def to_dict(self): env=self.env, display_name=self.display_name, language=self.language, + interrupt_mode=self.interrupt_mode, metadata=self.metadata, ) @@ -193,15 +200,39 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): return self.kernel_spec_class.from_resource_dir(resource_dir) + def _find_spec_directory(self, kernel_name): + """Find the resource directory of a named kernel spec""" + for kernel_dir in self.kernel_dirs: + try: + files = os.listdir(kernel_dir) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.ENOENT): + continue + raise + for f in files: + path = pjoin(kernel_dir, f) + if f.lower() == kernel_name and _is_kernel_dir(path): + return path + + if kernel_name == NATIVE_KERNEL_NAME: + try: + from ipykernel.kernelspec import RESOURCES + except ImportError: + pass + else: + return RESOURCES + def get_kernel_spec(self, kernel_name): """Returns a :class:`KernelSpec` instance for the given kernel_name. Raises :exc:`NoSuchKernel` if the given kernel name is not found. """ - d = self.find_kernel_specs() - try: - resource_dir = d[kernel_name.lower()] - except KeyError: + if not _is_valid_kernel_name(kernel_name): + self.log.warning("Kernelspec name %r is invalid: %s", kernel_name, + _kernel_name_description) + + resource_dir = self._find_spec_directory(kernel_name.lower()) + if resource_dir is None: raise NoSuchKernel(kernel_name) return self._get_kernel_spec_by_name(kernel_name, resource_dir) @@ -220,14 +251,28 @@ def get_all_specs(self): } """ d = self.find_kernel_specs() - return {kname: { - "resource_dir": d[kname], - "spec": self._get_kernel_spec_by_name(kname, d[kname]).to_dict() - } for kname in d} + res = {} + for kname, resource_dir in d.items(): + try: + if self.__class__ is KernelSpecManager: + spec = self._get_kernel_spec_by_name(kname, resource_dir) + else: + # avoid calling private methods in subclasses, + # which may have overridden find_kernel_specs + # and get_kernel_spec, but not the newer get_all_specs + spec = self.get_kernel_spec(kname) + + res[kname] = { + "resource_dir": resource_dir, + "spec": spec.to_dict() + } + except Exception: + self.log.warning("Error loading kernelspec %r", kname, exc_info=True) + return res def remove_kernel_spec(self, name): """Remove a kernel spec directory by name. - + Returns the path that was deleted. """ save_native = self.ensure_native_kernel @@ -263,7 +308,7 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, If ``user`` is False, it will attempt to install into the systemwide kernel registry. If the process does not have appropriate permissions, an :exc:`OSError` will be raised. - + If ``prefix`` is given, the kernelspec will be installed to PREFIX/share/jupyter/kernels/KERNEL_NAME. This can be sys.prefix for installation inside virtual or conda envs. @@ -284,16 +329,16 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, DeprecationWarning, stacklevel=2, ) - + destination = self._get_destination_dir(kernel_name, user=user, prefix=prefix) self.log.debug('Installing kernelspec in %s', destination) - + kernel_dir = os.path.dirname(destination) if kernel_dir not in self.kernel_dirs: self.log.warning("Installing to %s, which is not in %s. The kernelspec may not be found.", kernel_dir, self.kernel_dirs, ) - + if os.path.isdir(destination): self.log.info('Removing existing kernelspec in %s', destination) shutil.rmtree(destination) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 285778a68..33f35a7c7 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -111,6 +111,10 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, DUPLICATE_SAME_ACCESS) env['JPY_PARENT_PID'] = str(int(handle)) + # Prevent creating new console window on pythonw + if redirect_out: + kwargs['creationflags'] = kwargs.setdefault('creationflags', 0) | 0x08000000 # CREATE_NO_WINDOW + else: # Create a new session. # This makes it easier to interrupt the kernel, diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index d50a5fbb8..bf94ad002 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -11,25 +11,19 @@ import signal import sys import time -import warnings -try: - from queue import Empty # Py 3 -except ImportError: - from Queue import Empty # Py 2 import zmq from ipython_genutils.importstring import import_item from .localinterfaces import is_local_ip, local_ips from traitlets import ( - Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName + Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName, Dict ) from jupyter_client import ( launch_kernel, kernelspec, ) from .connect import ConnectionFileMixin -from .session import Session from .managerabc import ( KernelManagerABC ) @@ -87,23 +81,13 @@ def kernel_spec(self): self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec - kernel_cmd = List(Unicode(), config=True, - help="""DEPRECATED: Use kernel_name instead. - - The Popen Command to launch the kernel. - Override this if you have a custom kernel. - If kernel_cmd is specified in a configuration file, - Jupyter does not pass any arguments to the kernel, - because it cannot make any assumptions about the - arguments that the kernel understands. In particular, - this means that the kernel does not receive the - option --debug if it given on the Jupyter command line. - """ + kernel_cmd = List(Unicode(), + help="""The Popen Command to launch the kernel.""" ) - def _kernel_cmd_changed(self, name, old, new): - warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " - "start different kernels.") + extra_env = Dict( + help="""Extra environment variables to be set for the kernel.""" + ) @property def ipykernel(self): @@ -174,8 +158,10 @@ def format_kernel_cmd(self, extra_arguments=None): else: cmd = self.kernel_spec.argv + extra_arguments - if cmd and cmd[0] == 'python': - # executable is 'python', use sys.executable. + if cmd and cmd[0] in {'python', + 'python%i' % sys.version_info[0], + 'python%i.%i' % sys.version_info[:2]}: + # executable is 'python' or 'python3', use sys.executable. # These will typically be the same, # but if the current process is in an env # and has been launched by abspath without @@ -233,9 +219,10 @@ def start_kernel(self, **kw): """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " + "This one is not: %s." "Make sure that the '*_address' attributes are " "configured properly. " - "Currently valid addresses are: %s" % local_ips() + "Currently valid addresses are: %s" % (self.ip, local_ips()) ) # write connection file / get default ports @@ -254,7 +241,9 @@ def start_kernel(self, **kw): # If kernel_cmd has been set manually, don't refer to a kernel spec # Environment variables from kernel spec are added to os.environ env.update(self.kernel_spec.env or {}) - + elif self.extra_env: + env.update(self.extra_env) + # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = self._launch_kernel(kernel_cmd, env=env, @@ -283,6 +272,10 @@ def finish_shutdown(self, waittime=None, pollinterval=0.1): if self.is_alive(): time.sleep(pollinterval) else: + # If there's still a proc, wait and clear + if self.has_kernel: + self.kernel.wait() + self.kernel = None break else: # OK, we've waited long enough. @@ -297,6 +290,7 @@ def cleanup(self, connection_file=True): self.cleanup_ipc_files() self._close_control_socket() + self.session.parent = None def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly. @@ -384,7 +378,10 @@ def _kill_kernel(self): # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: - self.kernel.kill() + if hasattr(signal, 'SIGKILL'): + self.signal_kernel(signal.SIGKILL) + else: + self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. @@ -411,11 +408,18 @@ def interrupt_kernel(self): platforms. """ if self.has_kernel: - if sys.platform == 'win32': - from .win_interrupt import send_interrupt - send_interrupt(self.kernel.win32_interrupt_event) - else: - self.signal_kernel(signal.SIGINT) + interrupt_mode = self.kernel_spec.interrupt_mode + if interrupt_mode == 'signal': + if sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.kernel.win32_interrupt_event) + else: + self.signal_kernel(signal.SIGINT) + + elif interrupt_mode == 'message': + msg = self.session.msg("interrupt_request", content={}) + self._connect_control_socket() + self.session.send(self._control_socket, msg) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index a83be953c..122e24cdb 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -6,6 +6,7 @@ from __future__ import absolute_import import os +import sys import uuid import zmq @@ -47,7 +48,7 @@ class MultiKernelManager(LoggingConfigurable): ) kernel_spec_manager = Instance(KernelSpecManager, allow_none=True) - + kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.IOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow @@ -82,15 +83,12 @@ def __len__(self): def __contains__(self, kernel_id): return kernel_id in self._kernels - def start_kernel(self, kernel_name=None, **kwargs): - """Start a new kernel. - - The caller can pick a kernel_id by passing one in as a keyword arg, - otherwise one will be picked using a uuid. - - The kernel ID for the newly started kernel is returned. + def _start_kernel(self, kernel_name, **kwargs): + """ + Core logic of start_kernel which is shared between the synchronous + `start_kernel` method and the asynchronous `start_kernel_async`. """ - kernel_id = kwargs.pop('kernel_id', unicode_type(uuid.uuid4())) + kernel_id = kwargs.pop('kernel_id', self.new_kernel_id(**kwargs)) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) @@ -107,7 +105,21 @@ def start_kernel(self, kernel_name=None, **kwargs): parent=self, log=self.log, kernel_name=kernel_name, **constructor_kwargs ) + + return kernel_id, kernel_name, km + + def start_kernel(self, kernel_name=None, **kwargs): + """Start a new kernel. + + The caller can pick a kernel_id by passing one in as a keyword arg, + otherwise one will be generated using new_kernel_id(). + + The kernel ID for the newly started kernel is returned. + """ + kernel_id, kernel_name, km = self._start_kernel(kernel_name, **kwargs) + km.start_kernel(**kwargs) + self._kernels[kernel_id] = km return kernel_id @@ -315,3 +327,19 @@ def connect_hb(self, kernel_id, identity=None): ======= stream : zmq Socket or ZMQStream """ + + def new_kernel_id(self, **kwargs): + """ + Returns the id to associate with the kernel for this request. Subclasses may override + this method to substitute other sources of kernel ids. + :param kwargs: + :return: string-ized version 4 uuid + """ + return unicode_type(uuid.uuid4()) + +if sys.version_info >= (3, 5): + from .multikernelmanager_async_patch import start_kernel_async + # this looks weird as we are extra indented, but we are + # indeed defining a conditional method depending on the version + # of Python. + MultiKernelManager.start_kernel_async = start_kernel_async diff --git a/jupyter_client/multikernelmanager_async_patch.py b/jupyter_client/multikernelmanager_async_patch.py new file mode 100644 index 000000000..d6ddefdbb --- /dev/null +++ b/jupyter_client/multikernelmanager_async_patch.py @@ -0,0 +1,27 @@ +""" +Patch for jupyter_client.MultiKernelManager which is invalid syntax on +Python 2.7, so need to be in a separate file and conditionally patched. +""" + +import asyncio +from inspect import iscoroutinefunction + +@asyncio.coroutine +def start_kernel_async(self, kernel_name=None, **kwargs): + """Start a new kernel. + + The caller can pick a kernel_id by passing one in as a keyword arg, + otherwise one will be generated using new_kernel_id(). + + The kernel ID for the newly started kernel is returned. + """ + + kernel_id, kernel_name, km = self._start_kernel(kernel_name, **kwargs) + + if iscoroutinefunction(km.start_kernel): + yield from km.start_kernel(**kwargs) + else: + km.start_kernel(**kwargs) + + self._kernels[kernel_id] = km + return kernel_id diff --git a/jupyter_client/session.py b/jupyter_client/session.py index af60ac259..38166bcd5 100644 --- a/jupyter_client/session.py +++ b/jupyter_client/session.py @@ -191,9 +191,9 @@ def _context_default(self): session = Instance('jupyter_client.session.Session', allow_none=True) - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return IOLoop.instance() + return IOLoop.current() def __init__(self, **kwargs): super(SessionFactory, self).__init__(**kwargs) @@ -779,7 +779,8 @@ def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): to_send.extend(ident) to_send.append(DELIM) - to_send.append(self.sign(msg_list)) + # Don't include buffers in signature (per spec). + to_send.append(self.sign(msg_list[0:4])) to_send.extend(msg_list) stream.send_multipart(to_send, flags, copy=copy) diff --git a/jupyter_client/ssh/__init__.py b/jupyter_client/ssh/__init__.py new file mode 100644 index 000000000..d7bc9d566 --- /dev/null +++ b/jupyter_client/ssh/__init__.py @@ -0,0 +1 @@ +from jupyter_client.ssh.tunnel import * diff --git a/jupyter_client/ssh/forward.py b/jupyter_client/ssh/forward.py new file mode 100644 index 000000000..a44c11769 --- /dev/null +++ b/jupyter_client/ssh/forward.py @@ -0,0 +1,92 @@ +# +# This file is adapted from a paramiko demo, and thus licensed under LGPL 2.1. +# Original Copyright (C) 2003-2007 Robey Pointer +# Edits Copyright (C) 2010 The IPython Team +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA. + +""" +Sample script showing how to do local port forwarding over paramiko. + +This script connects to the requested SSH server and sets up local port +forwarding (the openssh -L option) from a local port through a tunneled +connection to a destination reachable from the SSH server machine. +""" + +from __future__ import print_function + +import logging +import select +try: # Python 3 + import socketserver +except ImportError: # Python 2 + import SocketServer as socketserver + +logger = logging.getLogger('ssh') + + +class ForwardServer (socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + +class Handler (socketserver.BaseRequestHandler): + + def handle(self): + try: + chan = self.ssh_transport.open_channel('direct-tcpip', + (self.chain_host, self.chain_port), + self.request.getpeername()) + except Exception as e: + logger.debug('Incoming request to %s:%d failed: %s' % (self.chain_host, + self.chain_port, + repr(e))) + return + if chan is None: + logger.debug('Incoming request to %s:%d was rejected by the SSH server.' % + (self.chain_host, self.chain_port)) + return + + logger.debug('Connected! Tunnel open %r -> %r -> %r' % (self.request.getpeername(), + chan.getpeername(), (self.chain_host, self.chain_port))) + while True: + r, w, x = select.select([self.request, chan], [], []) + if self.request in r: + data = self.request.recv(1024) + if len(data) == 0: + break + chan.send(data) + if chan in r: + data = chan.recv(1024) + if len(data) == 0: + break + self.request.send(data) + chan.close() + self.request.close() + logger.debug('Tunnel closed ') + + +def forward_tunnel(local_port, remote_host, remote_port, transport): + # this is a little convoluted, but lets me configure things for the Handler + # object. (SocketServer doesn't give Handlers any way to access the outer + # server normally.) + class SubHander (Handler): + chain_host = remote_host + chain_port = remote_port + ssh_transport = transport + ForwardServer(('127.0.0.1', local_port), SubHander).serve_forever() + + +__all__ = ['forward_tunnel'] diff --git a/jupyter_client/ssh/tunnel.py b/jupyter_client/ssh/tunnel.py new file mode 100644 index 000000000..e1cd08027 --- /dev/null +++ b/jupyter_client/ssh/tunnel.py @@ -0,0 +1,375 @@ +"""Basic ssh tunnel utilities, and convenience functions for tunneling +zeromq connections. +""" + +# Copyright (C) 2010-2011 IPython Development Team +# Copyright (C) 2011- PyZMQ Developers +# +# Redistributed from IPython under the terms of the BSD License. + + +from __future__ import print_function + +import atexit +import os +import re +import signal +import socket +import sys +import warnings +from getpass import getpass, getuser +from multiprocessing import Process + +try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import paramiko + SSHException = paramiko.ssh_exception.SSHException +except ImportError: + paramiko = None + class SSHException(Exception): + pass +else: + from .forward import forward_tunnel + +try: + import pexpect +except ImportError: + pexpect = None + +from zmq.utils.strtypes import b + + +def select_random_ports(n): + """Select and return n random ports that are available.""" + ports = [] + sockets = [] + for i in range(n): + sock = socket.socket() + sock.bind(('', 0)) + ports.append(sock.getsockname()[1]) + sockets.append(sock) + for sock in sockets: + sock.close() + return ports + + +#----------------------------------------------------------------------------- +# Check for passwordless login +#----------------------------------------------------------------------------- +_password_pat = re.compile(b(r'pass(word|phrase):'), re.IGNORECASE) + + +def try_passwordless_ssh(server, keyfile, paramiko=None): + """Attempt to make an ssh connection without a password. + This is mainly used for requiring password input only once + when many tunnels may be connected to the same server. + + If paramiko is None, the default for the platform is chosen. + """ + if paramiko is None: + paramiko = sys.platform == 'win32' + if not paramiko: + f = _try_passwordless_openssh + else: + f = _try_passwordless_paramiko + return f(server, keyfile) + + +def _try_passwordless_openssh(server, keyfile): + """Try passwordless login with shell ssh command.""" + if pexpect is None: + raise ImportError("pexpect unavailable, use paramiko") + cmd = 'ssh -f ' + server + if keyfile: + cmd += ' -i ' + keyfile + cmd += ' exit' + + # pop SSH_ASKPASS from env + env = os.environ.copy() + env.pop('SSH_ASKPASS', None) + + ssh_newkey = 'Are you sure you want to continue connecting' + p = pexpect.spawn(cmd, env=env) + while True: + try: + i = p.expect([ssh_newkey, _password_pat], timeout=.1) + if i == 0: + raise SSHException('The authenticity of the host can\'t be established.') + except pexpect.TIMEOUT: + continue + except pexpect.EOF: + return True + else: + return False + + +def _try_passwordless_paramiko(server, keyfile): + """Try passwordless login with paramiko.""" + if paramiko is None: + msg = "Paramiko unavailable, " + if sys.platform == 'win32': + msg += "Paramiko is required for ssh tunneled connections on Windows." + else: + msg += "use OpenSSH." + raise ImportError(msg) + username, server, port = _split_server(server) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + try: + client.connect(server, port, username=username, key_filename=keyfile, + look_for_keys=True) + except paramiko.AuthenticationException: + return False + else: + client.close() + return True + + +def tunnel_connection(socket, addr, server, keyfile=None, password=None, paramiko=None, timeout=60): + """Connect a socket to an address via an ssh tunnel. + + This is a wrapper for socket.connect(addr), when addr is not accessible + from the local machine. It simply creates an ssh tunnel using the remaining args, + and calls socket.connect('tcp://localhost:lport') where lport is the randomly + selected local port of the tunnel. + + """ + new_url, tunnel = open_tunnel(addr, server, keyfile=keyfile, password=password, paramiko=paramiko, timeout=timeout) + socket.connect(new_url) + return tunnel + + +def open_tunnel(addr, server, keyfile=None, password=None, paramiko=None, timeout=60): + """Open a tunneled connection from a 0MQ url. + + For use inside tunnel_connection. + + Returns + ------- + + (url, tunnel) : (str, object) + The 0MQ url that has been forwarded, and the tunnel object + """ + + lport = select_random_ports(1)[0] + transport, addr = addr.split('://') + ip, rport = addr.split(':') + rport = int(rport) + if paramiko is None: + paramiko = sys.platform == 'win32' + if paramiko: + tunnelf = paramiko_tunnel + else: + tunnelf = openssh_tunnel + + tunnel = tunnelf(lport, rport, server, remoteip=ip, keyfile=keyfile, password=password, timeout=timeout) + return 'tcp://127.0.0.1:%i' % lport, tunnel + + +def openssh_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): + """Create an ssh tunnel using command-line ssh that connects port lport + on this machine to localhost:rport on server. The tunnel + will automatically close when not in use, remaining open + for a minimum of timeout seconds for an initial connection. + + This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, + as seen from `server`. + + keyfile and password may be specified, but ssh config is checked for defaults. + + Parameters + ---------- + + lport : int + local port for connecting to the tunnel from this machine. + rport : int + port on the remote machine to connect to. + server : str + The ssh server to connect to. The full ssh server string will be parsed. + user@server:port + remoteip : str [Default: 127.0.0.1] + The remote ip, specifying the destination of the tunnel. + Default is localhost, which means that the tunnel would redirect + localhost:lport on this machine to localhost:rport on the *server*. + + keyfile : str; path to public key file + This specifies a key to be used in ssh login, default None. + Regular default ssh keys will be used without specifying this argument. + password : str; + Your ssh password to the ssh server. Note that if this is left None, + you will be prompted for it if passwordless key based login is unavailable. + timeout : int [default: 60] + The time (in seconds) after which no activity will result in the tunnel + closing. This prevents orphaned tunnels from running forever. + """ + if pexpect is None: + raise ImportError("pexpect unavailable, use paramiko_tunnel") + ssh = "ssh " + if keyfile: + ssh += "-i " + keyfile + + if ':' in server: + server, port = server.split(':') + ssh += " -p %s" % port + + cmd = "%s -O check %s" % (ssh, server) + (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) + if not exitstatus: + pid = int(output[output.find(b"(pid=")+5:output.find(b")")]) + cmd = "%s -O forward -L 127.0.0.1:%i:%s:%i %s" % ( + ssh, lport, remoteip, rport, server) + (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) + if not exitstatus: + atexit.register(_stop_tunnel, cmd.replace("-O forward", "-O cancel", 1)) + return pid + cmd = "%s -f -S none -L 127.0.0.1:%i:%s:%i %s sleep %i" % ( + ssh, lport, remoteip, rport, server, timeout) + + # pop SSH_ASKPASS from env + env = os.environ.copy() + env.pop('SSH_ASKPASS', None) + + ssh_newkey = 'Are you sure you want to continue connecting' + tunnel = pexpect.spawn(cmd, env=env) + failed = False + while True: + try: + i = tunnel.expect([ssh_newkey, _password_pat], timeout=.1) + if i == 0: + raise SSHException('The authenticity of the host can\'t be established.') + except pexpect.TIMEOUT: + continue + except pexpect.EOF: + if tunnel.exitstatus: + print(tunnel.exitstatus) + print(tunnel.before) + print(tunnel.after) + raise RuntimeError("tunnel '%s' failed to start" % (cmd)) + else: + return tunnel.pid + else: + if failed: + print("Password rejected, try again") + password = None + if password is None: + password = getpass("%s's password: " % (server)) + tunnel.sendline(password) + failed = True + + +def _stop_tunnel(cmd): + pexpect.run(cmd) + + +def _split_server(server): + if '@' in server: + username, server = server.split('@', 1) + else: + username = getuser() + if ':' in server: + server, port = server.split(':') + port = int(port) + else: + port = 22 + return username, server, port + + +def paramiko_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): + """launch a tunner with paramiko in a subprocess. This should only be used + when shell ssh is unavailable (e.g. Windows). + + This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, + as seen from `server`. + + If you are familiar with ssh tunnels, this creates the tunnel: + + ssh server -L localhost:lport:remoteip:rport + + keyfile and password may be specified, but ssh config is checked for defaults. + + + Parameters + ---------- + + lport : int + local port for connecting to the tunnel from this machine. + rport : int + port on the remote machine to connect to. + server : str + The ssh server to connect to. The full ssh server string will be parsed. + user@server:port + remoteip : str [Default: 127.0.0.1] + The remote ip, specifying the destination of the tunnel. + Default is localhost, which means that the tunnel would redirect + localhost:lport on this machine to localhost:rport on the *server*. + + keyfile : str; path to public key file + This specifies a key to be used in ssh login, default None. + Regular default ssh keys will be used without specifying this argument. + password : str; + Your ssh password to the ssh server. Note that if this is left None, + you will be prompted for it if passwordless key based login is unavailable. + timeout : int [default: 60] + The time (in seconds) after which no activity will result in the tunnel + closing. This prevents orphaned tunnels from running forever. + + """ + if paramiko is None: + raise ImportError("Paramiko not available") + + if password is None: + if not _try_passwordless_paramiko(server, keyfile): + password = getpass("%s's password: " % (server)) + + p = Process(target=_paramiko_tunnel, + args=(lport, rport, server, remoteip), + kwargs=dict(keyfile=keyfile, password=password)) + p.daemon = True + p.start() + return p + + +def _paramiko_tunnel(lport, rport, server, remoteip, keyfile=None, password=None): + """Function for actually starting a paramiko tunnel, to be passed + to multiprocessing.Process(target=this), and not called directly. + """ + username, server, port = _split_server(server) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + + try: + client.connect(server, port, username=username, key_filename=keyfile, + look_for_keys=True, password=password) +# except paramiko.AuthenticationException: +# if password is None: +# password = getpass("%s@%s's password: "%(username, server)) +# client.connect(server, port, username=username, password=password) +# else: +# raise + except Exception as e: + print('*** Failed to connect to %s:%d: %r' % (server, port, e)) + sys.exit(1) + + # Don't let SIGINT kill the tunnel subprocess + signal.signal(signal.SIGINT, signal.SIG_IGN) + + try: + forward_tunnel(lport, remoteip, rport, client.get_transport()) + except KeyboardInterrupt: + print('SIGINT: Port forwarding stopped cleanly') + sys.exit(0) + except Exception as e: + print("Port forwarding stopped uncleanly: %s" % e) + sys.exit(255) + + +if sys.platform == 'win32': + ssh_tunnel = paramiko_tunnel +else: + ssh_tunnel = openssh_tunnel + + +__all__ = ['tunnel_connection', 'ssh_tunnel', 'openssh_tunnel', 'paramiko_tunnel', 'try_passwordless_ssh'] diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py new file mode 100644 index 000000000..9d7833ba3 --- /dev/null +++ b/jupyter_client/tests/test_discovery.py @@ -0,0 +1,32 @@ +import sys + +from jupyter_client import KernelManager +from jupyter_client import discovery + +def test_ipykernel_provider(): + import ipykernel # Fail clearly if ipykernel not installed + ikf = discovery.IPykernelProvider() + + res = list(ikf.find_kernels()) + assert len(res) == 1, res + id, info = res[0] + assert id == 'kernel' + assert info['argv'][0] == sys.executable + +class DummyKernelProvider(discovery.KernelProviderBase): + """A dummy kernel provider for testing KernelFinder""" + id = 'dummy' + + def find_kernels(self): + yield 'sample', {'argv': ['dummy_kernel']} + + def make_manager(self, name): + return KernelManager(kernel_cmd=['dummy_kernel']) + +def test_meta_kernel_finder(): + kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) + assert list(kf.find_kernels()) == \ + [('dummy/sample', {'argv': ['dummy_kernel']})] + + manager = kf.make_manager('dummy/sample') + assert manager.kernel_cmd == ['dummy_kernel'] diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py new file mode 100644 index 000000000..2533472d4 --- /dev/null +++ b/jupyter_client/tests/test_kernelapp.py @@ -0,0 +1,67 @@ +from __future__ import division + +import os +import shutil +from subprocess import Popen, PIPE +import sys +from tempfile import mkdtemp +import time + +PY3 = sys.version_info[0] >= 3 + +def _launch(extra_env): + env = os.environ.copy() + env.update(extra_env) + return Popen([sys.executable, '-c', + 'from jupyter_client.kernelapp import main; main()'], + env=env, stderr=(PIPE if PY3 else None)) + +WAIT_TIME = 10 +POLL_FREQ = 10 + +def hacky_wait(p): + """Python 2 subprocess doesn't have timeouts :-(""" + for _ in range(WAIT_TIME * POLL_FREQ): + if p.poll() is not None: + return p.returncode + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("Process didn't exit in {} seconds" + .format(WAIT_TIME)) + +def test_kernelapp_lifecycle(): + # Check that 'jupyter kernel' starts and terminates OK. + runtime_dir = mkdtemp() + startup_dir = mkdtemp() + started = os.path.join(startup_dir, 'started') + try: + p = _launch({'JUPYTER_RUNTIME_DIR': runtime_dir, + 'JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE': started, + }) + # Wait for start + for _ in range(WAIT_TIME * POLL_FREQ): + if os.path.isfile(started): + break + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("No started file created in {} seconds" + .format(WAIT_TIME)) + + # Connection file should be there by now + files = os.listdir(runtime_dir) + assert len(files) == 1 + cf = files[0] + assert cf.startswith('kernel') + assert cf.endswith('.json') + + # Send SIGTERM to shut down + p.terminate() + if PY3: + _, stderr = p.communicate(timeout=WAIT_TIME) + assert cf in stderr.decode('utf-8', 'replace') + else: + hacky_wait(p) + finally: + shutil.rmtree(runtime_dir) + shutil.rmtree(startup_dir) + diff --git a/jupyter_client/tests/test_kernelspec.py b/jupyter_client/tests/test_kernelspec.py index b2ec4195c..2919923ef 100644 --- a/jupyter_client/tests/test_kernelspec.py +++ b/jupyter_client/tests/test_kernelspec.py @@ -4,6 +4,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import copy import io import json from logging import StreamHandler @@ -11,6 +12,7 @@ from os.path import join as pjoin from subprocess import Popen, PIPE, STDOUT import sys +import tempfile import unittest import pytest @@ -156,7 +158,7 @@ def test_validate_kernel_name(self): 'Haskell-1-2-3', ]: assert kernelspec._is_valid_kernel_name(good) - + for bad in [ 'has space', u'ünicode', @@ -165,4 +167,33 @@ def test_validate_kernel_name(self): ]: assert not kernelspec._is_valid_kernel_name(bad) - + def test_subclass(self): + """Test get_all_specs in subclasses that override find_kernel_specs""" + ksm = self.ksm + resource_dir = tempfile.gettempdir() + native_name = kernelspec.NATIVE_KERNEL_NAME + native_kernel = ksm.get_kernel_spec(native_name) + + class MyKSM(kernelspec.KernelSpecManager): + def get_kernel_spec(self, name): + spec = copy.copy(native_kernel) + if name == 'fake': + spec.name = name + spec.resource_dir = resource_dir + elif name == native_name: + pass + else: + raise KeyError(name) + return spec + + def find_kernel_specs(self): + return { + 'fake': resource_dir, + native_name: native_kernel.resource_dir, + } + + # ensure that get_all_specs doesn't raise if only + # find_kernel_specs and get_kernel_spec are defined + myksm = MyKSM() + specs = myksm.get_all_specs() + assert sorted(specs) == ['fake', native_name] diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index 43819a898..2eec57076 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -8,6 +8,10 @@ import sys import uuid from datetime import datetime +try: + from unittest import mock +except ImportError: + import mock import pytest @@ -34,6 +38,14 @@ def setUp(self): self.session = ss.Session() +@pytest.fixture +def no_copy_threshold(): + """Disable zero-copy optimizations in pyzmq >= 17""" + with mock.patch.object(zmq, 'COPY_THRESHOLD', 1, create=True): + yield + + +@pytest.mark.usefixtures('no_copy_threshold') class TestSession(SessionTestCase): def test_msg(self): @@ -129,7 +141,7 @@ def test_send(self): # buffers must be contiguous buf = memoryview(os.urandom(16)) - if sys.version_info >= (3,3): + if sys.version_info >= (3,4): with self.assertRaises(ValueError): self.session.send(A, msg, ident=b'foo', buffers=[buf[::2]]) @@ -327,7 +339,7 @@ def test_send_raw(self): A.close() B.close() ctx.term() - + def test_clone(self): s = self.session s._add_digest('initial') diff --git a/jupyter_client/tests/test_ssh.py b/jupyter_client/tests/test_ssh.py new file mode 100644 index 000000000..e1673f9f4 --- /dev/null +++ b/jupyter_client/tests/test_ssh.py @@ -0,0 +1,8 @@ +from jupyter_client.ssh.tunnel import select_random_ports + +def test_random_ports(): + for i in range(4096): + ports = select_random_ports(10) + assert len(ports) == 10 + for p in ports: + assert ports.count(p) == 1 diff --git a/jupyter_client/threaded.py b/jupyter_client/threaded.py index f437aa58b..83a6ad0eb 100644 --- a/jupyter_client/threaded.py +++ b/jupyter_client/threaded.py @@ -3,7 +3,8 @@ from __future__ import absolute_import import atexit import errno -from threading import Thread +import sys +from threading import Thread, Event import time # import ZMQError in top-level namespace, to avoid ugly attribute-error messages @@ -41,9 +42,15 @@ def __init__(self, socket, session, loop): self.socket = socket self.session = session self.ioloop = loop + evt = Event() - self.stream = zmqstream.ZMQStream(self.socket, self.ioloop) - self.stream.on_recv(self._handle_recv) + def setup_stream(): + self.stream = zmqstream.ZMQStream(self.socket, self.ioloop) + self.stream.on_recv(self._handle_recv) + evt.set() + + self.ioloop.add_callback(setup_stream) + evt.wait() _is_alive = False def is_alive(self): @@ -142,19 +149,40 @@ class IOLoopThread(Thread): """Run a pyzmq ioloop in a thread to send and receive messages """ _exiting = False + ioloop = None - def __init__(self, loop): + def __init__(self): super(IOLoopThread, self).__init__() self.daemon = True - self.ioloop = loop or ioloop.IOLoop() @staticmethod @atexit.register def _notice_exit(): - IOLoopThread._exiting = True + # Class definitions can be torn down during interpreter shutdown. + # We only need to set _exiting flag if this hasn't happened. + if IOLoopThread is not None: + IOLoopThread._exiting = True + + def start(self): + """Start the IOLoop thread + + Don't return until self.ioloop is defined, + which is created in the thread + """ + self._start_event = Event() + Thread.start(self) + self._start_event.wait() def run(self): """Run my loop, ignoring EINTR events in the poller""" + if 'asyncio' in sys.modules: + # tornado may be using asyncio, + # ensure an eventloop exists for this thread + import asyncio + asyncio.set_event_loop(asyncio.new_event_loop()) + self.ioloop = ioloop.IOLoop() + # signal that self.ioloop is defined + self._start_event.set() while True: try: self.ioloop.start() @@ -179,9 +207,10 @@ def stop(self): :meth:`~threading.Thread.start` is called again. """ if self.ioloop is not None: - self.ioloop.stop() + self.ioloop.add_callback(self.ioloop.stop) self.join() self.close() + self.ioloop = None def close(self): if self.ioloop is not None: @@ -195,22 +224,19 @@ class ThreadedKernelClient(KernelClient): """ A KernelClient that provides thread-safe sockets with async callbacks on message replies. """ - _ioloop = None @property def ioloop(self): - if self._ioloop is None: - self._ioloop = ioloop.IOLoop() - return self._ioloop + return self.ioloop_thread.ioloop ioloop_thread = Instance(IOLoopThread, allow_none=True) def start_channels(self, shell=True, iopub=True, stdin=True, hb=True): + self.ioloop_thread = IOLoopThread() + self.ioloop_thread.start() + if shell: self.shell_channel._inspect = self._check_kernel_info_reply - self.ioloop_thread = IOLoopThread(self.ioloop) - self.ioloop_thread.start() - super(ThreadedKernelClient, self).start_channels(shell, iopub, stdin, hb) def _check_kernel_info_reply(self, msg): diff --git a/scripts/jupyter-kernel b/scripts/jupyter-kernel new file mode 100755 index 000000000..31144d405 --- /dev/null +++ b/scripts/jupyter-kernel @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from jupyter_client.kernelapp import main + +if __name__ == '__main__': + main() diff --git a/setup.cfg b/setup.cfg index e0ca7a784..a2327e90f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,8 @@ [bdist_wheel] universal=1 +[metadata] +license_file = COPYING.md + [nosetests] warningfilters=default diff --git a/setup.py b/setup.py index 341af7fb2..c641e7469 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ import sys v = sys.version_info -if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,3)): - error = "ERROR: %s requires Python version 2.7 or 3.3 or above." % name +if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,4)): + error = "ERROR: %s requires Python version 2.7 or 3.4 or above." % name print(error, file=sys.stderr) sys.exit(1) @@ -60,31 +60,42 @@ def run(self): name = name, version = version_ns['__version__'], packages = packages, - description = "Jupyter protocol implementation and client libraries", + description = 'Jupyter protocol implementation and client libraries', author = 'Jupyter Development Team', author_email = 'jupyter@googlegroups.com', url = 'https://jupyter.org', license = 'BSD', platforms = "Linux, Mac OS X, Windows", keywords = ['Interactive', 'Interpreter', 'Shell', 'Web'], + project_urls = { + 'Documentation': 'https://jupyter-client.readthedocs.io', + 'Source': 'https://github.com/jupyter/jupyter_client/', + 'Tracker': 'https://github.com/jupyter/jupyter_client/issues', + }, classifiers = [ + 'Framework :: Jupyter', 'Intended Audience :: Developers', + 'Intended Audience :: Education', 'Intended Audience :: System Administrators', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', ], install_requires = [ 'traitlets', 'jupyter_core', 'pyzmq>=13', 'python-dateutil>=2.1', + 'entrypoints', + 'tornado>=4.1', ], + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', extras_require = { - 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test': ['ipykernel', 'ipython', 'mock'], + 'test:(python_version >= "3.4" or python_version == "2.7")': ['pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, @@ -93,6 +104,11 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + 'jupyter-kernel = jupyter_client.kernelapp:main', + ], + 'jupyter_client.kernel_providers' : [ + 'spec = jupyter_client.discovery:KernelSpecProvider', + 'pyimport = jupyter_client.discovery:IPykernelProvider', ] }, )