From 250178fe53dcf5c20098e29ea94951caa3aa371e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 10 Feb 2017 16:57:22 +0000 Subject: [PATCH 1/7] Add 'jupyter kernel' command A simple lead in to the 'kernel nanny' work, this adds a command so you can do: jupyter kernel --kernel python --- jupyter_client/kernelapp.py | 66 +++++++++++++++++++++++++++++++++++++ scripts/jupyter-kernel | 5 +++ setup.py | 1 + 3 files changed, 72 insertions(+) create mode 100644 jupyter_client/kernelapp.py create mode 100755 scripts/jupyter-kernel diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py new file mode 100644 index 000000000..4c1c99e3c --- /dev/null +++ b/jupyter_client/kernelapp.py @@ -0,0 +1,66 @@ +import os +import signal +import uuid + +from jupyter_core.application import JupyterApp +from tornado.ioloop import IOLoop +from traitlets import Unicode + +from . import __version__ +from .kernelspec import KernelSpecManager +from .manager import KernelManager + +class KernelApp(JupyterApp): + version = __version__ + description = "Run a kernel locally" + + classes = [KernelManager, KernelSpecManager] + + aliases = { + 'kernel': 'KernelApp.kernel_name', + 'ip': 'KernelManager.ip', + } + + kernel_name = Unicode( + help = 'The name of a kernel to start' + ).tag(config=True) + + def initialize(self, argv=None): + super(KernelApp, self).initialize(argv) + self.km = KernelManager(kernel_name=self.kernel_name, + config=self.config) + cf_basename = 'kernel-%s.json' % uuid.uuid4() + self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) + self.loop = IOLoop.current() + + def setup_signals(self): + if os.name == 'nt': + return + + def shutdown_handler(signo, frame): + self.loop.add_callback_from_signal(self.shutdown, signo) + for sig in [signal.SIGTERM, signal.SIGINT]: + signal.signal(sig, shutdown_handler) + + def shutdown(self, signo): + self.log.info('Shutting down on signal %d' % signo) + self.km.shutdown_kernel() + self.loop.stop() + + def log_connection_info(self): + cf = self.km.connection_file + self.log.info('Connection file: %s', cf) + self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + + def start(self): + self.log.info('Starting kernel %r', self.kernel_name) + try: + self.km.start_kernel() + self.log_connection_info() + self.setup_signals() + self.loop.start() + finally: + self.km.cleanup() + + +main = KernelApp.launch_instance diff --git a/scripts/jupyter-kernel b/scripts/jupyter-kernel new file mode 100755 index 000000000..31144d405 --- /dev/null +++ b/scripts/jupyter-kernel @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from jupyter_client.kernelapp import main + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index f042f00b3..022cbc56e 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + 'jupyter-kernel = jupyter_client.kernelapp:main', ], 'jupyter_client.kernel_providers' : [ 'spec = jupyter_client.discovery:KernelSpecProvider', From 9359b338c90f8e259ac4b307d99ce20ca3b2cbf7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:23:18 +0000 Subject: [PATCH 2/7] Use native kernel by default --- jupyter_client/kernelapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 4c1c99e3c..071a0f3ed 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -7,7 +7,7 @@ from traitlets import Unicode from . import __version__ -from .kernelspec import KernelSpecManager +from .kernelspec import KernelSpecManager, NATIVE_KERNEL_NAME from .manager import KernelManager class KernelApp(JupyterApp): @@ -21,7 +21,7 @@ class KernelApp(JupyterApp): 'ip': 'KernelManager.ip', } - kernel_name = Unicode( + kernel_name = Unicode(NATIVE_KERNEL_NAME, help = 'The name of a kernel to start' ).tag(config=True) From ae03ddde10c215a8df1efe4a29d5bfa91b1efdfa Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:32:40 +0000 Subject: [PATCH 3/7] More description --- jupyter_client/kernelapp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 071a0f3ed..799d85ee4 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -2,7 +2,7 @@ import signal import uuid -from jupyter_core.application import JupyterApp +from jupyter_core.application import JupyterApp, base_flags from tornado.ioloop import IOLoop from traitlets import Unicode @@ -11,8 +11,10 @@ from .manager import KernelManager class KernelApp(JupyterApp): + """Launch a kernel by name in a local subprocess. + """ version = __version__ - description = "Run a kernel locally" + description = "Run a kernel locally in a subprocess" classes = [KernelManager, KernelSpecManager] @@ -20,9 +22,10 @@ class KernelApp(JupyterApp): 'kernel': 'KernelApp.kernel_name', 'ip': 'KernelManager.ip', } + flags = {'debug': base_flags['debug']} kernel_name = Unicode(NATIVE_KERNEL_NAME, - help = 'The name of a kernel to start' + help = 'The name of a kernel type to start' ).tag(config=True) def initialize(self, argv=None): @@ -34,6 +37,7 @@ def initialize(self, argv=None): self.loop = IOLoop.current() def setup_signals(self): + """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" if os.name == 'nt': return From 7e6d16711c6f16782a497dc6dbf76911c334f46e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:22:38 +0000 Subject: [PATCH 4/7] Add test of 'jupyter kernel' --- jupyter_client/kernelapp.py | 11 +++++ jupyter_client/tests/test_kernelapp.py | 57 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 jupyter_client/tests/test_kernelapp.py diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 799d85ee4..a2ab17812 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -35,6 +35,7 @@ def initialize(self, argv=None): cf_basename = 'kernel-%s.json' % uuid.uuid4() self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) self.loop = IOLoop.current() + self.loop.add_callback(self._record_started) def setup_signals(self): """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" @@ -56,6 +57,16 @@ def log_connection_info(self): self.log.info('Connection file: %s', cf) self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + def _record_started(self): + """For tests, create a file to indicate that we've started + + Do not rely on this except in our own tests! + """ + fn = os.environ.get('JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE') + if fn is not None: + with open(fn, 'wb'): + pass + def start(self): self.log.info('Starting kernel %r', self.kernel_name) try: diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py new file mode 100644 index 000000000..b41a02bc6 --- /dev/null +++ b/jupyter_client/tests/test_kernelapp.py @@ -0,0 +1,57 @@ +from __future__ import division + +import os +import shutil +from subprocess import Popen, PIPE +import sys +from tempfile import mkdtemp +import time + +def _launch(extra_env): + env = os.environ.copy() + env.update(extra_env) + return Popen([sys.executable, '-c', + 'from jupyter_client.kernelapp import main; main()'], + env=env, stderr=PIPE) + +WAIT_TIME = 10 +POLL_FREQ = 10 + +def test_kernelapp_lifecycle(): + # Check that 'jupyter kernel' starts and terminates OK. + runtime_dir = mkdtemp() + startup_dir = mkdtemp() + started = os.path.join(startup_dir, 'started') + try: + p = _launch({'JUPYTER_RUNTIME_DIR': runtime_dir, + 'JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE': started, + }) + # Wait for start + for _ in range(WAIT_TIME * POLL_FREQ): + if os.path.isfile(started): + break + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("No started file created in {} seconds" + .format(WAIT_TIME)) + + # Connection file should be there by now + files = os.listdir(runtime_dir) + assert len(files) == 1 + cf = files[0] + assert cf.startswith('kernel') + assert cf.endswith('.json') + + # Read the first three lines from stderr. This will hang if there are + # fewer lines to read; I don't see any way to avoid that without lots + # of extra complexity. + b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') + assert cf in b + + # Send SIGTERM to shut down + p.terminate() + p.wait(timeout=10) + finally: + shutil.rmtree(runtime_dir) + shutil.rmtree(startup_dir) + From 28f908f0da34ed0e0c85f58016334c434c18bb5f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:32:35 +0000 Subject: [PATCH 5/7] Workaround lack of timeout on Py2 --- jupyter_client/tests/test_kernelapp.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py index b41a02bc6..2533472d4 100644 --- a/jupyter_client/tests/test_kernelapp.py +++ b/jupyter_client/tests/test_kernelapp.py @@ -7,16 +7,28 @@ from tempfile import mkdtemp import time +PY3 = sys.version_info[0] >= 3 + def _launch(extra_env): env = os.environ.copy() env.update(extra_env) return Popen([sys.executable, '-c', 'from jupyter_client.kernelapp import main; main()'], - env=env, stderr=PIPE) + env=env, stderr=(PIPE if PY3 else None)) WAIT_TIME = 10 POLL_FREQ = 10 +def hacky_wait(p): + """Python 2 subprocess doesn't have timeouts :-(""" + for _ in range(WAIT_TIME * POLL_FREQ): + if p.poll() is not None: + return p.returncode + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("Process didn't exit in {} seconds" + .format(WAIT_TIME)) + def test_kernelapp_lifecycle(): # Check that 'jupyter kernel' starts and terminates OK. runtime_dir = mkdtemp() @@ -42,15 +54,13 @@ def test_kernelapp_lifecycle(): assert cf.startswith('kernel') assert cf.endswith('.json') - # Read the first three lines from stderr. This will hang if there are - # fewer lines to read; I don't see any way to avoid that without lots - # of extra complexity. - b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') - assert cf in b - # Send SIGTERM to shut down p.terminate() - p.wait(timeout=10) + if PY3: + _, stderr = p.communicate(timeout=WAIT_TIME) + assert cf in stderr.decode('utf-8', 'replace') + else: + hacky_wait(p) finally: shutil.rmtree(runtime_dir) shutil.rmtree(startup_dir) From aa8b184c9c8134cae731c4652a87a704b6ee9f65 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:34:19 +0000 Subject: [PATCH 6/7] Restrict to older pytest on Python 3.3 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 022cbc56e..1230f2142 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def run(self): ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test:python_version == "3.3"': ['pytest<3.3.0'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From 5291f940c8cac341ed96c6b2dd73bbdd11db1df5 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:36:43 +0000 Subject: [PATCH 7/7] Another go at fixing pytest dependency on Python 3.3 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1230f2142..233f83a0e 100644 --- a/setup.py +++ b/setup.py @@ -85,8 +85,9 @@ def run(self): 'entrypoints', ], extras_require = { - 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], + 'test:python_version >= "3.4" or python_version == "2.7"': ['pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled,