From c0703621148194f36f615d13473a6b249ecc841f Mon Sep 17 00:00:00 2001 From: Roland Weber Date: Thu, 6 Jul 2017 14:56:42 +0200 Subject: [PATCH 1/6] record by name which ports are chosen at random --- jupyter_client/connect.py | 21 +++++++++++++++++++++ jupyter_client/tests/test_connect.py | 13 +++++++++++++ 2 files changed, 34 insertions(+) diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 042904f0e..0b44e209d 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -333,6 +333,9 @@ def _ip_changed(self, name, old, new): control_port = Integer(0, config=True, help="set the control (ROUTER) port [default: random]") + # names of the ports with random assignment + _random_port_names = None + @property def ports(self): return [ getattr(self, name) for name in port_names ] @@ -417,6 +420,22 @@ def cleanup_ipc_files(self): except (IOError, OSError): pass + def _record_random_port_names(self): + """Records which of the ports are randomly assigned. + + Records on first invocation. Does nothing on later invocations.""" + + if self.transport != 'tcp': + return + if self._random_port_names is not None: + return + + self._random_port_names = [] + for name in port_names: + if getattr(self, name) <= 0: + self._random_port_names.append(name) + + def write_connection_file(self): """Write connection info to JSON dict in self.connection_file.""" if self._connection_file_written and os.path.exists(self.connection_file): @@ -431,6 +450,7 @@ def write_connection_file(self): kernel_name=self.kernel_name ) # write_connection_file also sets default ports: + self._record_random_port_names() for name in port_names: setattr(self, name, cfg[name]) @@ -467,6 +487,7 @@ def load_connection_info(self, info): self.transport = info.get('transport', self.transport) self.ip = info.get('ip', self._ip_default()) + self._record_random_port_names() for name in port_names: if getattr(self, name) == 0 and name in info: # not overridden by config or cl_args diff --git a/jupyter_client/tests/test_connect.py b/jupyter_client/tests/test_connect.py index e1985e75e..54c095c8b 100644 --- a/jupyter_client/tests/test_connect.py +++ b/jupyter_client/tests/test_connect.py @@ -21,6 +21,10 @@ def initialize(self, argv=[]): JupyterApp.initialize(self, argv=argv) self.init_connection_file() +class DummyConfigurable(connect.ConnectionFileMixin): + def initialize(self): + pass + sample_info = dict(ip='1.2.3.4', transport='ipc', shell_port=1, hb_port=2, iopub_port=3, stdin_port=4, control_port=5, key=b'abc123', signature_scheme='hmac-md5', kernel_name='python' @@ -31,6 +35,7 @@ def initialize(self, argv=[]): key=b'abc123', signature_scheme='hmac-md5', kernel_name='test' ) + def test_write_connection_file(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') @@ -172,3 +177,11 @@ def test_find_connection_file_abspath(): f.write('{}') assert connect.find_connection_file(abs_cf, path=jupyter_runtime_dir()) == abs_cf + +def test_mixin_record_random_ports(): + with TemporaryDirectory() as d: + dc = DummyConfigurable(data_dir=d, kernel_name='via-tcp', transport='tcp') + dc.write_connection_file() + assert dc._connection_file_written + assert os.path.exists(dc.connection_file) + assert dc._random_port_names == connect.port_names From abbf40e2cc207aa4f8dadce58539190f26f60245 Mon Sep 17 00:00:00 2001 From: Roland Weber Date: Thu, 6 Jul 2017 14:57:21 +0200 Subject: [PATCH 2/6] remove non-temporary file created by a unit test --- jupyter_client/tests/test_connect.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jupyter_client/tests/test_connect.py b/jupyter_client/tests/test_connect.py index 54c095c8b..d0a4c7d32 100644 --- a/jupyter_client/tests/test_connect.py +++ b/jupyter_client/tests/test_connect.py @@ -176,6 +176,7 @@ def test_find_connection_file_abspath(): with open(cf, 'w') as f: f.write('{}') assert connect.find_connection_file(abs_cf, path=jupyter_runtime_dir()) == abs_cf + os.remove(abs_cf) def test_mixin_record_random_ports(): From b6130126270d4b1078947cb3c821e9784b1bcd03 Mon Sep 17 00:00:00 2001 From: Roland Weber Date: Tue, 18 Jul 2017 11:17:19 +0200 Subject: [PATCH 3/6] optionally choose new random ports on kernel restart --- jupyter_client/connect.py | 17 ++++++++++++++++- jupyter_client/manager.py | 16 ++++++++++++---- jupyter_client/tests/test_connect.py | 13 +++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 0b44e209d..afe197688 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -423,7 +423,8 @@ def cleanup_ipc_files(self): def _record_random_port_names(self): """Records which of the ports are randomly assigned. - Records on first invocation. Does nothing on later invocations.""" + Records on first invocation, if the transport is tcp. + Does nothing on later invocations.""" if self.transport != 'tcp': return @@ -435,6 +436,20 @@ def _record_random_port_names(self): if getattr(self, name) <= 0: self._random_port_names.append(name) + def cleanup_random_ports(self): + """Forgets randomly assigned port numbers and cleans up the connection file. + + Does nothing if no port numbers have been randomly assigned. + In particular, does nothing unless the transport is tcp. + """ + + if not self._random_port_names: + return + + for name in self._random_port_names: + setattr(self, name, 0) + + self.cleanup_connection_file() def write_connection_file(self): """Write connection info to JSON dict in self.connection_file.""" diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 373105e20..8d5d053c7 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -326,12 +326,9 @@ def shutdown_kernel(self, now=False, restart=False): self.cleanup(connection_file=not restart) - def restart_kernel(self, now=False, **kw): + def restart_kernel(self, now=False, newports=False, **kw): """Restarts a kernel with the arguments that were used to launch it. - If the old kernel was launched with random ports, the same ports will be - used for the new kernel. The same connection file is used again. - Parameters ---------- now : bool, optional @@ -342,6 +339,14 @@ def restart_kernel(self, now=False, **kw): In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. + newports : bool, optional + If the old kernel was launched with random ports, this flag decides + whether the same ports and connection file will be used again. + If False, the same ports and connection file are used. This is + the default. If True, new random port numbers are chosen and a + new connection file is written. It is still possible that the newly + chosen random port numbers happen to be the same as the old ones. + `**kw` : optional Any options specified here will overwrite those used to launch the kernel. @@ -353,6 +358,9 @@ def restart_kernel(self, now=False, **kw): # Stop currently running kernel. self.shutdown_kernel(now=now, restart=True) + if newports: + self.cleanup_random_ports() + # Start new kernel. self._launch_args.update(kw) self.start_kernel(**self._launch_args) diff --git a/jupyter_client/tests/test_connect.py b/jupyter_client/tests/test_connect.py index d0a4c7d32..efc2e75c9 100644 --- a/jupyter_client/tests/test_connect.py +++ b/jupyter_client/tests/test_connect.py @@ -183,6 +183,19 @@ def test_mixin_record_random_ports(): with TemporaryDirectory() as d: dc = DummyConfigurable(data_dir=d, kernel_name='via-tcp', transport='tcp') dc.write_connection_file() + assert dc._connection_file_written assert os.path.exists(dc.connection_file) assert dc._random_port_names == connect.port_names + + +def test_mixin_cleanup_random_ports(): + with TemporaryDirectory() as d: + dc = DummyConfigurable(data_dir=d, kernel_name='via-tcp', transport='tcp') + dc.write_connection_file() + filename = dc.connection_file + dc.cleanup_random_ports() + + assert not os.path.exists(filename) + for name in dc._random_port_names: + assert getattr(dc, name) == 0 From 7d00686b71298cd6bd307d89398866c43ad21b79 Mon Sep 17 00:00:00 2001 From: Roland Weber Date: Tue, 18 Jul 2017 13:49:16 +0200 Subject: [PATCH 4/6] restart with new random ports if kernel was never alive --- jupyter_client/restarter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 1c30fcad0..22e2a423d 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -36,6 +36,7 @@ class KernelRestarter(LoggingConfigurable): ) _restarting = Bool(False) _restart_count = Integer(0) + _initial_startup = Bool(True) callbacks = Dict() def _callbacks_default(self): @@ -98,14 +99,17 @@ def poll(self): self._restart_count = 0 self.stop() else: - self.log.info('KernelRestarter: restarting kernel (%i/%i)', + self.log.info('KernelRestarter: restarting kernel (%i/%i), %s random ports', self._restart_count, - self.restart_limit + self.restart_limit, + 'new' if self._initial_startup else 'keep' ) self._fire_callbacks('restart') - self.kernel_manager.restart_kernel(now=True) + self.kernel_manager.restart_kernel(now=True, newports=self._initial_startup) self._restarting = True else: + if self._initial_startup: + self._initial_startup = False if self._restarting: self.log.debug("KernelRestarter: restart apparently succeeded") self._restarting = False From 41dc025bb9f0ce09d052edcd8915ddd8157ac758 Mon Sep 17 00:00:00 2001 From: Roland Weber Date: Mon, 24 Jul 2017 14:46:09 +0200 Subject: [PATCH 5/6] make restarter behavior configurable, default is to keep random ports --- jupyter_client/restarter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 22e2a423d..442cb47d6 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -34,6 +34,10 @@ class KernelRestarter(LoggingConfigurable): restart_limit = Integer(5, config=True, help="""The number of consecutive autorestarts before the kernel is presumed dead.""" ) + + random_ports_until_alive = Bool(False, config=True, + help="""Whether to choose new random ports when restarting before the kernel is alive.""" + ) _restarting = Bool(False) _restart_count = Integer(0) _initial_startup = Bool(True) @@ -99,13 +103,14 @@ def poll(self): self._restart_count = 0 self.stop() else: + newports = self.random_ports_until_alive and self._initial_startup self.log.info('KernelRestarter: restarting kernel (%i/%i), %s random ports', self._restart_count, self.restart_limit, - 'new' if self._initial_startup else 'keep' + 'new' if newports else 'keep' ) self._fire_callbacks('restart') - self.kernel_manager.restart_kernel(now=True, newports=self._initial_startup) + self.kernel_manager.restart_kernel(now=True, newports=newports) self._restarting = True else: if self._initial_startup: From ed7f9d050da5239d8fe27d413f131836613779f7 Mon Sep 17 00:00:00 2001 From: Roland Weber Date: Wed, 9 Aug 2017 13:48:20 +0200 Subject: [PATCH 6/6] switch the default to generate new random ports when restarting early --- jupyter_client/restarter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 442cb47d6..83d4356d0 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -35,7 +35,7 @@ class KernelRestarter(LoggingConfigurable): help="""The number of consecutive autorestarts before the kernel is presumed dead.""" ) - random_ports_until_alive = Bool(False, config=True, + random_ports_until_alive = Bool(True, config=True, help="""Whether to choose new random ports when restarting before the kernel is alive.""" ) _restarting = Bool(False)