From 5baa2e9a0ac0989f8b3a4d8eb16edd9c51ea1c3b Mon Sep 17 00:00:00 2001 From: Min RK Date: Sat, 20 Jun 2015 11:08:25 -0700 Subject: [PATCH 1/6] migrate config at application start write `.jupyter/migrated` to avoid re-running migration on every start --- jupyter_core/application.py | 15 +++++++++++++++ jupyter_core/migrate.py | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/jupyter_core/application.py b/jupyter_core/application.py index ca66b5e..637ec61 100644 --- a/jupyter_core/application.py +++ b/jupyter_core/application.py @@ -143,6 +143,20 @@ def ask(): with open(config_file, mode='w') as f: f.write(config_text) + def migrate_config(self): + """Migrate config/data from IPython 3""" + if os.path.exists(os.path.join(self.config_dir, 'migrated')): + # already migrated + return + + ipdir = os.environ.get('IPYTHONDIR') or os.path.expanduser('~/.ipython') + # No IPython dir, nothing to migrate + if not os.path.exists(ipdir): + return + + from .migrate import migrate + migrate() + def load_config_file(self, suppress_errors=True): """Load the config file. @@ -215,6 +229,7 @@ def initialize(self, argv=None): cl_config = self.config if self._dispatching: return + self.migrate_config() self.load_config_file() # enforce cl-opts override configfile opts: self.update_config(cl_config) diff --git a/jupyter_core/migrate.py b/jupyter_core/migrate.py index 7e80285..bb5ab38 100644 --- a/jupyter_core/migrate.py +++ b/jupyter_core/migrate.py @@ -23,6 +23,7 @@ import os import re import shutil +from datetime import datetime from traitlets.config import PyFileConfigLoader, JSONFileConfigLoader @@ -213,6 +214,10 @@ def migrate(): if os.path.exists(custom_src): migrate_static_custom(custom_src, custom_dst) + # write a marker to avoid re-running migration checks + ensure_dir_exists(env['jupyter_config']) + with open(os.path.join(env['jupyter_config'], 'migrated'), 'w') as f: + f.write(datetime.now().isoformat()) class MigrateApp(JupyterApp): From 9e9f52c898c9c6459fd2804295b868235f5b93f2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Sat, 20 Jun 2015 11:18:08 -0700 Subject: [PATCH 2/6] make default log-level INFO for Jupyter applications this seems to always be what we want --- jupyter_core/application.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jupyter_core/application.py b/jupyter_core/application.py index 637ec61..ae00270 100644 --- a/jupyter_core/application.py +++ b/jupyter_core/application.py @@ -59,6 +59,9 @@ class JupyterApp(Application): aliases = base_aliases flags = base_flags + def _log_level_default(self): + return logging.INFO + jupyter_path = List(Unicode) def _jupyter_path_default(self): return jupyter_path() From db9ef68e963a1c79c09c07e909b1dfe57b734372 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 23 Jun 2015 09:51:36 -0700 Subject: [PATCH 3/6] use logging for migration output --- jupyter_core/migrate.py | 52 +++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/jupyter_core/migrate.py b/jupyter_core/migrate.py index bb5ab38..385c3d8 100644 --- a/jupyter_core/migrate.py +++ b/jupyter_core/migrate.py @@ -26,13 +26,18 @@ from datetime import datetime from traitlets.config import PyFileConfigLoader, JSONFileConfigLoader +from traitlets.log import get_logger from ipython_genutils.path import ensure_dir_exists try: - from IPython.paths import get_ipython_dir, locate_profile + from IPython.paths import get_ipython_dir except ImportError: # IPython < 4 - from IPython.utils.path import get_ipython_dir, locate_profile + try: + from IPython.utils.path import get_ipython_dir + except ImportError: + def get_ipython_dir(): + return os.environ.get('IPYTHONDIR', os.path.expanduser('~/.ipython')) from .paths import jupyter_config_dir, jupyter_data_dir from .application import JupyterApp @@ -65,17 +70,18 @@ def migrate_dir(src, dst): """Migrate a directory from src to dst""" + log = get_logger() if not os.listdir(src): - print("No files in %s" % src) + log.debug("No files in %s" % src) return False if os.path.exists(dst): if os.listdir(dst): # already exists, non-empty - print("%s already exists" % dst) + log.debug("%s already exists" % dst) return False else: os.rmdir(dst) - print("Copying %s -> %s" % (src, dst)) + log.info("Copying %s -> %s" % (src, dst)) ensure_dir_exists(os.path.dirname(dst)) shutil.copytree(src, dst, symlinks=True) return True @@ -86,11 +92,12 @@ def migrate_file(src, dst, substitutions=None): substitutions is an optional dict of {regex: replacement} for performing replacements on the file. """ + log = get_logger() if os.path.exists(dst): # already exists - print("%s already exists" % dst) + log.debug("%s already exists" % dst) return False - print("Copying %s -> %s" % (src, dst)) + log.info("Copying %s -> %s" % (src, dst)) ensure_dir_exists(os.path.dirname(dst)) shutil.copy(src, dst) if substitutions: @@ -108,12 +115,13 @@ def migrate_one(src, dst): dispatches to migrate_dir/_file """ + log = get_logger() if os.path.isfile(src): return migrate_file(src, dst) elif os.path.isdir(src): return migrate_dir(src, dst) else: - print("Nothing to migrate for %s" % src) + log.debug("Nothing to migrate for %s" % src) return False @@ -122,6 +130,7 @@ def migrate_static_custom(src, dst): src, dst are 'custom' directories containing custom.{js,css} """ + log = get_logger() migrated = False custom_js = pjoin(src, 'custom.js') @@ -143,9 +152,9 @@ def migrate_static_custom(src, dst): custom_css_empty = css.startswith('/*') and css.endswith('*/') if custom_js_empty: - print("Ignoring empty %s" % custom_js) + log.debug("Ignoring empty %s" % custom_js) if custom_css_empty: - print("Ignoring empty %s" % custom_css) + log.debug("Ignoring empty %s" % custom_css) if custom_js_empty and custom_css_empty: # nothing to migrate @@ -170,6 +179,7 @@ def migrate_config(name, env): Includes substitutions for updated configurable names. """ + log = get_logger() src_base = pjoin('{profile}', 'ipython_{name}_config').format(name=name, **env) dst_base = pjoin('{jupyter_config}', 'jupyter_{name}_config').format(name=name, **env) loaders = { @@ -187,7 +197,7 @@ def migrate_config(name, env): migrated.append(src) else: # don't migrate empty config files - print("Not migrating empty config file: %s" % src) + log.debug("Not migrating empty config file: %s" % src) return migrated @@ -199,28 +209,35 @@ def migrate(): 'ipython_dir': get_ipython_dir(), 'profile': os.path.join(get_ipython_dir(), 'profile_default'), } + migrated = False for src_t, dst_t in migrations.items(): src = src_t.format(**env) dst = dst_t.format(**env) if os.path.exists(src): if migrate_one(src, dst): - pass + migrated = True for name in config_migrations: - migrate_config(name, env) + if migrate_config(name, env): + migrated = True custom_src = custom_src_t.format(**env) custom_dst = custom_dst_t.format(**env) if os.path.exists(custom_src): - migrate_static_custom(custom_src, custom_dst) + if migrate_static_custom(custom_src, custom_dst): + migrated = True + # write a marker to avoid re-running migration checks ensure_dir_exists(env['jupyter_config']) with open(os.path.join(env['jupyter_config'], 'migrated'), 'w') as f: f.write(datetime.now().isoformat()) + + return migrated + -class MigrateApp(JupyterApp): +class JupyterMigrate(JupyterApp): name = 'jupyter-migrate' description = """ Migrate configuration and data from .ipython prior to 4.0 to Jupyter locations. @@ -239,10 +256,11 @@ class MigrateApp(JupyterApp): """ def start(self): - migrate() + if not migrate(): + self.log.info("Found nothing to migrate.") -main = MigrateApp.launch_instance +main = JupyterMigrate.launch_instance if __name__ == '__main__': From 6c81ad4a0780e9a3b80d48ad6ad5134bf8e41c2d Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 23 Jun 2015 11:49:27 -0700 Subject: [PATCH 4/6] update tests to expect .jupyter/migrated --- jupyter_core/tests/test_migrate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_core/tests/test_migrate.py b/jupyter_core/tests/test_migrate.py index 8685d2b..b83d164 100644 --- a/jupyter_core/tests/test_migrate.py +++ b/jupyter_core/tests/test_migrate.py @@ -199,14 +199,14 @@ def test_migrate_custom_default(td): def test_migrate_nothing(env): migrate() - assert not os.path.exists(env['JUPYTER_CONFIG_DIR']) + assert os.listdir(env['JUPYTER_CONFIG_DIR']) == ['migrated'] assert not os.path.exists(env['JUPYTER_DATA_DIR']) def test_migrate_default(env): shutil.copytree(dotipython_empty, env['IPYTHONDIR']) migrate() - assert not os.path.exists(env['JUPYTER_CONFIG_DIR']) + assert os.listdir(env['JUPYTER_CONFIG_DIR']) == ['migrated'] assert not os.path.exists(env['JUPYTER_DATA_DIR']) From 8c3ff53e007729c02191925765922247c96558e1 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Jun 2015 21:35:37 +0100 Subject: [PATCH 5/6] when in doubt, utc --- jupyter_core/migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_core/migrate.py b/jupyter_core/migrate.py index 385c3d8..78be5ed 100644 --- a/jupyter_core/migrate.py +++ b/jupyter_core/migrate.py @@ -231,7 +231,7 @@ def migrate(): # write a marker to avoid re-running migration checks ensure_dir_exists(env['jupyter_config']) with open(os.path.join(env['jupyter_config'], 'migrated'), 'w') as f: - f.write(datetime.now().isoformat()) + f.write(datetime.utcnow().isoformat()) return migrated From cdd01a12ca96f5b75501919286662d0b10734833 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Jun 2015 21:35:50 +0100 Subject: [PATCH 6/6] migrate.get_ipython_dir is always defined --- jupyter_core/application.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyter_core/application.py b/jupyter_core/application.py index ae00270..ce056c4 100644 --- a/jupyter_core/application.py +++ b/jupyter_core/application.py @@ -152,12 +152,13 @@ def migrate_config(self): # already migrated return - ipdir = os.environ.get('IPYTHONDIR') or os.path.expanduser('~/.ipython') + from .migrate import get_ipython_dir, migrate + + ipdir = get_ipython_dir() # No IPython dir, nothing to migrate if not os.path.exists(ipdir): return - from .migrate import migrate migrate() def load_config_file(self, suppress_errors=True):