Add support for dynamic configuration updates

docs/source/config.rst

@@ -32,7 +32,7 @@ Application: :class:`~traitlets.config.Application`
     of the application and know how to configure themselves given the
     configuration object.
 
-    Applications always have a `log` attribute that is a configured Logger.
+    Applications always have a ``log`` attribute that is a configured Logger.
     This allows centralized logging configuration per-application.
 
 Configurable: :class:`~traitlets.config.Configurable`
@@ -407,6 +407,77 @@ To see a list of the available aliases, flags, and subcommands for a configurabl
 application, simply pass ``-h`` or ``--help``. And to see the full list of
 configurable options (*very* long), pass ``--help-all``.
 
+Dynamic Configurations
+======================
+
+Applications can choose to participate in *dynamic configurations* where updates
+made to configuration files are periodically included and reflected in their
+corresponding traitlet values.  This feature is meant for long-running, service-oriented,
+applications that cannot afford to be restarted if a change is configuration is
+desired. Examples of uses would include the toggling of ``DEBUG`` logging or updates to a
+whitelist of values.
+
+To enable this functionality, the application first registers itself, and any
+:class:`~traitlets.config.Configurable` instances it manages for which dynamic updates
+should be applied, by calling its :meth:`add_dynamic_configurable()` method.  The application
+then calls its :meth:`update_dynamic_configurables()` method at periodic intervals.  As a
+result, use of :meth:`IOLoop.periodicCallback()` is recommended.
+
+.. sourcecode:: python
+
+    self.add_dynamic_configurable('MyApp', self)
+    self.add_dynamic_configurable('MyConfigurableA', self.my_configurable_a)
+    self.add_dynamic_configurable('MyConfigurableB', self.my_configurable_b)
+    self.add_dynamic_configurable('MyConfigurableC', self.my_configurable_c)
+
+    self.dynamic_config_poller = ioloop.PeriodicCallback(self.update_dynamic_configurables,
+                                                         self.dynamic_config_interval * 1000)
+    self.dynamic_config_poller.start()
+
+Once registration has occurred, the application will have its configuration-based traitlets
+reflect any updates that have taken place within the previously loaded configuration files since
+the last update.
+
+    .. note::
+       There are some caveats that should be understood up front.
+
+       1. Command-line options (CLI) always take precedence over file-based options.  As a result,
+       CLI options updated within configuration files will not be reflected.
+
+       2. Some configurable options are used to control other execution paths - like their own periodic
+       intervals or applications may copy the value from the configurable trailet to another location.  If
+       it's desired to have those traitlets participate in dynamic configuration updates, an ``@observe``
+       handler should be implemented to reset the traitlet such that its associated behavior is affected
+       by the new value (see example below).
+
+Since dynamic configurations are periodic in nature, applications will tend to use an interval
+that a) enables dynamic configurations and b) indicates the period with which the changes should
+occur.  Here's an example of how an application may manage that interval, and changes to its value,
+via its ``dynamic_config_interval`` trailet.
+
+.. sourcecode:: python
+
+    dynamic_config_interval = Integer(0, config=True,
+                              help="""Specifies the number of seconds configuration files are
+                              polled for changes. A value of 0 or less disables dynamic config
+                              updates.""")
+
+    @observe('dynamic_config_interval')
+    def dynamic_config_interval_changed(self, event):
+        prev_val = event['old']
+        self.dynamic_config_interval = event['new']
+        if self.dynamic_config_interval != prev_val:
+            # Values are different. Stop the current poller. If new value is > 0, start a poller.
+            if self.dynamic_config_poller:
+                self.dynamic_config_poller.stop()
+                self.dynamic_config_poller = None
+
+            if self.dynamic_config_interval <= 0:
+                self.log.warning("Dynamic configuration updates have been disabled and
+                                 "cannot be re-enabled without restarting application!")
+            elif prev_val > 0:  # The interval has been changed, but still positive
+                self.init_dynamic_configurables()  # Restart the poller
+
 
 Design requirements
 ===================

traitlets/config/application.py

@@ -15,6 +15,8 @@
 import pprint
 import re
 import sys
+import time
+import weakref
 
 from traitlets.config.configurable import Configurable, SingletonConfigurable
 from traitlets.config.loader import (
@@ -315,6 +317,9 @@ def __init__(self, **kwargs):
             else:
                 self.classes.insert(0, self.__class__)
 
+        self.last_config_update = time.time()
+        self.dynamic_configurables = {}
+
     @observe('config')
     @observe_compat
     def _config_changed(self, change):
@@ -329,7 +334,6 @@ def initialize(self, argv=None):
         """
         self.parse_command_line(argv)
 
-
     def start(self):
         """Start the app mainloop.
 
@@ -763,7 +767,8 @@ def load_config_file(self, filename, path=None):
             raise_config_file_errors=self.raise_config_file_errors,
         ):
             new_config.merge(config)
-            self._loaded_config_files.append(filename)
+            if filename not in self._loaded_config_files:  # only add if not there (support reloads)
+                self._loaded_config_files.append(filename)
         # add self.cli_config to preserve CLI config priority
         new_config.merge(self.cli_config)
         self.update_config(new_config)
@@ -820,6 +825,60 @@ def exit(self, exit_status=0):
         self.log.debug("Exiting application: %s" % self.name)
         sys.exit(exit_status)
 
+    def _config_files_updated(self):
+        """
+        Checks the currently loaded config file modification times to see if any are
+        more recent than the last update.  If newer files are detected, True is returned.
+        :return: bool
+        """
+        updated = False
+        for file in self._loaded_config_files:
+            mod_time = os.path.getmtime(file)
+            if mod_time > self.last_config_update:
+                self.log.debug("Config file was updated: {}!".format(file))
+                self.last_config_update = mod_time
+                updated = True
+                # Rather than break here, exhaust all files so last_config_update is the latest.
+        return updated
+
+    def update_dynamic_configurables(self):
+        """
+        Called periodically, this method checks if configuration file updates have occurred.  If
+        updates where detected (last mod time changed), reload the configuration files and update
+        the list of configurables participating in dynamic updates.
+        :return: True if files were updated
+        """
+        updated = False
+        configs = []
+        if self._config_files_updated():
+            # If files were updated, reload the config files into self.config, then
+            # update the config of each configurable from the newly loaded values.
+            # Note: We must be explicit when calling load_config_file() so as to not conflict
+            # with child class implementations (that are not overrides, e.g., JupyterApp).
+            for file in self._loaded_config_files:
+                Application.load_config_file(self, file)
+
+            for config_name, configurable in self.dynamic_configurables.items():
+                configurable.update_config(self.config)
+                configs.append(config_name)
+
+            updated = True
+            self.log.info("Configuration file changes detected.  Instances for the following "
+                          "configurables have been updated: {}".format(configs))
+        return updated
+
+    def add_dynamic_configurable(self, config_name, configurable):
+        """
+        Adds the configurable instance associated with the given name to the list of Configurables
+        that can have their configurations updated when configuration file updates are detected.
+        :param config_name: the name of the config within this application
+        :param configurable: the configurable instance corresponding to that config
+        """
+        if not isinstance(configurable, Configurable):
+            raise RuntimeError("'{}' is not a subclass of Configurable!".format(configurable))
+
+        self.dynamic_configurables[config_name] = weakref.proxy(configurable)
+
     @classmethod
     def launch_instance(cls, argv=None, **kwargs):
         """Launch a global instance of this Application

traitlets/config/tests/test_application.py

@@ -12,6 +12,7 @@
 import logging
 import os
 import sys
+import time
 from io import StringIO
 from unittest import TestCase, skip
 
@@ -358,7 +359,6 @@ def test_warn_autocorrect(self):
         self.assertIn("warn_typo", stream.getvalue())
         self.assertIn("warn_tpyo", stream.getvalue())
 
-
     def test_flatten_flags(self):
         cfg = Config()
         cfg.MyApp.log_level = logging.WARN
@@ -547,6 +547,52 @@ def test_subcommands_instanciation(self):
         self.assertIs(app.subapp.parent, app)
         self.assertIs(app.subapp.subapp.parent, app.subapp)     # Set by factory.
 
+    def test_dynamic_updates(self):
+        app = MyApp()
+        s1 = time.time()
+        app.log = logging.getLogger()
+        name = 'config.py'
+        with TemporaryDirectory('_1') as td1:
+            with open(pjoin(td1, name), 'w') as f:
+                f.writelines([
+                    "c.MyApp.running = True\n",
+                    "c.MyApp.Bar.b = 1\n"
+                ])
+
+            app.load_config_file(name, path=[td1])
+            app.init_bar()
+            app.add_dynamic_configurable("MyApp", app)
+            app.add_dynamic_configurable("Bar", app.bar)
+            with self.assertRaises(RuntimeError):
+                app.add_dynamic_configurable("Bogus", app.log)
+
+            app.start()
+            self.assertEqual(app.running, True)
+            self.assertEqual(app.bar.b, 1)
+
+            # Ensure file update doesn't happen during same second as initial value.
+            # This is necessary on test systems that don't have finer-grained
+            # timestamps (of less than a second).
+            s2 = time.time()
+            if s2 - s1 < 1.0:
+                time.sleep(1.0 - (s2-s1))
+            # update config file
+            with open(pjoin(td1, name), 'w') as f:
+                f.writelines([
+                    "c.MyApp.running = False\n",
+                    "c.MyApp.Bar.b = 2\n"
+                ])
+
+            # trigger reload and verify updates
+            app.update_dynamic_configurables()
+            self.assertEqual(app.running, False)
+            self.assertEqual(app.bar.b, 2)
+
+            # repeat to ensure no unexpected changes occurred
+            app.update_dynamic_configurables()
+            self.assertEqual(app.running, False)
+            self.assertEqual(app.bar.b, 2)
+
 
 class Root(Application):
     subcommands = {