Refactor internal package

alphaconf/__init__.py

@@ -6,11 +6,11 @@
 from contextvars import ContextVar
 from typing import Any, Callable, Dict, Union, cast
 
-from omegaconf import DictConfig, MissingMandatoryValue, OmegaConf
+from omegaconf import Container, DictConfig, MissingMandatoryValue, OmegaConf
 
-from .application import Application, _log
-from .arg_parser import ArgumentError, ExitApplication
-from .type_resolvers import convert_to_type
+from .internal import Application
+from .internal.arg_parser import ArgumentError, ExitApplication
+from .internal.type_resolvers import convert_to_type
 
 __doc__ = """AlphaConf
 
@@ -49,33 +49,50 @@
 _helpers: ContextVar[Dict[str, str]] = ContextVar('configuration_helpers', default={})
 
 
-def select(container, key: str, type=None, *, default=None) -> Any:
-    """Select a configuration item from the container"""
-    if isinstance(container, DictConfig):
-        conf = container
-    else:
-        conf = OmegaConf.create(container)
-    if key:
-        c = OmegaConf.select(conf, key, throw_on_missing=True)
+def select(container: Any, key: str, type=None, *, default=None, required: bool = False) -> Any:
+    """Select a configuration item from the container
+
+    :param container: The container to select from (Container, dict, etc.)
+    :param key: The selection key
+    :param type: The type of the object to return
+    :param default: The default value is selected value is None
+    :param required: Raise MissingMandatoryValue if the selected value and default are None
+    :return: The selected value in the container
+    """
+    c: Any
+    # make sure we have a container and select from it
+    if isinstance(container, Container):
+        c = container
     else:
-        c = conf
-    if isinstance(c, DictConfig):
-        c = OmegaConf.to_object(c)
-    elif c is None:
+        c = OmegaConf.create(container)
+    c = OmegaConf.select(c, key, throw_on_missing=required)
+    # handle empty result
+    if c is None:
+        if default is None and required:
+            raise MissingMandatoryValue("Key not found: %s" % key)
         return default
-    if type and c is not None:
+    # check the returned type and convert when necessary
+    if type is not None and isinstance(c, type):
+        return c
+    if isinstance(c, Container):
+        c = OmegaConf.to_object(c)
+    if type is not None:
         c = convert_to_type(c, type)
     return c
 
 
-def get(config_key: str, type=None, *, default=None) -> Any:
+def get(config_key: str, type=None, *, default=None, required: bool = False) -> Any:
     """Select a configuration item from the current configuration"""
-    return select(configuration.get(), config_key, type=type, default=default)
+    return select(configuration.get(), config_key, type=type, default=default, required=required)
 
 
 @contextlib.contextmanager
 def set(**kw):
-    """Update the configuration in a with block"""
+    """Update the configuration in a with block
+
+    with alphaconf.set(a=value):
+        assert alphaconf.get('a') == value
+    """
     if not kw:
         yield
         return
@@ -113,6 +130,8 @@ def run(main: Callable, arguments=True, *, should_exit=True, app: Application =
     :param config: Arguments passed to Application.__init__() and Application.setup_configuration()
     :return: The result of main
     """
+    from .internal import application_log as log
+
     if 'setup_logging' not in config:
         config['setup_logging'] = True
     if app is None:
@@ -136,17 +155,17 @@ def run(main: Callable, arguments=True, *, should_exit=True, app: Application =
     try:
         app.setup_configuration(arguments, **config)
     except MissingMandatoryValue as e:
-        _log.error(e)
+        log.error(e)
         if should_exit:
             sys.exit(99)
         raise
     except ArgumentError as e:
-        _log.error(e)
+        log.error(e)
         if should_exit:
             sys.exit(2)
         raise
     except ExitApplication:
-        _log.debug('Normal application exit')
+        log.debug('Normal application exit')
         if should_exit:
             sys.exit()
         return
@@ -155,7 +174,7 @@ def run(main: Callable, arguments=True, *, should_exit=True, app: Application =
         return context.run(__run_application, app=app, main=main, exc_info=should_exit)
     except Exception:
         if should_exit:
-            _log.debug('Exit application')
+            log.debug('Exit application')
             sys.exit(1)
         raise
 

alphaconf/interactive.py

@@ -1,8 +1,8 @@
 import logging
 
 from . import set_application, setup_configuration
-from .application import Application
-from .load_file import read_configuration_file
+from .internal import Application
+from .internal.load_file import read_configuration_file
 
 __doc__ = """Helpers for interactive applications."""
 __all__ = ['mount', 'read_configuration_file', 'load_configuration_file']

alphaconf/application.py → alphaconf/internal/__init__.py

@@ -8,12 +8,9 @@
 
 from . import arg_parser, load_file
 
-__doc__ = """Application
+__doc__ = """Representation of an application with its configuration."""
 
-Representation of an application with its configuration.
-"""
-
-_log = logging.getLogger(__name__)
+application_log = logging.getLogger(__name__)
 
 
 class Application:
@@ -47,7 +44,7 @@ def __init__(self, **properties) -> None:
         self.__initialize_parser()
 
     def __initialize_parser(self):
-        from . import _helpers
+        from .. import _helpers
 
         self.parsed = None
         self.argument_parser = parser = arg_parser.ArgumentParser()
@@ -89,7 +86,7 @@ def configuration(self) -> DictConfig:
             self.setup_configuration(
                 arguments=False, resolve_configuration=False, setup_logging=False
             )
-            _log.info('alphaconf initialized')
+            application_log.info('alphaconf initialized')
             assert self.__config is not None
         return self.__config
 
@@ -118,12 +115,12 @@ def _load_dotenv(self, load_dotenv: Optional[bool] = None):
             import dotenv
 
             path = dotenv.find_dotenv(usecwd=True)
-            _log.debug('Loading dotenv: %s', path or '(none)')
+            application_log.debug('Loading dotenv: %s', path or '(none)')
             dotenv.load_dotenv(path)
         except ModuleNotFoundError:
             if load_dotenv:
                 raise
-            _log.debug('dotenv is not installed')
+            application_log.debug('dotenv is not installed')
 
     def __load_environ(self, prefixes: Iterable[str]) -> DictConfig:
         """Load environment variables into a dict configuration"""
@@ -160,17 +157,17 @@ def _get_configurations(
         :param env_prefixes: Prefixes of environment variables to load
         :return: OmegaConf configurations (to be merged)
         """
-        from . import configuration as ctx_configuration
+        from .. import configuration as ctx_configuration
 
-        _log.debug('Loading default and app configurations')
+        application_log.debug('Loading default and app configurations')
         default_configuration = ctx_configuration.get()
         yield default_configuration
         yield self._app_configuration()
         # Read files
         for path in self._get_possible_configuration_paths(configuration_paths):
             if not (path in configuration_paths or os.path.isfile(path)):
                 continue
-            _log.debug('Load configuration from %s', path)
+            application_log.debug('Load configuration from %s', path)
             conf = load_file.read_configuration_file(path)
             if isinstance(conf, DictConfig):
                 yield conf
@@ -179,7 +176,7 @@ def _get_configurations(
         # Environment
         prefixes: Optional[Tuple[str, ...]]
         if env_prefixes is True:
-            _log.debug('Detecting accepted env prefixes')
+            application_log.debug('Detecting accepted env prefixes')
             default_keys = {str(k) for k in default_configuration.keys()}
             prefixes = tuple(
                 k.upper() + '_'
@@ -191,7 +188,7 @@ def _get_configurations(
         else:
             prefixes = None
         if prefixes:
-            _log.debug('Loading env configuration from prefixes %s' % (prefixes,))
+            application_log.debug('Loading env configuration from prefixes %s' % (prefixes,))
             yield self.__load_environ(prefixes)
 
     def setup_configuration(
@@ -217,7 +214,7 @@ def setup_configuration(
         """
         if self.__config is not None:
             raise RuntimeError('Configuration already set')
-        _log.debug('Start setup application')
+        application_log.debug('Start setup application')
 
         # Parse arguments
         if arguments is True:
@@ -237,7 +234,7 @@ def setup_configuration(
         if self.parsed:
             configurations.extend(self.parsed.configurations())
         self.__config = cast(DictConfig, OmegaConf.merge(*configurations))
-        _log.debug('Merged %d configurations', len(configurations))
+        application_log.debug('Merged %d configurations', len(configurations))
 
         # Handle the result
         self._handle_parsed_result()
@@ -248,7 +245,7 @@ def setup_configuration(
 
         # Logging
         if setup_logging:
-            _log.debug('Setup logging')
+            application_log.debug('Setup logging')
             self._setup_logging()
 
     def _handle_parsed_result(self):
@@ -266,7 +263,7 @@ def _setup_logging(self) -> None:
         """
         import logging
 
-        from . import logging_util
+        from .. import logging_util
 
         logging_util.set_gmt()
         log = logging.getLogger()
@@ -297,7 +294,7 @@ def masked_configuration(
         :param mask_keys: Which keys to mask
         :return: Configuration copy with masked values
         """
-        from . import SECRET_MASKS
+        from .. import SECRET_MASKS
 
         config = cast(dict, OmegaConf.to_container(self.configuration))
         if mask_secrets:

alphaconf/invoke.py

@@ -2,7 +2,16 @@
 from omegaconf import OmegaConf
 
 from . import run as _application_run
-from .application import Application, arg_parser
+from .internal import Application, arg_parser
+
+__doc__ = """Invoke wrapper for an application
+
+Adding the following lines at the end of the file adds support for alphaconf
+to inject the configuration.
+
+    ns = Collection(...)
+    alphaconf.invoke.run(__name__, ns)
+"""
 
 
 class InvokeAction(arg_parser.Action):

alphaconf/logging_util.py

@@ -10,6 +10,15 @@
 except ImportError:
     colorama = None
 
+__doc__ = """Logging utils
+
+- set_gmt: set the GMT time for time in logging
+- DynamicLogRecord.set_generator: add dynamic information to all logs
+- ColorFormatter: add colors
+- JSONFormatter: dump the message and attributes of the log record as JSON
+"""
+
+
 """Colors used by Colorama (if installed)"""
 LOG_COLORS = {}
 if colorama:

tests/test_alphaconf.py

@@ -2,7 +2,6 @@
 from omegaconf import DictConfig, OmegaConf
 
 import alphaconf
-from alphaconf.application import Application
 
 
 def test_default_app_and_configuration():
@@ -36,7 +35,7 @@ def test_run_application_app(application):
 
 def test_run_application_help(capsys):
     alphaconf.setup_configuration({'helptest': 1}, {'helptest': 'HELPER_TEST'})
-    application = Application(name='myapp', description='my test description')
+    application = alphaconf.Application(name='myapp', description='my test description')
     r = alphaconf.run(lambda: 1, app=application, arguments=['--help'], should_exit=False)
     assert r is None
     captured = capsys.readouterr()
@@ -89,7 +88,7 @@ def test_set_application(application):
 def test_setup_configuration():
     alphaconf.setup_configuration({'setup': 'config'}, helpers={'setup': 'help1'})
     assert alphaconf.get('setup') == 'config'
-    helpers = Application().argument_parser.help_messages
+    helpers = alphaconf.Application().argument_parser.help_messages
     assert helpers.get('setup') == 'help1'
 
 
@@ -103,6 +102,7 @@ def test_setup_configuration_invalid():
 
 
 def test_set():
+    # test that the set() is active only within the block
     value = '124'
     default = '125-def'
     assert alphaconf.get('value', default=default) is default
@@ -115,7 +115,7 @@ def test_secret_masks():
     masks = list(alphaconf.SECRET_MASKS)
     try:
         alphaconf.setup_configuration({'a': {'mypassword': 's3cret'}})
-        conf = Application().masked_configuration()
+        conf = alphaconf.Application().masked_configuration()
         assert conf['a']['mypassword'] != 's3cret'
     finally:
         alphaconf.SECRET_MASKS.clear()

tests/test_arg_parser.py

@@ -1,7 +1,7 @@
 import pytest
 from omegaconf import OmegaConf
 
-import alphaconf.arg_parser as ap
+import alphaconf.internal.arg_parser as ap
 
 
 @pytest.fixture(scope='function')