Typing and secrets

README.md

@@ -32,7 +32,7 @@ alphaconf.setup_configuration({
 def main():
     log = logging.getLogger()
     log.info('server.url:', alphaconf.get('server.url'))
-    log.info('has server.user:', alphaconf.get('server.user', bool))
+    log.info('has server.user:', alphaconf.get('server.user', bool, default=False))
 
 if __name__ == '__main__':
     alphaconf.cli.run(main)
@@ -76,17 +76,17 @@ Finally, the configuration is fully resolved and logging is configured.
 
 ## Configuration templates and resolvers
 
-[OmegaConf]'s resolvers may be used as configuration values.
-For example, `${oc.env:USER,me}` would resolve to the environment variable
-USER with a default value "me".
-Similarly, `${oc.select:path}` will resolve to another configuration value.
-
-Additional resolvers are added to read file contents.
-These are the same as type casts: read_text, read_strip, read_bytes.
--- TODO use secrets for v1
-
-The select is used to build multiple templates for configurations by providing
-base configurations.
+Configuration values are resolved by [OmegaConf].
+Some of the resolvers (standard and custom):
+- `${oc.env:USER,me}`: resolve the environment variable USER
+  with a default value "me"
+- `${oc.select:config_path}`: resolve to another configuration value
+- `${read_text:file_path}`: read text contents of a file as `str`
+- `${read_bytes:file_path}`: read contents of a file as `bytes`
+- `${read_strip:file_path}`: read text contents of a file as strip spaces
+
+The *oc.select* is used to build multiple templates for configurations
+by providing base configurations.
 An argument `--select key=template` is a shortcut for
 `key=${oc.select:base.key.template}`.
 So, `logging: ${oc.select:base.logging.default}` resolves to the configuration
@@ -96,20 +96,31 @@ dict defined in base.logging.default and you can select it using
 ## Configuration values and integrations
 
 ### Typed-configuration
--- TODO update to pydantic
 
-You can use *omegaconf* with *dataclasses* to specify which values are
-enforced in the configuration.
-Alternatively, the *get* method can receive a data type or a function
-which will parse the value.
-By default, bool, str, Path, DateTime, etc. are supported.
+You can use [OmegaConf] with [pydantic] to *get* typed values.
+```python
+class MyConf(pydantic.BaseModel):
+    value: int = 0
+
+    def build(self):
+        # use as a factory pattern to create more complex objects
+        return self.value * 2
+
+# setup the configuration
+alphaconf.setup_configuration(MyConf, path='a')
+# read the value
+alphaconf.get('a', MyConf)
+v = alphaconf.get(MyConf)  # because it's registered as a type
+```
 
 ### Secrets
 
 When showing the configuration, by default configuration keys which are
 secrets, keys or passwords will be masked.
-Another good practice is to have a file containing the password which
-you can retrieve using `alphaconf.get('secret_file', 'read_strip')`.
+You can read values or passwords from files, by using the template
+`${read_strip:/path_to_file}`
+or, more securely, read the file in the code
+`alphaconf.get('secret_file', Path).read_text().strip()`.
 
 ### Invoke integration
 
@@ -125,9 +136,10 @@ alphaconf.invoke.run(__name__, ns)
 ```
 
 ## Way to 1.0
-- Secret management
-- Install completions for bash
-- Run a function after importing the module
+- Run function `@alphaconf.inject`
+- Run a specific function `alphaconf.cli.run_module()`:
+  find functions and parse their args
+- Install completions for bash `alphaconf --install-autocompletion`
 
 [OmegaConf]: https://omegaconf.readthedocs.io/
 [pydantic]: https://docs.pydantic.dev/latest/

alphaconf/__init__.py

@@ -1,6 +1,6 @@
 import re
 import warnings
-from typing import Callable, Optional, Sequence, TypeVar, Union
+from typing import Callable, MutableSequence, Optional, Sequence, TypeVar, Union
 
 from .frozendict import frozendict  # noqa: F401 (expose)
 from .internal.application import Application
@@ -26,9 +26,11 @@
 
 """
 
-SECRET_MASKS = [
+SECRET_MASKS: MutableSequence[Callable] = [
     # mask if contains a kind of secret and it's not in a file
-    re.compile(r'.*(key|password|secret)s?(?!_file)(_|$)|^private(_key|$)').match,
+    re.compile(
+        r'.*(key|password|secret)s?(?!_file)(?!_path)(_|$)|^(authentication|private)(_key|$)'
+    ).match,
 ]
 """A list of functions which given a key indicate whether it's a secret"""
 

alphaconf/interactive.py

@@ -16,10 +16,10 @@ def mount(configuration_paths: List[str] = [], setup_logging: bool = True):
     application.setup_configuration(configuration_paths=configuration_paths)
     set_application(application)
     if setup_logging:
-        import logging_util
+        from . import logging_util
 
         logging_util.setup_application_logging(
-            application.configuration.get('logging'), default=None
+            application.configuration.get('logging', default=None)
         )
     logging.info('Mounted interactive application')
 

alphaconf/internal/application.py

@@ -3,7 +3,7 @@
 import os
 import sys
 import uuid
-from typing import Iterable, List, MutableMapping, Optional, Tuple, Union, cast
+from typing import Callable, Iterable, List, MutableMapping, Optional, Tuple, Union, cast
 
 from omegaconf import DictConfig, OmegaConf
 
@@ -24,7 +24,7 @@ class Application:
     def __init__(
         self,
         *,
-        name=None,
+        name: Optional[str] = None,
         **properties,
     ) -> None:
         """Initialize the application.
@@ -237,7 +237,7 @@ def masked_configuration(
         return config
 
     @staticmethod
-    def __mask_config(obj, check, replace, path=''):
+    def __mask_config(obj, check: Callable[[str], bool], replace: Callable, path: str = ''):
         """Alter the configuration dict
 
         :param config: The value to mask
@@ -264,7 +264,7 @@ def __mask_config(obj, check, replace, path=''):
             ]
         return obj
 
-    def print_help(self, *, arguments=True):
+    def print_help(self, *, arguments: bool = True):
         """Print the help message
         Set the arguments to False to disable printing them."""
         prop = self.properties

alphaconf/internal/configuration.py

@@ -7,6 +7,7 @@
     Any,
     Dict,
     Iterable,
+    List,
     MutableMapping,
     Optional,
     Type,
@@ -62,7 +63,7 @@ def get(
     def get(
         self,
         key: str,
-        type: Union[str, Type[T], None] = None,
+        type: Union[str, None] = None,
         *,
         default: Any = raise_on_missing,
     ) -> Any:
@@ -146,6 +147,7 @@ def setup_configuration(
         if conf_type:
             # if already registered, set path to None
             self.__type_path[conf_type] = None if conf_type in self.__type_path else path
+            self.__type_value.pop(conf_type, None)
         if path and not path.endswith('.'):
             path += "."
         if isinstance(conf, str):
@@ -164,7 +166,9 @@ def setup_configuration(
         # add path and merge
         if path:
             config = self.__add_path(config, path.rstrip("."))
+            helpers = {path + k: v for k, v in helpers.items()}
         self._merge([config])
+        # helpers
         self.helpers.update(**helpers)
 
     def add_helper(self, key, description):
@@ -178,20 +182,39 @@ def from_environ(self, prefixes: Iterable[str]) -> DictConfig:
         trans = str.maketrans('_', '.', '"\\=')
         prefixes = tuple(prefixes)
         dotlist = [
-            (name.lower().translate(trans), value)
+            (name.lower().translate(trans).strip('.'), value)
             for name, value in os.environ.items()
             if name.startswith(prefixes)
         ]
         conf = OmegaConf.create({})
         for name, value in dotlist:
-            # TODO adapt name something.my_config from something.my.config
+            name = Configuration._find_name(name.split('.'), self.c)
             try:
                 conf.merge_with_dotlist([f"{name}={value}"])
             except YAMLError:
                 # if cannot load the value as a dotlist, just add the string
                 OmegaConf.update(conf, name, value)
         return conf
 
+    @staticmethod
+    def _find_name(parts: List[str], conf: DictConfig) -> str:
+        """Find a name from parts, by trying joining with '.' (default) or '_'"""
+        if len(parts) < 2:
+            return "".join(parts)
+        name = ""
+        for next_offset, part in enumerate(parts, 1):
+            if name:
+                name += "_"
+            name += part
+            if name in conf.keys():
+                sub_conf = conf.get(name)
+                if next_offset == len(parts):
+                    return name
+                elif isinstance(sub_conf, DictConfig):
+                    return name + "." + Configuration._find_name(parts[next_offset:], sub_conf)
+                return ".".join([name, *parts[next_offset:]])
+        return ".".join(parts)
+
     def __prepare_dictconfig(
         self, obj: DictConfig, path: str, recursive: bool = True
     ) -> DictConfig:
@@ -248,9 +271,15 @@ def __prepare_pydantic(self, obj, path):
                     defaults[k] = "???"
                 else:
                     defaults[k] = None
+                # description
                 if desc := (field.description or field.title):
                     self.add_helper(path + k, desc)
-                if check_type and field.annotation:
+                # check the type
+                if field.annotation == pydantic.SecretStr:
+                    from alphaconf import SECRET_MASKS
+
+                    SECRET_MASKS.append(lambda s: s == path)
+                elif check_type and field.annotation:
                     self.__prepare_pydantic(field.annotation, path + k + ".")
             return defaults
         return None

alphaconf/internal/type_resolvers.py

@@ -34,11 +34,11 @@ def parse_bool(value) -> bool:
     datetime.date: lambda s: datetime.datetime.strptime(s, '%Y-%m-%d').date(),
     datetime.time: datetime.time.fromisoformat,
     Path: lambda s: Path(s).expanduser(),
+    str: lambda v: str(v),
     'read_text': read_text,
     'read_strip': lambda s: read_text(s).strip(),
     'read_bytes': lambda s: Path(s).expanduser().read_bytes(),
 }
-_type = type
 
 # register resolved from strings
 for _name, _function in TYPE_CONVERTER.items():
@@ -50,14 +50,18 @@ def convert_to_type(value, type):
     """Converts a value to the given type.
 
     :param value: Any value
-    :param type: A class or a callable used to convert the value
+    :param type: A class used to convert the value
     :return: Result of the callable
     """
+    if isinstance(type, str):
+        return TYPE_CONVERTER[type](value)
+    # assert isinstance(type, globals().type)
+    if pydantic and issubclass(type, pydantic.BaseModel):
+        return type.model_validate(value)
+    if isinstance(value, type):
+        return value
+    if type in TYPE_CONVERTER:
+        return TYPE_CONVERTER[type](value)
     if pydantic:
-        if issubclass(type, pydantic.BaseModel):
-            type.model_construct
-            return type.model_validate(value)
-        if isinstance(type, _type):
-            return pydantic.TypeAdapter(type).validate_python(value)
-    type = TYPE_CONVERTER.get(type, type)
+        return pydantic.TypeAdapter(type).validate_python(value)
     return type(value)

pyproject.toml

@@ -17,7 +17,7 @@ requires-python = ">=3.9"
 classifiers = [
     # https://pypi.org/pypi?%3Aaction=list_classifiers
     "Programming Language :: Python :: 3",
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "License :: OSI Approved :: BSD License",
     "Environment :: Console",
 ]

tests/test_configuration.py

@@ -14,6 +14,7 @@ def config():
         'b': True,
         'num': 5,
         'home': '/home',
+        'with_underscore': "/_\\",
     }
     conf = Configuration()
     conf.setup_configuration(c)
@@ -83,6 +84,38 @@ def test_select_required_incomplete(config_req):
         print(config_req.get('req'))
 
 
+@pytest.mark.parametrize(
+    "name,expected",
+    [
+        ('a.b', 'a.b'),
+        ('unknown', 'unknown'),
+        ('a.b.zz', 'a.b.zz'),
+        ('b', 'b'),
+    ],
+)
+def test_env_find_name_simple(config, name, expected):
+    assert Configuration._find_name(name.split('.'), config.c) == expected
+
+
+def test_env_find_name_complex():
+    config = Configuration()
+    config.setup_configuration(
+        {
+            'a': {'b': 1},
+            'my_test': {'a': 2},
+            'test_test': {
+                'x': 3,
+                'my_test': 4,
+            },
+        }
+    )
+    c = config.c
+    assert Configuration._find_name(['a', 'b'], c) == 'a.b'
+    assert Configuration._find_name(['my', 'test'], c) == 'my_test'
+    assert Configuration._find_name(['my', 'test', 'a'], c) == 'my_test.a'
+    assert Configuration._find_name(['test', 'test', 'my', 'test'], c) == 'test_test.my_test'
+
+
 def test_config_setup_dots(config):
     config.setup_configuration(
         {