From 7858049abcfe5d6517b62c09805792a844ab4aee Mon Sep 17 00:00:00 2001 From: Krzysztof Magusiak Date: Sun, 26 Nov 2023 22:14:34 +0100 Subject: [PATCH 1/4] pydantic description and usage --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2cb9fc1..5c8ade9 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Finally, the configuration is fully resolved and logging is configured. ## Configuration templates and resolvers -[OmegaConf]'s resolvers may be used as configuration values. +Configuration values are resolved by [OmegaConf]. For example, `${oc.env:USER,me}` would resolve to the environment variable USER with a default value "me". Similarly, `${oc.select:path}` will resolve to another configuration value. @@ -96,13 +96,13 @@ dict defined in base.logging.default and you can select it using ## Configuration values and integrations ### Typed-configuration --- TODO update to pydantic -You can use *omegaconf* with *dataclasses* to specify which values are +You can use [OmegaConf] with [pydantic] to specify which values are enforced in the configuration. Alternatively, the *get* method can receive a data type or a function which will parse the value. By default, bool, str, Path, DateTime, etc. are supported. +TODO describe more, use pydantic to build factories ### Secrets From 9d6ec832346a5ccfe968d657f240d27ed0929d87 Mon Sep 17 00:00:00 2001 From: Krzysztof Magusiak Date: Sun, 26 Nov 2023 22:41:06 +0100 Subject: [PATCH 2/4] Fix interactive --- README.md | 9 +++++---- alphaconf/interactive.py | 4 ++-- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5c8ade9..7bc0ebd 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ alphaconf.setup_configuration({ def main(): log = logging.getLogger() log.info('server.url:', alphaconf.get('server.url')) - log.info('has server.user:', alphaconf.get('server.user', bool)) + log.info('has server.user:', alphaconf.get('server.user', bool, default=False)) if __name__ == '__main__': alphaconf.cli.run(main) @@ -110,6 +110,7 @@ When showing the configuration, by default configuration keys which are secrets, keys or passwords will be masked. Another good practice is to have a file containing the password which you can retrieve using `alphaconf.get('secret_file', 'read_strip')`. +TODO do better than this, also pydantic.SecretStr? ### Invoke integration @@ -125,9 +126,9 @@ alphaconf.invoke.run(__name__, ns) ``` ## Way to 1.0 -- Secret management -- Install completions for bash -- Run a function after importing the module +- Run a specific function `alphaconf.cli.run_module()`: + find functions and parse their args +- Install completions for bash `alphaconf --install-autocompletion` [OmegaConf]: https://omegaconf.readthedocs.io/ [pydantic]: https://docs.pydantic.dev/latest/ diff --git a/alphaconf/interactive.py b/alphaconf/interactive.py index 004b654..53216a7 100644 --- a/alphaconf/interactive.py +++ b/alphaconf/interactive.py @@ -16,10 +16,10 @@ def mount(configuration_paths: List[str] = [], setup_logging: bool = True): application.setup_configuration(configuration_paths=configuration_paths) set_application(application) if setup_logging: - import logging_util + from . import logging_util logging_util.setup_application_logging( - application.configuration.get('logging'), default=None + application.configuration.get('logging', default=None) ) logging.info('Mounted interactive application') diff --git a/pyproject.toml b/pyproject.toml index 9580fbc..af22d06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ requires-python = ">=3.9" classifiers = [ # https://pypi.org/pypi?%3Aaction=list_classifiers "Programming Language :: Python :: 3", - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "License :: OSI Approved :: BSD License", "Environment :: Console", ] From 2d183dc28b3b4b9b2dee5cd4600454b87b411615 Mon Sep 17 00:00:00 2001 From: Krzysztof Magusiak Date: Mon, 27 Nov 2023 20:17:51 +0100 Subject: [PATCH 3/4] Update README --- README.md | 49 +++++++++++++++++----------- alphaconf/__init__.py | 8 +++-- alphaconf/internal/application.py | 8 ++--- alphaconf/internal/configuration.py | 37 ++++++++++++++++++--- alphaconf/internal/type_resolvers.py | 20 +++++++----- tests/test_configuration.py | 33 +++++++++++++++++++ 6 files changed, 117 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 7bc0ebd..4e1a194 100644 --- a/README.md +++ b/README.md @@ -77,16 +77,16 @@ Finally, the configuration is fully resolved and logging is configured. ## Configuration templates and resolvers Configuration values are resolved by [OmegaConf]. -For example, `${oc.env:USER,me}` would resolve to the environment variable -USER with a default value "me". -Similarly, `${oc.select:path}` will resolve to another configuration value. - -Additional resolvers are added to read file contents. -These are the same as type casts: read_text, read_strip, read_bytes. --- TODO use secrets for v1 - -The select is used to build multiple templates for configurations by providing -base configurations. +Some of the resolvers (standard and custom): +- `${oc.env:USER,me}`: resolve the environment variable USER + with a default value "me" +- `${oc.select:config_path}`: resolve to another configuration value +- `${read_text:file_path}`: read text contents of a file as `str` +- `${read_bytes:file_path}`: read contents of a file as `bytes` +- `${read_strip:file_path}`: read text contents of a file as strip spaces + +The *oc.select* is used to build multiple templates for configurations +by providing base configurations. An argument `--select key=template` is a shortcut for `key=${oc.select:base.key.template}`. So, `logging: ${oc.select:base.logging.default}` resolves to the configuration @@ -97,20 +97,30 @@ dict defined in base.logging.default and you can select it using ### Typed-configuration -You can use [OmegaConf] with [pydantic] to specify which values are -enforced in the configuration. -Alternatively, the *get* method can receive a data type or a function -which will parse the value. -By default, bool, str, Path, DateTime, etc. are supported. -TODO describe more, use pydantic to build factories +You can use [OmegaConf] with [pydantic] to *get* typed values. +```python +class MyConf(pydantic.BaseModel): + value: int = 0 + + def build(self): + # use as a factory pattern to create more complex objects + return self.value * 2 + +# setup the configuration +alphaconf.setup_configuration(MyConf, path='a') +# read the value +alphaconf.get('a', MyConf) +v = alphaconf.get(MyConf) # because it's registered as a type +``` ### Secrets When showing the configuration, by default configuration keys which are secrets, keys or passwords will be masked. -Another good practice is to have a file containing the password which -you can retrieve using `alphaconf.get('secret_file', 'read_strip')`. -TODO do better than this, also pydantic.SecretStr? +You can read values or passwords from files, by using the template +`${read_strip:/path_to_file}` +or, more securely, read the file in the code +`alphaconf.get('secret_file', Path).read_text().strip()`. ### Invoke integration @@ -126,6 +136,7 @@ alphaconf.invoke.run(__name__, ns) ``` ## Way to 1.0 +- Run function `@alphaconf.inject` - Run a specific function `alphaconf.cli.run_module()`: find functions and parse their args - Install completions for bash `alphaconf --install-autocompletion` diff --git a/alphaconf/__init__.py b/alphaconf/__init__.py index fcffc63..c05cc25 100644 --- a/alphaconf/__init__.py +++ b/alphaconf/__init__.py @@ -1,6 +1,6 @@ import re import warnings -from typing import Callable, Optional, Sequence, TypeVar, Union +from typing import Callable, MutableSequence, Optional, Sequence, TypeVar, Union from .frozendict import frozendict # noqa: F401 (expose) from .internal.application import Application @@ -26,9 +26,11 @@ """ -SECRET_MASKS = [ +SECRET_MASKS: MutableSequence[Callable] = [ # mask if contains a kind of secret and it's not in a file - re.compile(r'.*(key|password|secret)s?(?!_file)(_|$)|^private(_key|$)').match, + re.compile( + r'.*(key|password|secret)s?(?!_file)(?!_path)(_|$)|^(authentication|private)(_key|$)' + ).match, ] """A list of functions which given a key indicate whether it's a secret""" diff --git a/alphaconf/internal/application.py b/alphaconf/internal/application.py index 2815921..b4e3691 100644 --- a/alphaconf/internal/application.py +++ b/alphaconf/internal/application.py @@ -3,7 +3,7 @@ import os import sys import uuid -from typing import Iterable, List, MutableMapping, Optional, Tuple, Union, cast +from typing import Callable, Iterable, List, MutableMapping, Optional, Tuple, Union, cast from omegaconf import DictConfig, OmegaConf @@ -24,7 +24,7 @@ class Application: def __init__( self, *, - name=None, + name: Optional[str] = None, **properties, ) -> None: """Initialize the application. @@ -237,7 +237,7 @@ def masked_configuration( return config @staticmethod - def __mask_config(obj, check, replace, path=''): + def __mask_config(obj, check: Callable[[str], bool], replace: Callable, path: str = ''): """Alter the configuration dict :param config: The value to mask @@ -264,7 +264,7 @@ def __mask_config(obj, check, replace, path=''): ] return obj - def print_help(self, *, arguments=True): + def print_help(self, *, arguments: bool = True): """Print the help message Set the arguments to False to disable printing them.""" prop = self.properties diff --git a/alphaconf/internal/configuration.py b/alphaconf/internal/configuration.py index 4f958f1..8da1d1e 100644 --- a/alphaconf/internal/configuration.py +++ b/alphaconf/internal/configuration.py @@ -7,6 +7,7 @@ Any, Dict, Iterable, + List, MutableMapping, Optional, Type, @@ -62,7 +63,7 @@ def get( def get( self, key: str, - type: Union[str, Type[T], None] = None, + type: Union[str, None] = None, *, default: Any = raise_on_missing, ) -> Any: @@ -146,6 +147,7 @@ def setup_configuration( if conf_type: # if already registered, set path to None self.__type_path[conf_type] = None if conf_type in self.__type_path else path + self.__type_value.pop(conf_type, None) if path and not path.endswith('.'): path += "." if isinstance(conf, str): @@ -164,7 +166,9 @@ def setup_configuration( # add path and merge if path: config = self.__add_path(config, path.rstrip(".")) + helpers = {path + k: v for k, v in helpers.items()} self._merge([config]) + # helpers self.helpers.update(**helpers) def add_helper(self, key, description): @@ -178,13 +182,13 @@ def from_environ(self, prefixes: Iterable[str]) -> DictConfig: trans = str.maketrans('_', '.', '"\\=') prefixes = tuple(prefixes) dotlist = [ - (name.lower().translate(trans), value) + (name.lower().translate(trans).strip('.'), value) for name, value in os.environ.items() if name.startswith(prefixes) ] conf = OmegaConf.create({}) for name, value in dotlist: - # TODO adapt name something.my_config from something.my.config + name = Configuration._find_name(name.split('.'), self.c) try: conf.merge_with_dotlist([f"{name}={value}"]) except YAMLError: @@ -192,6 +196,25 @@ def from_environ(self, prefixes: Iterable[str]) -> DictConfig: OmegaConf.update(conf, name, value) return conf + @staticmethod + def _find_name(parts: List[str], conf: DictConfig) -> str: + """Find a name from parts, by trying joining with '.' (default) or '_'""" + if len(parts) < 2: + return "".join(parts) + name = "" + for next_offset, part in enumerate(parts, 1): + if name: + name += "_" + name += part + if name in conf.keys(): + sub_conf = conf.get(name) + if next_offset == len(parts): + return name + elif isinstance(sub_conf, DictConfig): + return name + "." + Configuration._find_name(parts[next_offset:], sub_conf) + return ".".join([name, *parts[next_offset:]]) + return ".".join(parts) + def __prepare_dictconfig( self, obj: DictConfig, path: str, recursive: bool = True ) -> DictConfig: @@ -248,9 +271,15 @@ def __prepare_pydantic(self, obj, path): defaults[k] = "???" else: defaults[k] = None + # description if desc := (field.description or field.title): self.add_helper(path + k, desc) - if check_type and field.annotation: + # check the type + if field.annotation == pydantic.SecretStr: + from alphaconf import SECRET_MASKS + + SECRET_MASKS.append(lambda s: s == path) + elif check_type and field.annotation: self.__prepare_pydantic(field.annotation, path + k + ".") return defaults return None diff --git a/alphaconf/internal/type_resolvers.py b/alphaconf/internal/type_resolvers.py index dc7494d..be49a47 100644 --- a/alphaconf/internal/type_resolvers.py +++ b/alphaconf/internal/type_resolvers.py @@ -34,11 +34,11 @@ def parse_bool(value) -> bool: datetime.date: lambda s: datetime.datetime.strptime(s, '%Y-%m-%d').date(), datetime.time: datetime.time.fromisoformat, Path: lambda s: Path(s).expanduser(), + str: lambda v: str(v), 'read_text': read_text, 'read_strip': lambda s: read_text(s).strip(), 'read_bytes': lambda s: Path(s).expanduser().read_bytes(), } -_type = type # register resolved from strings for _name, _function in TYPE_CONVERTER.items(): @@ -50,14 +50,18 @@ def convert_to_type(value, type): """Converts a value to the given type. :param value: Any value - :param type: A class or a callable used to convert the value + :param type: A class used to convert the value :return: Result of the callable """ + if isinstance(type, str): + return TYPE_CONVERTER[type](value) + # assert isinstance(type, globals().type) + if pydantic and issubclass(type, pydantic.BaseModel): + return type.model_validate(value) + if isinstance(value, type): + return value + if type in TYPE_CONVERTER: + return TYPE_CONVERTER[type](value) if pydantic: - if issubclass(type, pydantic.BaseModel): - type.model_construct - return type.model_validate(value) - if isinstance(type, _type): - return pydantic.TypeAdapter(type).validate_python(value) - type = TYPE_CONVERTER.get(type, type) + return pydantic.TypeAdapter(type).validate_python(value) return type(value) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 8fa81ce..b9386f0 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -14,6 +14,7 @@ def config(): 'b': True, 'num': 5, 'home': '/home', + 'with_underscore': "/_\\", } conf = Configuration() conf.setup_configuration(c) @@ -83,6 +84,38 @@ def test_select_required_incomplete(config_req): print(config_req.get('req')) +@pytest.mark.parametrize( + "name,expected", + [ + ('a.b', 'a.b'), + ('unknown', 'unknown'), + ('a.b.zz', 'a.b.zz'), + ('b', 'b'), + ], +) +def test_env_find_name_simple(config, name, expected): + assert Configuration._find_name(name.split('.'), config.c) == expected + + +def test_env_find_name_complex(): + config = Configuration() + config.setup_configuration( + { + 'a': {'b': 1}, + 'my_test': {'a': 2}, + 'test_test': { + 'x': 3, + 'my_test': 4, + }, + } + ) + c = config.c + assert Configuration._find_name(['a', 'b'], c) == 'a.b' + assert Configuration._find_name(['my', 'test'], c) == 'my_test' + assert Configuration._find_name(['my', 'test', 'a'], c) == 'my_test.a' + assert Configuration._find_name(['test', 'test', 'my', 'test'], c) == 'test_test.my_test' + + def test_config_setup_dots(config): config.setup_configuration( { From 65dae743695a326e1cfe13df4eb52696e4eff326 Mon Sep 17 00:00:00 2001 From: Krzysztof Magusiak Date: Mon, 27 Nov 2023 20:37:54 +0100 Subject: [PATCH 4/4] .