From 9877429bb055015a080bf1d50e02f68e814cab40 Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Fri, 1 Dec 2023 04:09:36 +0000 Subject: [PATCH] add pydantic v2 support and drop v1 pydantic v2 has a fair amount of API changes and deprecations that make simultaneously supporting it and v1 impossible. antsibull, antsibull-docs, and other dependent packages will require some small changes, as well. --- changelogs/fragments/122-pydanticv2.yaml | 5 ++ pyproject.toml | 2 +- src/antsibull_core/app_context.py | 8 +-- src/antsibull_core/config.py | 6 +-- src/antsibull_core/galaxy.py | 4 +- src/antsibull_core/schemas/config.py | 21 ++++---- src/antsibull_core/schemas/context.py | 69 +++++++++--------------- src/antsibull_core/utils/collections.py | 4 -- tests/units/test_context.py | 38 ++++++------- 9 files changed, 71 insertions(+), 86 deletions(-) create mode 100644 changelogs/fragments/122-pydanticv2.yaml diff --git a/changelogs/fragments/122-pydanticv2.yaml b/changelogs/fragments/122-pydanticv2.yaml new file mode 100644 index 0000000..134be8d --- /dev/null +++ b/changelogs/fragments/122-pydanticv2.yaml @@ -0,0 +1,5 @@ +--- +breaking_changes: + - "antsibull-core now requires major version 2 of the ``pydantic`` library. + Version 1 is no longer supported + (https://github.com/ansible-community/antsibull-core/pull/122)." diff --git a/pyproject.toml b/pyproject.toml index 415a676..e2be0d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "packaging >= 20.0", "perky", # pydantic v2 is a major rewrite - "pydantic >= 1.0.0, < 2.0.0", + "pydantic ~= 2.0", "PyYAML", "semantic_version", # sh v2 has breaking changes. diff --git a/src/antsibull_core/app_context.py b/src/antsibull_core/app_context.py index a12c97b..e10c1ad 100644 --- a/src/antsibull_core/app_context.py +++ b/src/antsibull_core/app_context.py @@ -280,8 +280,8 @@ def create_contexts( If the field is only used via the :attr:`AppContext.extra` mechanism (not explictly set), then you should ignore this section and use :python:mod:`argparse`'s default mechanism. """ - fields_in_lib_ctx = set(LibContext.__fields__) - fields_in_app_ctx = set(app_context_model.__fields__) + fields_in_lib_ctx = set(LibContext.model_fields) + fields_in_app_ctx = set(app_context_model.model_fields) known_fields = fields_in_app_ctx.union(fields_in_lib_ctx) normalized_cfg = dict(cfg) @@ -335,7 +335,7 @@ def _copy_lib_context() -> LibContext: old_context = LibContext() # Copy just in case contexts are allowed to be writable in the the future - return old_context.copy() + return old_context.model_copy() def _copy_app_context() -> AppContext: @@ -345,7 +345,7 @@ def _copy_app_context() -> AppContext: old_context = AppContext() # Copy just in case contexts are allowed to be writable in the the future - return old_context.copy() + return old_context.model_copy() @contextmanager diff --git a/src/antsibull_core/config.py b/src/antsibull_core/config.py index df4e031..3a54cb4 100644 --- a/src/antsibull_core/config.py +++ b/src/antsibull_core/config.py @@ -71,7 +71,7 @@ def validate_config( splits up the config into lib context and app context part and validates both parts with the given model. Raises a :obj:`ConfigError` if validation fails. """ - lib_fields = set(LibContext.__fields__) + lib_fields = set(LibContext.model_fields) lib = {} app = {} for key, value in config.items(): @@ -82,8 +82,8 @@ def validate_config( # Note: We parse the object but discard the model because we want to validate the config but let # the context handle all setting of defaults try: - LibContext.parse_obj(lib) - app_context_model.parse_obj(app) + LibContext.model_validate(lib) + app_context_model.model_validate(app) except p.ValidationError as exc: joined_filenames = ", ".join(f"{fn}" for fn in filenames) raise ConfigError( diff --git a/src/antsibull_core/galaxy.py b/src/antsibull_core/galaxy.py index 1e64229..2edd9db 100644 --- a/src/antsibull_core/galaxy.py +++ b/src/antsibull_core/galaxy.py @@ -73,7 +73,7 @@ async def create( :kwarg galaxy_server: A Galaxy server URL. Defaults to ``lib_ctx.get().galaxy_server``. """ if galaxy_server is None: - galaxy_server = app_context.lib_ctx.get().galaxy_url + galaxy_server = str(app_context.lib_ctx.get().galaxy_url) api_url = urljoin(galaxy_server, "api/") async with retry_get( aio_session, api_url, headers={"Accept": "application/json"} @@ -144,7 +144,7 @@ def __init__( """ if galaxy_server is None and context is None: # TODO: deprecate - galaxy_server = app_context.lib_ctx.get().galaxy_url + galaxy_server = str(app_context.lib_ctx.get().galaxy_url) elif context is not None: # TODO: deprecate if galaxy_server is not None and galaxy_server != context.server: diff --git a/src/antsibull_core/schemas/config.py b/src/antsibull_core/schemas/config.py index ed46347..a46f3c3 100644 --- a/src/antsibull_core/schemas/config.py +++ b/src/antsibull_core/schemas/config.py @@ -15,11 +15,11 @@ #: Valid choices for a logging level field LEVEL_CHOICES_F = p.Field( - ..., regex="^(CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG|DISABLED)$" + ..., pattern="^(CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG|DISABLED)$" ) #: Valid choice of the logging version field -VERSION_CHOICES_F = p.Field(..., regex=r"1\.0") +VERSION_CHOICES_F = p.Field(..., pattern=r"1\.0") # @@ -28,10 +28,7 @@ class BaseModel(p.BaseModel): - class Config: - allow_mutation = False - extra = p.Extra.forbid - validate_all = True + model_config = p.ConfigDict(frozen=True, extra="forbid", validate_default=True) # pyre-ignore[13]: BaseModel initializes attributes when data is loaded @@ -55,23 +52,25 @@ class LogOutputModel(BaseModel): format: t.Union[str, Callable] = twiggy.formats.line_format kwargs: Mapping[str, t.Any] = {} - @p.validator("args") + @p.field_validator("args") # pylint:disable=no-self-argument def expand_home_dir_args( - cls, args_field: MutableSequence, values: Mapping + cls, args_field: MutableSequence, info: p.ValidationInfo ) -> MutableSequence: """Expand tilde in the arguments of specific outputs.""" + values = info.data if values["output"] in ("twiggy.outputs.FileOutput", twiggy.outputs.FileOutput): if args_field: args_field[0] = os.path.expanduser(args_field[0]) return args_field - @p.validator("kwargs") + @p.field_validator("kwargs") # pylint:disable=no-self-argument def expand_home_dir_kwargs( - cls, kwargs_field: MutableMapping, values: Mapping + cls, kwargs_field: MutableMapping, info: p.ValidationInfo ) -> MutableMapping: """Expand tilde in the keyword arguments of specific outputs.""" + values = info.data if values["output"] in ("twiggy.outputs.FileOutput", twiggy.outputs.FileOutput): if "name" in kwargs_field: kwargs_field["name"] = os.path.expanduser(kwargs_field["name"]) @@ -86,7 +85,7 @@ class LoggingModel(BaseModel): #: Default logging configuration -DEFAULT_LOGGING_CONFIG = LoggingModel.parse_obj( +DEFAULT_LOGGING_CONFIG = LoggingModel.model_validate( { "version": "1.0", "outputs": { diff --git a/src/antsibull_core/schemas/context.py b/src/antsibull_core/schemas/context.py index 4215a11..634b0a7 100644 --- a/src/antsibull_core/schemas/context.py +++ b/src/antsibull_core/schemas/context.py @@ -6,6 +6,7 @@ """Schemas for app and lib contexts.""" import typing as t +from functools import cached_property import pydantic as p @@ -18,25 +19,14 @@ class BaseModel(p.BaseModel): """ Configuration for all Context object classes. - :cvar Config: Sets the following information + :cvar model_config: Sets the following information - :cvar allow_mutation: ``False``. Prevents setattr on the contexts. - :cvar extra: ``p.Extra.forbid``. Prevents extra fields on the contexts. - :cvar validate_all: ``True``. Validates default values as well as user supplied ones. + :arg allow_mutation: ``False``. Prevents setattr on the contexts. + :arg extra: ``p.Extra.forbid``. Prevents extra fields on the contexts. + :arg validate_all: ``True``. Validates default values as well as user supplied ones. """ - class Config: - """ - Set default configuration for building the context models. - - :cvar allow_mutation: ``False``. Prevents setattr on the contexts. - :cvar extra: ``p.Extra.forbid``. Prevents extra fields on the contexts. - :cvar validate_all: ``True``. Validates default values as well as user supplied ones. - """ - - allow_mutation = False - extra = p.Extra.forbid - validate_all = True + model_config = p.ConfigDict(frozen=True, extra="forbid", validate_default=True) class AppContext(BaseModel): @@ -60,29 +50,29 @@ class AppContext(BaseModel): use the field of the same name in library context instead. """ - extra: ContextDict = ContextDict() + model_config = p.ConfigDict(frozen=True, extra="allow", validate_default=True) + + @cached_property + def extra(self) -> ContextDict: + # pylint: disable-next=no-member + d = (self.__pydantic_extra__ or {}).get("extra", {}) + return ContextDict.validate_and_convert(d) # DEPRECATED: ansible_base_url will be removed in antsibull-core 3.0.0. - # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684 - ansible_base_url: p.HttpUrl = "https://github.com/ansible/ansible/" # type: ignore[assignment] + ansible_base_url: p.HttpUrl = p.HttpUrl("https://github.com/ansible/ansible/") # DEPRECATED: galaxy_url will be removed in antsibull-core 3.0.0. - # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684 - galaxy_url: p.HttpUrl = "https://galaxy.ansible.com/" # type: ignore[assignment] + galaxy_url: p.HttpUrl = p.HttpUrl("https://galaxy.ansible.com/") - logging_cfg: LoggingModel = LoggingModel.parse_obj(DEFAULT_LOGGING_CONFIG) + logging_cfg: LoggingModel = LoggingModel.model_validate(DEFAULT_LOGGING_CONFIG) # DEPRECATED: pypi_url will be removed in antsibull-core 3.0.0. - # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684 - pypi_url: p.HttpUrl = "https://pypi.org/" # type: ignore[assignment] + pypi_url: p.HttpUrl = p.HttpUrl("https://pypi.org/") # DEPRECATED: collection_cache will be removed in antsibull-core 3.0.0. collection_cache: t.Optional[str] = None - # pylint: disable-next=unused-private-member - __convert_paths = p.validator("collection_cache", pre=True, allow_reuse=True)( - convert_path - ) + __convert_paths = p.field_validator("collection_cache", mode="before")(convert_path) class LibContext(BaseModel): @@ -132,28 +122,21 @@ class LibContext(BaseModel): process_max: t.Optional[int] = None thread_max: int = 8 file_check_content: int = 262144 - # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684 - ansible_core_repo_url: p.HttpUrl = ( - "https://github.com/ansible/ansible/" # type: ignore[assignment] - ) - # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684 - galaxy_url: p.HttpUrl = "https://galaxy.ansible.com/" # type: ignore[assignment] - # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684 - pypi_url: p.HttpUrl = "https://pypi.org/" # type: ignore[assignment] + ansible_core_repo_url: p.HttpUrl = p.HttpUrl("https://github.com/ansible/ansible/") + galaxy_url: p.HttpUrl = p.HttpUrl("https://galaxy.ansible.com/") + pypi_url: p.HttpUrl = p.HttpUrl("https://pypi.org/") collection_cache: t.Optional[str] = None trust_collection_cache: bool = False ansible_core_cache: t.Optional[str] = None trust_ansible_core_cache: bool = False # pylint: disable-next=unused-private-member - __convert_nones = p.validator("process_max", pre=True, allow_reuse=True)( - convert_none - ) + __convert_nones = p.field_validator("process_max", mode="before")(convert_none) # pylint: disable-next=unused-private-member - __convert_paths = p.validator( - "ansible_core_cache", "collection_cache", pre=True, allow_reuse=True + __convert_paths = p.field_validator( + "ansible_core_cache", "collection_cache", mode="before" )(convert_path) # pylint: disable-next=unused-private-member - __convert_bools = p.validator( - "trust_ansible_core_cache", "trust_collection_cache", pre=True, allow_reuse=True + __convert_bools = p.field_validator( + "trust_ansible_core_cache", "trust_collection_cache", mode="before" )(convert_bool) diff --git a/src/antsibull_core/utils/collections.py b/src/antsibull_core/utils/collections.py index 5659b29..98e69da 100644 --- a/src/antsibull_core/utils/collections.py +++ b/src/antsibull_core/utils/collections.py @@ -123,10 +123,6 @@ def __init__(self, *args, **kwargs) -> None: toplevel[key] = _make_immutable(value) super().__init__(toplevel) - @classmethod - def __get_validators__(cls): - yield cls.validate_and_convert - @classmethod def validate_and_convert(cls, value: Mapping) -> "ContextDict": if isinstance(value, ContextDict): diff --git a/tests/units/test_context.py b/tests/units/test_context.py index 65ff207..fa958d0 100644 --- a/tests/units/test_context.py +++ b/tests/units/test_context.py @@ -4,6 +4,8 @@ import argparse +from pydantic import HttpUrl + import antsibull_core.app_context as ap from antsibull_core.schemas.config import LoggingModel from antsibull_core.utils.collections import ContextDict @@ -23,9 +25,9 @@ def test_default(): app_ctx = ap.app_ctx.get() assert app_ctx.extra == ContextDict() - assert app_ctx.galaxy_url == "https://galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://galaxy.ansible.com/") assert isinstance(app_ctx.logging_cfg, LoggingModel) - assert app_ctx.pypi_url == "https://pypi.org/" + assert app_ctx.pypi_url == HttpUrl("https://pypi.org/") def test_create_contexts_with_cfg(): @@ -42,9 +44,9 @@ def test_create_contexts_with_cfg(): assert lib_ctx.max_retries == 10 assert app_ctx.extra == ContextDict({"unknown": True}) - assert app_ctx.galaxy_url == "https://galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://galaxy.ansible.com/") assert isinstance(app_ctx.logging_cfg, LoggingModel) - assert app_ctx.pypi_url == "https://test.pypi.org/" + assert app_ctx.pypi_url == HttpUrl("https://test.pypi.org/") def test_create_contexts_with_args(): @@ -62,9 +64,9 @@ def test_create_contexts_with_args(): assert lib_ctx.max_retries == 10 assert app_ctx.extra == ContextDict({"unknown": True}) - assert app_ctx.galaxy_url == "https://galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://galaxy.ansible.com/") assert isinstance(app_ctx.logging_cfg, LoggingModel) - assert app_ctx.pypi_url == "https://test.pypi.org/" + assert app_ctx.pypi_url == HttpUrl("https://test.pypi.org/") def test_create_contexts_with_args_and_cfg(): @@ -95,9 +97,9 @@ def test_create_contexts_with_args_and_cfg(): assert lib_ctx.max_retries == 10 assert app_ctx.extra == ContextDict({"unknown": False, "cfg": 1, "args": 2}) - assert app_ctx.galaxy_url == "https://dev.galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://dev.galaxy.ansible.com/") assert isinstance(app_ctx.logging_cfg, LoggingModel) - assert app_ctx.pypi_url == "https://other.pypi.org/" + assert app_ctx.pypi_url == HttpUrl("https://other.pypi.org/") def test_create_contexts_without_extra(): @@ -118,9 +120,9 @@ def test_create_contexts_without_extra(): assert lib_ctx.max_retries == 10 assert app_ctx.extra == ContextDict() - assert app_ctx.galaxy_url == "https://galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://galaxy.ansible.com/") assert isinstance(app_ctx.logging_cfg, LoggingModel) - assert app_ctx.pypi_url == "https://pypi.org/" + assert app_ctx.pypi_url == HttpUrl("https://pypi.org/") # @@ -135,11 +137,11 @@ def test_context_overrides(): with ap.app_context(data.app_ctx) as app_ctx: # Test that the app_context that was returned has the new values - assert app_ctx.galaxy_url == "https://dev.galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://dev.galaxy.ansible.com/") # Test that the context that we can retrieve has the new values too app_ctx = ap.app_ctx.get() - assert app_ctx.galaxy_url == "https://dev.galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://dev.galaxy.ansible.com/") with ap.lib_context(data.lib_ctx) as lib_ctx: # Test that the returned lib_ctx has the new values @@ -152,9 +154,9 @@ def test_context_overrides(): # Check that once we return from the context managers, the old values have been restored app_ctx = ap.app_ctx.get() assert app_ctx.extra == ContextDict() - assert app_ctx.galaxy_url == "https://galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://galaxy.ansible.com/") assert isinstance(app_ctx.logging_cfg, LoggingModel) - assert app_ctx.pypi_url == "https://pypi.org/" + assert app_ctx.pypi_url == HttpUrl("https://pypi.org/") lib_ctx = ap.lib_ctx.get() assert lib_ctx.chunksize == 4096 @@ -200,11 +202,11 @@ def test_app_and_lib_context(): with ap.app_and_lib_context(data) as (app_ctx, lib_ctx): # Test that the app_context that was returned has the new values - assert app_ctx.galaxy_url == "https://dev.galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://dev.galaxy.ansible.com/") # Test that the context that we can retrieve has the new values too app_ctx = ap.app_ctx.get() - assert app_ctx.galaxy_url == "https://dev.galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://dev.galaxy.ansible.com/") # Test that the returned lib_ctx has the new values assert lib_ctx.chunksize == 5 @@ -216,9 +218,9 @@ def test_app_and_lib_context(): # Check that once we return from the context manager, the old values have been restored app_ctx = ap.app_ctx.get() assert app_ctx.extra == ContextDict() - assert app_ctx.galaxy_url == "https://galaxy.ansible.com/" + assert app_ctx.galaxy_url == HttpUrl("https://galaxy.ansible.com/") assert isinstance(app_ctx.logging_cfg, LoggingModel) - assert app_ctx.pypi_url == "https://pypi.org/" + assert app_ctx.pypi_url == HttpUrl("https://pypi.org/") lib_ctx = ap.lib_ctx.get() assert lib_ctx.chunksize == 4096