diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45bc05d4..9f634097 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,9 +58,8 @@ jobs: - uses: codecov/codecov-action@v1 with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - file: ./coverage.xml # optional - flags: tests-${{ matrix.python-version }}-${{ matrix.mongo-version }} # optional + file: ./coverage.xml + flags: tests-${{ matrix.python-version }}-${{ matrix.mongo-version }} fail_ci_if_error: true services: diff --git a/CHANGELOG.md b/CHANGELOG.md index e826038a..bd0b8e9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- Support CPython 3.9 ([#32](https://github.com/art049/odmantic/pull/32) by [@art049](https://github.com/art049)) +#### Deprecated + +- Usage of `__collection__` to customize the collection name. Prefer the `collection` + Config option ([more details](https://art049.github.io/odmantic/modeling/#collection)) + +#### Added + +- Integration with Pydantic `Config` class: + + - It's now possible to define custom `json_encoders` on the Models + - Some other `Config` options provided by Pydantic are now available ([more + details](https://art049.github.io/odmantic/modeling/#advanced-configuration)) -- Unpin pydantic to support 1.7.0 ([#29](https://github.com/art049/odmantic/pull/29) by [@art049](https://github.com/art049)) +- Support CPython 3.9 ([#32](https://github.com/art049/odmantic/pull/32) by + [@art049](https://github.com/art049)) -- Adding the latest change github action ([#30](https://github.com/art049/odmantic/pull/30) by [@art049](https://github.com/art049)) +- Unpin pydantic to support 1.7.0 ([#29](https://github.com/art049/odmantic/pull/29) by + [@art049](https://github.com/art049)) ## [0.2.1] - 2020-10-25 diff --git a/docs/examples_src/usage_pydantic/custom_encoders.py b/docs/examples_src/usage_pydantic/custom_encoders.py new file mode 100644 index 00000000..431d17bb --- /dev/null +++ b/docs/examples_src/usage_pydantic/custom_encoders.py @@ -0,0 +1,18 @@ +from datetime import datetime + +from odmantic.bson import BSON_TYPES_ENCODERS, BaseBSONModel, ObjectId + + +class M(BaseBSONModel): + id: ObjectId + date: datetime + + class Config: + json_encoders = { + **BSON_TYPES_ENCODERS, + datetime: lambda dt: dt.year, + } + + +print(M(id=ObjectId(), date=datetime.utcnow()).json()) +#> {"id": "5fa3378c8fde3766574d874d", "date": 2020} diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index edbee95e..65ef752e 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -41,6 +41,7 @@ markdown_extensions: - pymdownx.tabbed - pymdownx.superfences - pymdownx.details + - pymdownx.inlinehilite - pymdownx.magiclink: user: art049 repo: odmantic diff --git a/docs/modeling.md b/docs/modeling.md index c063dcbe..f9c63983 100644 --- a/docs/modeling.md +++ b/docs/modeling.md @@ -16,17 +16,19 @@ If the class name ends with `Model`, ODMantic will remove it to create the colle name. For example, a model class named `PersonModel` will belong in the `person` collection. -It's possible to change the collection name of a model by specifying the `__collection__` -class variable in the class body. +It's possible to customize the collection name of a model by specifying the `collection` +option in the `Config` class. !!! example "Custom collection name example" - ```python + ```python hl_lines="7-8" from odmantic import Model class CapitalCity(Model): - __collection__ = "city" name: str population: int + + class Config: + collection = "city" ``` Now, when `CapitalCity` instances will be persisted to the database, they will belong in the `city` collection instead of `capital_city`. @@ -54,6 +56,48 @@ ensure that the area of the rectangle is less or equal to 9. construct, as done in this example with `MAX_AREA`. Those class variables will be completely ignored by ODMantic while persisting instances to the database. +### Advanced Configuration + +The model configuration is done in the same way as with Pydantic models, using a [Config +class](https://pydantic-docs.helpmanual.io/usage/model_config/){:target=blank_} defined +in the model body. + + + +`#!python collection: str` +: The collection name associated to the model. see [this + section](modeling.md#collection) for more details about collection naming. + + + +`#!python title: str` *(inherited from Pydantic)* +: The name generated in the JSON schema. + + + +`#!python json_encoders: dict` *(inherited from Pydantic)* +: Customize the way types used in the model are encoded. + + ??? example "`json_encoders` example" + + For example, in order to serialize `datetime` fields as timestamp values: + ```python + class Event(Model): + date: datetime + + class Config: + json_encoders = { + datetime: lambda v: v.timestamp() + } + ``` + +!!! warning + Only the options described above are supported and other options from Pydantic can't + be used with ODMantic. + + If you feel the need to have an additional option inherited from Pydantic, you can + [open an issue](https://github.com/art049/odmantic/issues/new){:target=blank}. + ## Embedded Models Using an embedded model will store it directly in the root model it's integrated diff --git a/docs/usage_pydantic.md b/docs/usage_pydantic.md index 25e1118b..ea657197 100644 --- a/docs/usage_pydantic.md +++ b/docs/usage_pydantic.md @@ -10,6 +10,17 @@ Also, you will have to use the `bson` equivalent types defined in the [odmantic.bson](api_reference/bson.md) module. Those types, add a validation logic to the native types from the `bson` module. +!!! note "Custom `json_encoders` with `BaseBSONModel`" + If you want to specify additional json encoders, with a Pydantic model containing + `BSON` fields, you will need to pass as well the ODMantic encoders + ([BSON_TYPES_ENCODERS][odmantic.bson.BSON_TYPES_ENCODERS]). + + ??? example "Custom encoders example" + ```python linenums="1" hl_lines="11-14 18" + --8<-- "usage_pydantic/custom_encoders.py" + ``` + + An issue that would simplify this behavior has been opened: [pydantic#2024](https://github.com/samuelcolvin/pydantic/issues/2024){:target=blank_} ## Accessing the underlying pydantic model Each ODMantic Model contain a pure version of the pydantic model used to build the diff --git a/odmantic/config.py b/odmantic/config.py new file mode 100644 index 00000000..50a3948c --- /dev/null +++ b/odmantic/config.py @@ -0,0 +1,55 @@ +from typing import Any, Dict, Optional, Type + +from pydantic.main import BaseConfig +from pydantic.typing import AnyCallable + +from odmantic.bson import BSON_TYPES_ENCODERS +from odmantic.utils import is_dunder + + +class BaseODMConfig: + """Base class of the Config defined in the Models""" + + collection: Optional[str] = None + + # Inherited from pydantic + title: Optional[str] = None + json_encoders: Dict[Type[Any], AnyCallable] = {} + + +ALLOWED_CONFIG_OPTIONS = {name for name in dir(BaseODMConfig) if not is_dunder(name)} + + +class EnforcedPydanticConfig: + """Configuration options enforced to work with Models""" + + validate_all = True + validate_assignment = True + + +def validate_config( + cls_config: Type[BaseODMConfig], cls_name: str +) -> Type[BaseODMConfig]: + """Validate and build the model configuration""" + for name in dir(cls_config): + if not is_dunder(name) and name not in ALLOWED_CONFIG_OPTIONS: + raise ValueError(f"'{cls_name}': 'Config.{name}' is not supported") + + if cls_config is BaseODMConfig: + bases = (EnforcedPydanticConfig, BaseODMConfig, BaseConfig) + else: + bases = ( + EnforcedPydanticConfig, + cls_config, + BaseODMConfig, + BaseConfig, + ) # type:ignore + + # Merge json_encoders to preserve bson type encoders + namespace = { + "json_encoders": { + **BSON_TYPES_ENCODERS, + **getattr(cls_config, "json_encoders", {}), + } + } + return type("Config", bases, namespace) diff --git a/odmantic/model.py b/odmantic/model.py index f54a2d36..88139257 100644 --- a/odmantic/model.py +++ b/odmantic/model.py @@ -2,9 +2,8 @@ import decimal import enum import pathlib -import re -import sys import uuid +import warnings from abc import ABCMeta from collections.abc import Callable as abcCallable from types import FunctionType @@ -44,6 +43,7 @@ ObjectId, _decimalDecimal, ) +from odmantic.config import BaseODMConfig, validate_config from odmantic.field import ( FieldProxy, ODMBaseField, @@ -53,6 +53,13 @@ ODMReference, ) from odmantic.reference import ODMReferenceInfo +from odmantic.typing import USES_OLD_TYPING_INTERFACE +from odmantic.utils import ( + is_dunder, + raise_on_invalid_collection_name, + raise_on_invalid_key_name, + to_snake_case, +) if TYPE_CHECKING: @@ -63,12 +70,11 @@ ReprArgs, ) -USES_OLD_TYPING_INTERFACE = sys.version_info[:3] < (3, 7, 0) # PEP 560 if USES_OLD_TYPING_INTERFACE: from typing import _subs_tree # type: ignore # noqa -UNTOUCHED_TYPES = FunctionType, property, classmethod, staticmethod +UNTOUCHED_TYPES = FunctionType, property, classmethod, staticmethod, type def should_touch_field(value: Any = None, type_: Optional[Type] = None) -> bool: @@ -79,23 +85,6 @@ def should_touch_field(value: Any = None, type_: Optional[Type] = None) -> bool: ) -def is_valid_odm_field_name(name: str) -> bool: - return not name.startswith("__") and not name.endswith("__") - - -def raise_on_invalid_key_name(name: str) -> None: - # https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names - if name.startswith("$"): - raise TypeError("key_name cannot start with the dollar sign ($) character") - if "." in name: - raise TypeError("key_name cannot contain the dot (.) character") - - -def to_snake_case(s: str) -> str: - tmp = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", s) - return re.sub("([a-z0-9])([A-Z])", r"\1_\2", tmp).lower() - - def find_duplicate_key(fields: Iterable[ODMBaseField]) -> Optional[str]: seen: Set[str] = set() for f in fields: @@ -202,6 +191,7 @@ def __validate_cls_namespace__(name: str, namespace: Dict) -> None: # noqa C901 annotations = resolve_annotations( namespace.get("__annotations__", {}), namespace.get("__module__") ) + config = validate_config(namespace.get("Config", BaseODMConfig), name) odm_fields: Dict[str, ODMBaseField] = {} references: List[str] = [] bson_serialized_fields: Set[str] = set() @@ -211,7 +201,7 @@ def __validate_cls_namespace__(name: str, namespace: Dict) -> None: # noqa C901 for field_name, value in namespace.items(): if ( should_touch_field(value=value) - and is_valid_odm_field_name(field_name) + and not is_dunder(field_name) and field_name not in annotations ): raise TypeError( @@ -220,9 +210,7 @@ def __validate_cls_namespace__(name: str, namespace: Dict) -> None: # noqa C901 # Validate fields types and substitute bson fields for (field_name, field_type) in annotations.items(): - if is_valid_odm_field_name(field_name) and should_touch_field( - type_=field_type - ): + if not is_dunder(field_name) and should_touch_field(type_=field_type): substituted_type = validate_type(field_type) # Handle BSON serialized fields after substitution to allow some # builtin substitution @@ -235,9 +223,7 @@ def __validate_cls_namespace__(name: str, namespace: Dict) -> None: # noqa C901 for (field_name, field_type) in annotations.items(): value = namespace.get(field_name, Undefined) - if not is_valid_odm_field_name(field_name) or not should_touch_field( - value, field_type - ): + if is_dunder(field_name) or not should_touch_field(value, field_type): continue # pragma: no cover # https://github.com/nedbat/coveragepy/issues/198 @@ -311,6 +297,7 @@ def __validate_cls_namespace__(name: str, namespace: Dict) -> None: # noqa C901 namespace["__references__"] = tuple(references) namespace["__bson_serialized_fields__"] = frozenset(bson_serialized_fields) namespace["__mutable_fields__"] = frozenset(mutable_fields) + namespace["Config"] = config @no_type_check def __new__( @@ -344,6 +331,7 @@ def __new__( cls = super().__new__(mcs, name, bases, namespace, **kwargs) if is_custom_cls: + config: BaseODMConfig = namespace["Config"] # Patch Model related fields to build a "pure" pydantic model odm_fields: Dict[str, ODMBaseField] = namespace["__odm_fields__"] for field_name, field in odm_fields.items(): @@ -356,7 +344,8 @@ def __new__( mcs, f"{name}.__pydantic_model__", (BaseBSONModel,), namespace, **kwargs ) # Change the title to generate clean JSON schemas from this "pure" model - pydantic_cls.__config__.title = name + if config.title is None: + pydantic_cls.__config__.title = name cls.__pydantic_model__ = pydantic_cls for name, field in cls.__odm_fields__.items(): @@ -399,29 +388,24 @@ def __new__( # noqa C901 namespace["__primary_field__"] = primary_field - if "__collection__" in namespace: + config: BaseODMConfig = namespace["Config"] + if config.collection is not None: + collection_name = config.collection + elif "__collection__" in namespace: collection_name = namespace["__collection__"] + warnings.warn( + "Defining the collection name with `__collection__` is deprecated. " + "Please use `collection` config attribute instead.", + DeprecationWarning, + ) else: cls_name = name if cls_name.endswith("Model"): # TODO document this cls_name = cls_name[:-5] # Strip Model in the class name collection_name = to_snake_case(cls_name) - namespace["__collection__"] = collection_name - - # Validate the collection name - # https://docs.mongodb.com/manual/reference/limits/#Restriction-on-Collection-Names - if "$" in collection_name: - raise TypeError( - f"Invalid collection name for {name}: cannot contain '$'" - ) - if collection_name == "": - raise TypeError(f"Invalid collection name for {name}: cannot be empty") - if collection_name.startswith("system."): - raise TypeError( - f"Invalid collection name for {name}:" - " cannot start with 'system.'" - ) + raise_on_invalid_collection_name(collection_name, cls_name=name) + namespace["__collection__"] = collection_name return super().__new__(mcs, name, bases, namespace, **kwargs) diff --git a/odmantic/typing.py b/odmantic/typing.py index fc9dded8..c36a2fb5 100644 --- a/odmantic/typing.py +++ b/odmantic/typing.py @@ -15,3 +15,6 @@ from typing_extensions import Literal else: from typing import Literal # noqa: F401 + + +USES_OLD_TYPING_INTERFACE = sys.version_info[:3] < (3, 7, 0) # PEP 560 diff --git a/odmantic/utils.py b/odmantic/utils.py new file mode 100644 index 00000000..fc3d448e --- /dev/null +++ b/odmantic/utils.py @@ -0,0 +1,30 @@ +import re + + +def is_dunder(name: str) -> bool: + return name.startswith("__") and name.endswith("__") + + +def raise_on_invalid_key_name(name: str) -> None: + # https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + if name.startswith("$"): + raise TypeError("key_name cannot start with the dollar sign ($) character") + if "." in name: + raise TypeError("key_name cannot contain the dot (.) character") + + +def raise_on_invalid_collection_name(collection_name: str, cls_name: str) -> None: + # https://docs.mongodb.com/manual/reference/limits/#Restriction-on-Collection-Names + if "$" in collection_name: + raise TypeError(f"Invalid collection name for {cls_name}: cannot contain '$'") + if collection_name == "": + raise TypeError(f"Invalid collection name for {cls_name}: cannot be empty") + if collection_name.startswith("system."): + raise TypeError( + f"Invalid collection name for {cls_name}:" " cannot start with 'system.'" + ) + + +def to_snake_case(s: str) -> str: + tmp = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", s) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", tmp).lower() diff --git a/tests/fastapi/test_models.py b/tests/fastapi/test_models.py index e77c99b7..e016e3a7 100644 --- a/tests/fastapi/test_models.py +++ b/tests/fastapi/test_models.py @@ -142,7 +142,6 @@ class M(base): # type: ignore assert M.__pydantic_model__.schema()["title"] == "M" -@pytest.mark.skip("Not handled yet. Move all model config to the Config object") @pytest.mark.parametrize("base", (Model, EmbeddedModel)) def test_pydantic_model_custom_title(base: Type): class M(base): # type: ignore diff --git a/tests/unit/test_json_serialization.py b/tests/unit/test_json_serialization.py index 301a323f..cce33af1 100644 --- a/tests/unit/test_json_serialization.py +++ b/tests/unit/test_json_serialization.py @@ -1,4 +1,5 @@ import json +from datetime import datetime from typing import Dict import pytest @@ -117,3 +118,27 @@ def test_zoo_serialization_no_id(instance: Model, expected_parsed_json: Dict): parsed_data = json.loads(instance.json()) del parsed_data["id"] assert parsed_data == expected_parsed_json + + +def test_custom_json_encoders(): + class M(Model): + a: datetime = datetime.now() + + class Config: + json_encoders = {datetime: lambda _: "encoded"} + + instance = M() + parsed = json.loads(instance.json()) + assert parsed == {"id": str(instance.id), "a": "encoded"} + + +def test_custom_json_encoders_override_builtin_bson(): + class M(Model): + ... + + class Config: + json_encoders = {ObjectId: lambda _: "encoded"} + + instance = M() + parsed = json.loads(instance.json()) + assert parsed == {"id": "encoded"} diff --git a/tests/unit/test_model_definition.py b/tests/unit/test_model_definition.py index 44d2d318..d23b342a 100644 --- a/tests/unit/test_model_definition.py +++ b/tests/unit/test_model_definition.py @@ -34,7 +34,8 @@ class TheClassNameModel(Model): class TheClassNameOverriden(Model): - __collection__ = "collection_name" + class Config: + collection = "collection_name" def test_auto_collection_name(): @@ -53,7 +54,8 @@ class theNestedClassName(Model): assert theNestedClassName.__collection__ == "the_nested_class_name" class TheNestedClassNameOverriden(Model): - __collection__ = "collection_name" + class Config: + collection = "collection_name" assert TheNestedClassNameOverriden.__collection__ == "collection_name" @@ -272,21 +274,36 @@ def test_invalid_collection_name_dollar(): with pytest.raises(TypeError, match=r"cannot contain '\$'"): class A(Model): - __collection__ = "hello$world" + class Config: + collection = "hello$world" def test_invalid_collection_name_empty(): with pytest.raises(TypeError, match="cannot be empty"): class A(Model): - __collection__ = "" + class Config: + collection = "" def test_invalid_collection_name_contain_system_dot(): with pytest.raises(TypeError, match="cannot start with 'system.'"): class A(Model): - __collection__ = "system.hi" + class Config: + collection = "system.hi" + + +def test_legacy_custom_collection_name(): + with pytest.warns( + DeprecationWarning, + match="Defining the collection name with `__collection__` is deprecated", + ): + + class M(Model): + __collection__ = "collection_name" + + assert M.__collection__ == "collection_name" def test_embedded_model_key_name(): @@ -358,3 +375,21 @@ class M(Model): assert m.cls_var == "theclassvar" assert m.field == 5 assert "cls_var" not in m.doc().keys() + + +def test_forbidden_config_parameter_validate_all(): + with pytest.raises(ValueError, match="'Config.validate_all' is not supported"): + + class M(Model): + class Config: + validate_all = False + + +def test_forbidden_config_parameter_validate_assignment(): + with pytest.raises( + ValueError, match="'Config.validate_assignment' is not supported" + ): + + class M(Model): + class Config: + validate_assignment = False diff --git a/tests/zoo/person.py b/tests/zoo/person.py index 28966d32..7eb71c21 100644 --- a/tests/zoo/person.py +++ b/tests/zoo/person.py @@ -2,7 +2,8 @@ class PersonModel(Model): - __collection__ = "people" + class Config: + collection = "people" first_name: str last_name: str