From 0bc2428901dea9898d8a8185f35da39578316785 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Thu, 20 Apr 2023 16:06:27 +0100 Subject: [PATCH 1/4] Make all API models convert to camelCase --- src/blueapi/config.py | 12 ++++++----- src/blueapi/core/bluesky_types.py | 6 ++++-- src/blueapi/core/context.py | 7 ++----- src/blueapi/service/model.py | 17 ++++++++------- src/blueapi/utils/__init__.py | 4 ++++ src/blueapi/utils/base_model.py | 35 +++++++++++++++++++++++++++++++ src/blueapi/worker/event.py | 12 ++++++----- src/blueapi/worker/task.py | 7 ++++--- 8 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 src/blueapi/utils/base_model.py diff --git a/src/blueapi/config.py b/src/blueapi/config.py index 6000628d2..5ba813c04 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -1,10 +1,12 @@ from pathlib import Path from typing import Union -from pydantic import BaseModel, Field +from pydantic import Field +from blueapi.utils import BlueapiBaseModel -class StompConfig(BaseModel): + +class StompConfig(BlueapiBaseModel): """ Config for connecting to stomp broker """ @@ -13,7 +15,7 @@ class StompConfig(BaseModel): port: int = 61613 -class EnvironmentConfig(BaseModel): +class EnvironmentConfig(BlueapiBaseModel): """ Config for the RunEngine environment """ @@ -21,11 +23,11 @@ class EnvironmentConfig(BaseModel): startup_script: Union[Path, str] = "blueapi.startup.example" -class LoggingConfig(BaseModel): +class LoggingConfig(BlueapiBaseModel): level: str = "INFO" -class ApplicationConfig(BaseModel): +class ApplicationConfig(BlueapiBaseModel): """ Config for the worker application as a whole. Root of config tree. diff --git a/src/blueapi/core/bluesky_types.py b/src/blueapi/core/bluesky_types.py index aca9d6444..23792b118 100644 --- a/src/blueapi/core/bluesky_types.py +++ b/src/blueapi/core/bluesky_types.py @@ -21,6 +21,8 @@ from bluesky.utils import Msg from pydantic import BaseModel, Field +from blueapi.utils import BlueapiBaseModel + try: from typing import Protocol, runtime_checkable except ImportError: @@ -79,7 +81,7 @@ def is_bluesky_plan_generator(func: PlanGenerator) -> bool: ) -class Plan(BaseModel): +class Plan(BlueapiBaseModel): """ A plan that can be run """ @@ -90,7 +92,7 @@ class Plan(BaseModel): ) -class DataEvent(BaseModel): +class DataEvent(BlueapiBaseModel): """ Event representing collection of some data. Conforms to the Bluesky event model: https://github.com/bluesky/event-model diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 5d46506c1..9626c5cd6 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -10,6 +10,7 @@ from pydantic import BaseConfig from blueapi.utils import ( + BlueapiPlanModelConfig, TypeValidatorDefinition, create_model_with_type_validators, load_module_all, @@ -28,10 +29,6 @@ LOGGER = logging.getLogger(__name__) -class PlanModelConfig(BaseConfig): - arbitrary_types_allowed = True - - @dataclass class BlueskyContext: """ @@ -122,7 +119,7 @@ def my_plan(a: int, b: str): plan.__name__, validators, func=plan, - config=PlanModelConfig, + config=BlueapiPlanModelConfig, ) self.plans[plan.__name__] = Plan(name=plan.__name__, model=model) self.plan_functions[plan.__name__] = plan diff --git a/src/blueapi/service/model.py b/src/blueapi/service/model.py index ee220e64f..17cd57c06 100644 --- a/src/blueapi/service/model.py +++ b/src/blueapi/service/model.py @@ -1,14 +1,15 @@ from typing import Iterable, List from bluesky.protocols import HasName -from pydantic import BaseModel, Field +from pydantic import Field from blueapi.core import BLUESKY_PROTOCOLS, Device, Plan +from blueapi.utils import BlueapiBaseModel _UNKNOWN_NAME = "UNKNOWN" -class DeviceModel(BaseModel): +class DeviceModel(BlueapiBaseModel): """ Representation of a device """ @@ -30,7 +31,7 @@ def _protocol_names(device: Device) -> Iterable[str]: yield protocol.__name__ -class DeviceRequest(BaseModel): +class DeviceRequest(BlueapiBaseModel): """ A query for devices """ @@ -38,7 +39,7 @@ class DeviceRequest(BaseModel): ... -class DeviceResponse(BaseModel): +class DeviceResponse(BlueapiBaseModel): """ Response to a query for devices """ @@ -46,7 +47,7 @@ class DeviceResponse(BaseModel): devices: List[DeviceModel] = Field(description="Devices available to use in plans") -class PlanModel(BaseModel): +class PlanModel(BlueapiBaseModel): """ Representation of a plan """ @@ -58,7 +59,7 @@ def from_plan(cls, plan: Plan) -> "PlanModel": return cls(name=plan.name) -class PlanRequest(BaseModel): +class PlanRequest(BlueapiBaseModel): """ A query for plans """ @@ -66,7 +67,7 @@ class PlanRequest(BaseModel): ... -class PlanResponse(BaseModel): +class PlanResponse(BlueapiBaseModel): """ Response to a query for plans """ @@ -74,7 +75,7 @@ class PlanResponse(BaseModel): plans: List[PlanModel] = Field(description="Plans available to use by a worker") -class TaskResponse(BaseModel): +class TaskResponse(BlueapiBaseModel): """ Acknowledgement that a task has started, includes its ID """ diff --git a/src/blueapi/utils/__init__.py b/src/blueapi/utils/__init__.py index 4d1bff4dc..d12336a36 100644 --- a/src/blueapi/utils/__init__.py +++ b/src/blueapi/utils/__init__.py @@ -1,3 +1,4 @@ +from .base_model import BlueapiBaseModel, BlueapiModelConfig, BlueapiPlanModelConfig from .config import ConfigLoader from .modules import load_module_all from .serialization import serialize @@ -11,4 +12,7 @@ "create_model_with_type_validators", "TypeValidatorDefinition", "serialize", + "BlueapiBaseModel", + "BlueapiModelConfig", + "BlueapiPlanModelConfig", ] diff --git a/src/blueapi/utils/base_model.py b/src/blueapi/utils/base_model.py new file mode 100644 index 000000000..eced96436 --- /dev/null +++ b/src/blueapi/utils/base_model.py @@ -0,0 +1,35 @@ +from pydantic import BaseConfig, BaseModel, Extra + + +def _to_camel(string: str) -> str: + words = string.split("_") + return words[0] + "".join(word.capitalize() for word in words[1:]) + + +class BlueapiModelConfig(BaseConfig): + """ + Pydantic config for blueapi API models with + common config. + """ + + alias_generator = _to_camel + extra = Extra.forbid + + +class BlueapiPlanModelConfig(BlueapiModelConfig): + """ + Pydantic config for plan parameters. + Includes arbitrary type config so that devices + can be parameters. + """ + + arbitrary_types_allowed = True + + +class BlueapiBaseModel(BaseModel): + """ + Base class for blueapi API models. + Includes common config. + """ + + Config = BlueapiModelConfig diff --git a/src/blueapi/worker/event.py b/src/blueapi/worker/event.py index 457ec443c..6193f4068 100644 --- a/src/blueapi/worker/event.py +++ b/src/blueapi/worker/event.py @@ -2,9 +2,11 @@ from typing import List, Mapping, Optional, Union from bluesky.run_engine import RunEngineStateMachine -from pydantic import BaseModel, Field +from pydantic import Field from super_state_machine.extras import PropertyMachine, ProxyString +from blueapi.utils import BlueapiBaseModel + # The RunEngine can return any of these three types as its state RawRunEngineState = Union[PropertyMachine, ProxyString, str] @@ -41,7 +43,7 @@ def from_bluesky_state(cls, bluesky_state: RawRunEngineState) -> "WorkerState": return WorkerState(str(bluesky_state).upper()) -class StatusView(BaseModel): +class StatusView(BlueapiBaseModel): """ A snapshot of a Status of an operation, optionally representing progress """ @@ -80,7 +82,7 @@ class StatusView(BaseModel): ) -class ProgressEvent(BaseModel): +class ProgressEvent(BlueapiBaseModel): """ Event describing the progress of processes within a running task, such as moving motors and exposing detectors. @@ -90,7 +92,7 @@ class ProgressEvent(BaseModel): statuses: Mapping[str, StatusView] = Field(default_factory=dict) -class TaskStatus(BaseModel): +class TaskStatus(BlueapiBaseModel): """ Status of a task the worker is running. """ @@ -100,7 +102,7 @@ class TaskStatus(BaseModel): task_failed: bool -class WorkerEvent(BaseModel): +class WorkerEvent(BlueapiBaseModel): """ Event describing the state of the worker and any tasks it's running. Includes error and warning information. diff --git a/src/blueapi/worker/task.py b/src/blueapi/worker/task.py index efbb01313..43fd98daa 100644 --- a/src/blueapi/worker/task.py +++ b/src/blueapi/worker/task.py @@ -3,13 +3,14 @@ from dataclasses import dataclass from typing import Any, Mapping -from pydantic import BaseModel, Field, parse_obj_as +from pydantic import Field, parse_obj_as from blueapi.core import BlueskyContext, Plan +from blueapi.utils import BlueapiBaseModel # TODO: Make a TaggedUnion -class Task(ABC, BaseModel): +class Task(ABC, BlueapiBaseModel): """ Object that can run with a TaskContext """ @@ -49,7 +50,7 @@ def do_task(self, ctx: BlueskyContext) -> None: def _lookup_params( ctx: BlueskyContext, plan: Plan, params: Mapping[str, Any] -) -> BaseModel: +) -> BlueapiBaseModel: """ Checks plan parameters against context From 619fc3738bf9d63d9fac2f32d26f14aea32070f6 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Fri, 21 Apr 2023 08:16:40 +0100 Subject: [PATCH 2/4] Fix import --- src/blueapi/core/context.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 9626c5cd6..ecf5f5369 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -7,7 +7,6 @@ from bluesky import RunEngine from bluesky.protocols import Flyable, Readable -from pydantic import BaseConfig from blueapi.utils import ( BlueapiPlanModelConfig, From e22bb3f61fc06ce8b01d0b49a620e1b8ad0e584f Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Fri, 21 Apr 2023 08:42:18 +0100 Subject: [PATCH 3/4] Fix mypy error --- src/blueapi/worker/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blueapi/worker/task.py b/src/blueapi/worker/task.py index 43fd98daa..2a77a18ba 100644 --- a/src/blueapi/worker/task.py +++ b/src/blueapi/worker/task.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any, Mapping -from pydantic import Field, parse_obj_as +from pydantic import BaseModel, Field, parse_obj_as from blueapi.core import BlueskyContext, Plan from blueapi.utils import BlueapiBaseModel @@ -50,7 +50,7 @@ def do_task(self, ctx: BlueskyContext) -> None: def _lookup_params( ctx: BlueskyContext, plan: Plan, params: Mapping[str, Any] -) -> BlueapiBaseModel: +) -> BaseModel: """ Checks plan parameters against context From 4c35abb3bd498b201079f05c0e3386556083830c Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Fri, 21 Apr 2023 14:57:12 +0100 Subject: [PATCH 4/4] Document need for custom base model --- src/blueapi/utils/base_model.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/blueapi/utils/base_model.py b/src/blueapi/utils/base_model.py index eced96436..03d8b1de3 100644 --- a/src/blueapi/utils/base_model.py +++ b/src/blueapi/utils/base_model.py @@ -30,6 +30,20 @@ class BlueapiBaseModel(BaseModel): """ Base class for blueapi API models. Includes common config. + + Previously, with apischema, the API models + were serialized with camel case aliasing. + For example, converting a Python field + called foo_bar to a JSON field called fooBar + and vice versa. This is to comply with the + Google JSON style guide. + https://google.github.io/styleguide/jsoncstyleguide.xml?showone=Property_Name_Format#Property_Name_Format + + We have a custom base model with custom config + primarily to preserve this change and also + to prevent the ingestion of arbirtrary JSON + alongside a model's known fields, which + apischema also did not allow. """ Config = BlueapiModelConfig