From 04702534aa8faa81b7b4d41d4d7f513dc65a6228 Mon Sep 17 00:00:00 2001 From: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:33:47 +0000 Subject: [PATCH 1/5] Update for bluesky 1.13, remove dls-bluesky-core (#645) --- dev-requirements.txt | 138 +++++++++----------- docs/explanations/lifecycle.md | 4 +- docs/reference/asyncapi.yaml | 60 +++------ helm/blueapi/values.yaml | 4 +- pyproject.toml | 9 +- src/blueapi/config.py | 4 +- src/blueapi/core/bluesky_types.py | 3 +- src/blueapi/startup/example_plans.py | 6 +- tests/system_tests/plans.json | 70 +++++----- tests/unit_tests/core/fake_plan_module.py | 2 +- tests/unit_tests/core/test_context.py | 7 +- tests/unit_tests/test_config.py | 16 +-- tests/unit_tests/worker/test_task_worker.py | 17 ++- 13 files changed, 154 insertions(+), 186 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 58c2ea6b5..9739d422d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ accessible-pygments==0.0.5 -aioca==1.8 +aioca==1.8.1 aiofiles==24.1.0 aiohappyeyeballs==2.4.3 -aiohttp==3.10.10 +aiohttp==3.11.0 aiosignal==1.3.1 alabaster==1.0.0 annotated-types==0.7.0 @@ -11,47 +11,40 @@ appdirs==1.4.4 asciitree==0.3.3 asgiref==3.8.1 asttokens==2.4.1 -async-timeout==4.0.3 attrs==24.2.0 -autodoc_pydantic==2.2.0 babel==2.16.0 beautifulsoup4==4.12.3 bidict==0.23.1 -black==24.10.0 -bluesky==1.13.0a3 +bluesky==1.13 bluesky-kafka==0.10.0 bluesky-live==0.0.8 bluesky-stomp==0.1.2 -boltons==24.0.0 -bump-pydantic==0.8.0 +boltons==24.1.0 cachetools==5.5.0 caproto==1.1.1 certifi==2024.8.30 cfgv==3.4.0 -chardet==5.2.0 charset-normalizer==3.4.0 click==8.1.7 cloudpickle==3.1.0 colorama==0.4.6 -colorlog==6.8.2 +colorlog==6.9.0 comm==0.2.2 compress-pickle==2.1.0 confluent-kafka==2.6.0 -contourpy==1.3.0 +contourpy==1.3.1 copier==9.4.1 coverage==7.6.4 cycler==0.12.1 -dask==2024.10.0 +dask==2024.11.2 databroker==1.2.5 dataclasses-json==0.6.7 decorator==5.1.1 deepdiff==8.0.1 deepmerge==2.0 Deprecated==1.2.14 -diff_cover==9.2.0 distlib==0.3.9 -dls-bluesky-core==0.0.4 -dls-dodal==1.33.0 +dls-dodal==1.36.0 dnspython==2.7.0 docopt==0.6.2 doct==1.1.0 @@ -60,59 +53,55 @@ dunamai==1.22.0 email_validator==2.2.0 entrypoints==0.4 epicscorelibs==7.0.7.99.1.1 -event-model==1.21.0 -exceptiongroup==1.2.2 +event-model==1.22.1 executing==2.1.0 -fastapi==0.115.3 +fastapi==0.115.5 fastapi-cli==0.0.5 fasteners==0.19 filelock==3.16.1 flexcache==0.3 -flexparser==0.3.1 +flexparser==0.4 fonttools==4.54.1 frozenlist==1.5.0 fsspec==2024.10.0 funcy==2.0 gitdb==4.0.11 GitPython==3.1.43 -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 graypy==2.1.0 -grpcio==1.66.2 +grpcio==1.67.1 h11==0.14.0 h5py==3.12.1 HeapDict==1.0.1 historydict==1.2.6 httpcore==1.0.6 -httptools==0.6.1 +httptools==0.6.4 httpx==0.27.2 humanize==4.11.0 -identify==2.6.1 +identify==2.6.2 idna==3.10 imageio==2.36.0 imagesize==1.4.1 -importlib_metadata==8.4.0 +importlib_metadata==8.5.0 importlib_resources==6.4.5 iniconfig==2.0.0 intake==0.6.4 ipython==8.18.0 ipywidgets==8.1.5 itsdangerous==2.2.0 -jedi==0.19.1 +jedi==0.19.2 Jinja2==3.1.4 jinja2-ansible-filters==1.3.2 -jsonpointer==3.0.0 jsonschema==4.23.0 jsonschema-specifications==2024.10.1 jupyterlab_widgets==3.0.13 kiwisolver==1.4.7 ldap3==2.9.1 -libcst==1.4.0 -livereload==2.7.0 locket==1.0.0 lz4==4.3.3 markdown-it-py==3.0.0 MarkupSafe==3.0.2 -marshmallow==3.23.0 +marshmallow==3.23.1 matplotlib==3.9.2 matplotlib-inline==0.1.7 mdit-py-plugins==0.4.2 @@ -123,36 +112,37 @@ mongoquery==1.4.2 msgpack==1.1.0 msgpack-numpy==0.4.8 multidict==6.1.0 -mypy==1.13.0 +mypy==1.13.0 mypy-extensions==1.0.0 myst-parser==4.0.0 networkx==3.4.2 nodeenv==1.9.1 nose2==0.15.1 -nslsii==0.10.5 -numcodecs==0.13.1 +nslsii==0.10.7 +numcodecs==0.14.0 numpy==1.26.4 observability-utils==0.1.4 +opencv-python==4.10.0.84 opencv-python-headless==4.10.0.84 -opentelemetry-api==1.27.0 -opentelemetry-distro==0.48b0 -opentelemetry-exporter-otlp==1.27.0 -opentelemetry-exporter-otlp-proto-common==1.27.0 -opentelemetry-exporter-otlp-proto-grpc==1.27.0 -opentelemetry-exporter-otlp-proto-http==1.27.0 -opentelemetry-instrumentation==0.48b0 -opentelemetry-instrumentation-asgi==0.48b0 -opentelemetry-instrumentation-fastapi==0.48b0 -opentelemetry-proto==1.27.0 -opentelemetry-sdk==1.27.0 -opentelemetry-semantic-conventions==0.48b0 -opentelemetry-util-http==0.48b0 +opentelemetry-api==1.28.1 +opentelemetry-distro==0.49b1 +opentelemetry-exporter-otlp==1.28.1 +opentelemetry-exporter-otlp-proto-common==1.28.1 +opentelemetry-exporter-otlp-proto-grpc==1.28.1 +opentelemetry-exporter-otlp-proto-http==1.28.1 +opentelemetry-instrumentation==0.49b1 +opentelemetry-instrumentation-asgi==0.49b1 +opentelemetry-instrumentation-fastapi==0.49b1 +opentelemetry-proto==1.28.1 +opentelemetry-sdk==1.28.1 +opentelemetry-semantic-conventions==0.49b1 +opentelemetry-util-http==0.49b1 ophyd==1.9.0 -ophyd-async==0.6.0 +ophyd-async==0.8.0a4 orderly-set==5.2.2 -orjson==3.10.10 +orjson==3.10.11 p4p==4.2.0 -packaging==24.1 +packaging==24.2 pandas==2.2.3 parso==0.8.4 partd==1.4.2 @@ -163,17 +153,17 @@ picobox==4.0.0 pika==1.3.2 pillow==11.0.0 PIMS==0.7 -Pint==0.24.3 +Pint==0.24.4 pipdeptree==2.23.4 platformdirs==4.3.6 pluggy==1.5.0 plumbum==1.9.0 ply==3.11 pre_commit==4.0.1 -prettytable==3.11.0 +prettytable==3.12.0 prompt-toolkit==3.0.36 propcache==0.2.0 -protobuf==4.25.5 +protobuf==5.28.3 psutil==6.1.0 ptyprocess==0.7.0 pure_eval==0.2.3 @@ -182,8 +172,8 @@ py==1.11.0 pyasn1==0.6.1 pycryptodome==3.21.0 pydantic==2.9.2 -pydantic-extra-types==2.9.0 -pydantic-settings==2.6.0 +pydantic-extra-types==2.10.0 +pydantic-settings==2.6.1 pydantic_core==2.23.4 pydantic_numpy==5.0.2 pydata-sphinx-theme==0.16.0 @@ -192,32 +182,31 @@ Pygments==2.18.0 pymongo==4.10.1 pyOlog==4.5.0 pyparsing==3.2.0 -pyright==1.1.386 pytest==8.3.3 pytest-asyncio==0.24.0 -pytest-cov==5.0.0 -pytest-random-order==1.1.1 +pytest-cov==6.0.0 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 -python-multipart==0.0.9 +python-multipart==0.0.17 pytz==2024.2 PyYAML==6.0.2 -pyyaml-include==2.1 questionary==2.0.1 redis==5.2.0 redis-json-dict==0.2.1 referencing==0.35.1 requests==2.32.3 responses==0.25.3 -rich==13.7.1 -rpds-py==0.20.0 +rich==13.9.4 +rpds-py==0.21.0 ruamel.yaml==0.18.6 ruamel.yaml.clib==0.2.12 -ruff==0.7.1 +ruff==0.7.3 scanspec==0.7.6 semver==3.0.2 +setuptools==75.5.0 setuptools-dso==2.11 shellingham==1.5.4 +shortuuid==1.0.13 six==1.16.0 slicerator==1.1.0 smmap==5.0.1 @@ -226,11 +215,8 @@ snowballstemmer==2.2.0 soupsieve==2.6 Sphinx==8.1.3 sphinx-autobuild==2024.10.3 -sphinx-autodoc-typehints==2.3.0 sphinx-click==6.0.0 sphinx-copybutton==0.5.2 -sphinx-jsonschema==1.19.1 -sphinx-pydantic==0.1.1 sphinx_design==0.6.1 sphinx_mdinclude==0.6.2 sphinxcontrib-applehelp==2.0.0 @@ -238,27 +224,23 @@ sphinxcontrib-devhelp==2.0.0 sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-httpdomain==1.8.1 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-mermaid==1.0.0 sphinxcontrib-openapi==0.8.4 sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 stack-data==0.6.3 -starlette==0.41.0 -stomp-py==8.1.2 +starlette==0.41.2 +stomp.py==8.2.0 suitcase-mongo==0.6.0 suitcase-msgpack==0.3.0 suitcase-utils==0.5.4 super-state-machine==2.0.2 tifffile==2024.9.20 -tomli==2.0.1 toolz==1.0.0 -tornado==6.4.1 tox==3.28.0 tox-direct==0.4 -tqdm==4.66.5 +tqdm==4.67.0 traitlets==5.14.3 -typer==0.12.4 -types-aiofiles==24.1.0.20240626 +typer==0.13.0 types-mock==5.1.0.20240425 types-PyYAML==6.0.12.20240917 types-requests==2.32.0.20241016 @@ -270,18 +252,18 @@ tzlocal==5.2 ujson==5.10.0 urllib3==2.2.3 uvicorn==0.32.0 -uvloop==0.19.0 -virtualenv==20.27.0 +uvloop==0.21.0 +virtualenv==20.27.1 watchfiles==0.24.0 wcwidth==0.2.13 websocket-client==1.8.0 -websockets==13.1 +websockets==14.1 widgetsnbextension==4.0.13 -workflows==2.27 +workflows==2.28 wrapt==1.16.0 xarray==2024.10.0 -yarl==1.16.0 +yarl==1.17.1 zarr==2.18.3 zict==2.2.0 -zipp==3.20.2 +zipp==3.21.0 zocalo==1.1.1 diff --git a/docs/explanations/lifecycle.md b/docs/explanations/lifecycle.md index 862f3a754..f0455d28b 100644 --- a/docs/explanations/lifecycle.md +++ b/docs/explanations/lifecycle.md @@ -6,8 +6,8 @@ of being written, loaded and run. Take the following plan. from typing import Any, List, Mapping, Optional, Union import bluesky.plans as bp - from blueapi.core import MsgGenerator - from dls_bluesky_core.core import inject + from bluesky.utils import MsgGenerator + from dodal.common import inject from bluesky.protocols import Readable diff --git a/docs/reference/asyncapi.yaml b/docs/reference/asyncapi.yaml index 963998e35..96d314297 100644 --- a/docs/reference/asyncapi.yaml +++ b/docs/reference/asyncapi.yaml @@ -30,8 +30,6 @@ channels: - $ref: "#/components/messages/taggedDatumDocument" - $ref: "#/components/messages/taggedResourceStream" - $ref: "#/components/messages/taggedDatumStream" - - $ref: "#/components/messages/taggedEventPage" - - $ref: "#/components/messages/taggedDatumPage" components: messages: # Defined as Components to allow referencing from other APIs. # TODO: Define protocol specific correlationId bindings taggedStartDocument: @@ -43,9 +41,9 @@ components: type: object properties: name: - type: string + const: "start" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/run_start.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/run_start.json" taggedStopDocument: messageId: runStop summary: Indicates the completion of a Bluesky run based on a previously specified Plan; describes end conditions and metadata @@ -55,9 +53,9 @@ components: type: object properties: name: - type: string + const: "stop" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/run_stop.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/run_stop.json" taggedDescriptorDocument: messageId: eventStreamDescriptor summary: Describes the devices to be within a scientifically related stream of measurements @@ -67,9 +65,9 @@ components: type: object properties: name: - type: string + const: "descriptor" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/event_descriptor.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/event_descriptor.json" taggedEventDocument: messageId: dataEvent summary: Describes a point measurement for a number of scientifically related devices @@ -79,22 +77,9 @@ components: type: object properties: name: - type: string - doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/event.json" - taggedEventPage: - messageId: eventPage - deprecated: true - summary: Page style collection of events - headers: - $ref: "#/components/schemas/contextHeaders" - payload: - type: object - properties: - name: - type: string + const: "event" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/event_page.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/event.json" taggedResourceDocument: messageId: resource summary: Describes an external resource (file, database entry etc.) that is to be referenced by later datum @@ -104,9 +89,9 @@ components: type: object properties: name: - type: string + const: "resource" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/resource.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/resource.json" taggedDatumDocument: messageId: datum summary: Describes how to access a point measurement within an external resource @@ -116,22 +101,9 @@ components: type: object properties: name: - type: string - doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/datum.json" - taggedDatumPage: - messageId: datumPage - deprecated: true - summary: Page style collection of datum - headers: - $ref: "#/components/schemas/contextHeaders" - payload: - type: object - properties: - name: - type: string + const: "datum" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/datum_page.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/datum.json" taggedResourceStream: messageId: streamResource summary: Describes an external resource (file, database entry etc.) that is to be referenced by later Stream Datum @@ -141,9 +113,9 @@ components: type: object properties: name: - type: string + const: "stream_resource" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/stream_resource.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/stream_resource.json" taggedDatumStream: messageId: streamDatum summary: Describes how to access a slice of an external resource @@ -153,9 +125,9 @@ components: type: object properties: name: - type: string + const: "stream_datum" doc: - $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.21.0/event_model/schemas/stream_datum.json" + $ref: "https://raw.githubusercontent.com/bluesky/event-model/refs/tags/v1.22.1/src/event_model/schemas/stream_datum.json" workerStateEvent: messageId: stateEvent headers: diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index fe35c33e2..5bae760e5 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -108,9 +108,9 @@ worker: - kind: planFunctions module: blueapi.startup.example_plans - kind: planFunctions - module: dls_bluesky_core.plans + module: dodal.plans - kind: planFunctions - module: dls_bluesky_core.stubs + module: dodal.plan_stubs.wrapped stomp: auth: username: guest diff --git a/pyproject.toml b/pyproject.toml index a84e0b62b..a8a22e2d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ ] description = "Lightweight bluesky-as-a-service wrapper application. Also usable as a library." dependencies = [ - "bluesky==1.13.0a3", + "bluesky>=1.13", "ophyd", "nslsii", "pyepics", @@ -27,12 +27,11 @@ dependencies = [ "fastapi>=0.112.0", "uvicorn", "requests", - "dls-bluesky-core", #requires ophyd-async - "dls-dodal>=1.31.0", + "dls-dodal>=1.36.0", "super-state-machine", # https://github.com/DiamondLightSource/blueapi/issues/553 "GitPython", "bluesky-stomp>=0.1.2", - "event-model==1.21", # https://github.com/DiamondLightSource/blueapi/issues/684 + "event-model==1.22.1", # https://github.com/DiamondLightSource/blueapi/issues/684 "opentelemetry-distro>=0.48b0", "opentelemetry-instrumentation-fastapi>=0.48b0", "observability-utils>=0.1.4" @@ -150,5 +149,5 @@ extend-immutable-calls = [ "fastapi.Depends", "fastapi.Body", "fastapi.Task", - "dls_bluesky_core.core.inject", + "dodal.common.inject", ] diff --git a/src/blueapi/config.py b/src/blueapi/config.py index 16e470c0d..468e00f98 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -51,8 +51,8 @@ class EnvironmentConfig(BlueapiBaseModel): kind=SourceKind.DEVICE_FUNCTIONS, module="blueapi.startup.example_devices" ), Source(kind=SourceKind.PLAN_FUNCTIONS, module="blueapi.startup.example_plans"), - Source(kind=SourceKind.PLAN_FUNCTIONS, module="dls_bluesky_core.plans"), - Source(kind=SourceKind.PLAN_FUNCTIONS, module="dls_bluesky_core.stubs"), + Source(kind=SourceKind.PLAN_FUNCTIONS, module="dodal.plans"), + Source(kind=SourceKind.PLAN_FUNCTIONS, module="dodal.plan_stubs.wrapped"), ] events: WorkerEventConfig = Field(default_factory=WorkerEventConfig) diff --git a/src/blueapi/core/bluesky_types.py b/src/blueapi/core/bluesky_types.py index 1a2978213..690b2ec71 100644 --- a/src/blueapi/core/bluesky_types.py +++ b/src/blueapi/core/bluesky_types.py @@ -24,7 +24,8 @@ Triggerable, WritesExternalAssets, ) -from dls_bluesky_core.core import MsgGenerator, PlanGenerator +from bluesky.utils import MsgGenerator +from dodal.common import PlanGenerator from ophyd_async.core import Device as AsyncDevice from pydantic import BaseModel, Field diff --git a/src/blueapi/startup/example_plans.py b/src/blueapi/startup/example_plans.py index b3d969521..2915707a5 100644 --- a/src/blueapi/startup/example_plans.py +++ b/src/blueapi/startup/example_plans.py @@ -1,7 +1,7 @@ from bluesky.protocols import Movable, Readable -from dls_bluesky_core.core import inject -from dls_bluesky_core.plans import count -from dls_bluesky_core.stubs import move +from dodal.common import inject +from dodal.plan_stubs.wrapped import move +from dodal.plans import count from blueapi.core import MsgGenerator diff --git a/tests/system_tests/plans.json b/tests/system_tests/plans.json index d0bba9319..76d17457a 100644 --- a/tests/system_tests/plans.json +++ b/tests/system_tests/plans.json @@ -2,8 +2,8 @@ "plans": [ { "name": "count", - "description": "\n Take `n` readings from a device\n\n Args:\n detectors (Set[Readable]): Readable devices to read\n num (int, optional): Number of readings to take. Defaults to 1.\n delay (Optional[Union[float, List[float]]], optional): Delay between readings.\n Defaults to None.\n metadata (Optional[Mapping[str, Any]], optional): Key-value metadata to include\n in exported data.\n Defaults to None.\n\n Returns:\n MsgGenerator: _description_\n\n Yields:\n Iterator[MsgGenerator]: _description_\n ", - "parameter_schema": { + "description": "Reads from a number of devices.\n Wraps bluesky.plans.count(det, num, delay, md=metadata) exposing only serializable\n parameters and metadata.", + "schema": { "additionalProperties": false, "properties": { "detectors": { @@ -28,9 +28,6 @@ "type": "number" }, "type": "array" - }, - { - "type": "null" } ], "title": "Delay" @@ -56,8 +53,8 @@ }, { "name": "move", - "description": "\n Move a device, wrapper for `bp.mv`.\n\n Args:\n moves (Mapping[Movable, Any]): Mapping of Movables to target positions\n group (Optional[Group], optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "description": "\n Move a device, wrapper for `bp.mv`.\n\n Args:\n moves (Mapping[Movable, T]): Mapping of Movables to target positions\n group (Group | None, optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", + "schema": { "additionalProperties": false, "properties": { "moves": { @@ -86,7 +83,7 @@ { "name": "stp_snapshot", "description": "\n Moves devices for pressure and temperature (defaults fetched from the context)\n and captures a single frame from a collection of devices\n\n Args:\n detectors (List[Readable]): A list of devices to read while the sample is at STP\n temperature (Optional[Movable]): A device controlling temperature of the sample,\n defaults to fetching a device name \"sample_temperature\" from the context\n pressure (Optional[Movable]): A device controlling pressure on the sample,\n defaults to fetching a device name \"sample_pressure\" from the context\n Returns:\n MsgGenerator: Plan\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "schema": { "additionalProperties": false, "properties": { "detectors": { @@ -113,9 +110,9 @@ } }, { - "name": "scan", - "description": "\n Scan wrapping `bp.scan_nd`\n\n Args:\n detectors: Set of readable devices, will take a reading at\n each point\n axes_to_move: All axes involved in this scan, names and\n objects\n spec: ScanSpec modelling the path of the scan\n metadata: Key-value metadata to include\n in exported data, defaults to\n None.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "name": "spec_scan", + "description": "Generic plan for reading `detectors` at every point of a ScanSpec `Spec`.\n A `Spec` is an N-dimensional path.\n ", + "schema": { "$defs": { "Circle": { "additionalProperties": false, @@ -141,7 +138,7 @@ }, "radius": { "description": "Radius of the circle", - "exclusiveMinimum": 0.0, + "exclusiveMinimum": 0, "title": "Radius", "type": "number" }, @@ -288,18 +285,18 @@ }, "x_radius": { "description": "The radius along the x axis of the ellipse", - "exclusiveMinimum": 0.0, + "exclusiveMinimum": 0, "title": "X Radius", "type": "number" }, "y_radius": { "description": "The radius along the y axis of the ellipse", - "exclusiveMinimum": 0.0, + "exclusiveMinimum": 0, "title": "Y Radius", "type": "number" }, "angle": { - "default": 0.0, + "default": 0, "description": "The angle of the ellipse (degrees)", "title": "Angle", "type": "number" @@ -579,7 +576,7 @@ "type": "number" }, "angle": { - "default": 0.0, + "default": 0, "description": "Clockwise rotation angle of the rectangle", "title": "Angle", "type": "number" @@ -798,7 +795,7 @@ "type": "integer" }, "rotate": { - "default": 0.0, + "default": 0, "description": "How much to rotate the angle of the spiral", "title": "Rotate", "type": "number" @@ -990,13 +987,6 @@ "type": "array", "uniqueItems": true }, - "axes_to_move": { - "additionalProperties": { - "type": "bluesky.protocols.Movable" - }, - "title": "Axes To Move", - "type": "object" - }, "spec": { "$ref": "#/$defs/Spec" }, @@ -1014,17 +1004,16 @@ }, "required": [ "detectors", - "axes_to_move", "spec" ], - "title": "scan", + "title": "spec_scan", "type": "object" } }, { "name": "set_absolute", - "description": "\n Set a device, wrapper for `bp.abs_set`.\n\n Args:\n movable (Movable): The device to set\n value (T): The new value\n group (Optional[Group], optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n wait (bool, optional): The group should wait until all setting is complete\n (e.g. a motor has finished moving). Defaults to False.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "description": "\n Set a device, wrapper for `bp.abs_set`.\n\n Args:\n movable (Movable): The device to set\n value (T): The new value\n group (Group | None, optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n wait (bool, optional): The group should wait until all setting is complete\n (e.g. a motor has finished moving). Defaults to False.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", + "schema": { "additionalProperties": false, "properties": { "movable": { @@ -1060,8 +1049,8 @@ }, { "name": "set_relative", - "description": "\n Change a device, wrapper for `bp.rel_set`.\n\n Args:\n movable (Movable): The device to set\n value (T): The new value\n group (Optional[Group], optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n wait (bool, optional): The group should wait until all setting is complete\n (e.g. a motor has finished moving). Defaults to False.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "description": "\n Change a device, wrapper for `bp.rel_set`.\n\n Args:\n movable (Movable): The device to set\n value (T): The new value\n group (Group | None, optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n wait (bool, optional): The group should wait until all setting is complete\n (e.g. a motor has finished moving). Defaults to False.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", + "schema": { "additionalProperties": false, "properties": { "movable": { @@ -1097,8 +1086,8 @@ }, { "name": "move_relative", - "description": "\n Move a device relative to its current position, wrapper for `bp.mvr`.\n\n Args:\n moves (Mapping[Movable, Any]): Mapping of Movables to target deltas\n group (Optional[Group], optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "description": "\n Move a device relative to its current position, wrapper for `bp.mvr`.\n\n Args:\n moves (Mapping[Movable, T]): Mapping of Movables to target deltas\n group (Group | None, optional): The message group to associate with the\n setting, for sequencing. Defaults to None.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", + "schema": { "additionalProperties": false, "properties": { "moves": { @@ -1127,7 +1116,7 @@ { "name": "sleep", "description": "\n Suspend all action for a given time, wrapper for `bp.sleep`\n\n Args:\n time (float): Time to wait in seconds\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "schema": { "additionalProperties": false, "properties": { "time": { @@ -1144,8 +1133,8 @@ }, { "name": "wait", - "description": "\n Wait for a group status to complete, wrapper for `bp.wait`\n\n Args:\n group (Optional[Group], optional): The name of the group to wait for, defaults\n to None.\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", - "parameter_schema": { + "description": "\n Wait for a group status to complete, wrapper for `bp.wait`.\n Does not expose move_on, as when used as a stub will not fail on Timeout.\n\n Args:\n group (Group | None, optional): The name of the group to wait for, defaults\n to None, in which case waits for all\n groups that have not yet been awaited.\n timeout (float | None, default=None): a timeout in seconds\n\n\n Returns:\n MsgGenerator: Plan\n\n Yields:\n Iterator[MsgGenerator]: Bluesky messages\n ", + "schema": { "additionalProperties": false, "properties": { "group": { @@ -1158,6 +1147,17 @@ } ], "title": "Group" + }, + "timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Timeout" } }, "title": "wait", diff --git a/tests/unit_tests/core/fake_plan_module.py b/tests/unit_tests/core/fake_plan_module.py index 5fe5f1279..2bda43f92 100644 --- a/tests/unit_tests/core/fake_plan_module.py +++ b/tests/unit_tests/core/fake_plan_module.py @@ -1 +1 @@ -from dls_bluesky_core.plans import scan # noqa: F401 +from dodal.plans import spec_scan # noqa: F401 diff --git a/tests/unit_tests/core/test_context.py b/tests/unit_tests/core/test_context.py index 0540401f2..d01db462e 100644 --- a/tests/unit_tests/core/test_context.py +++ b/tests/unit_tests/core/test_context.py @@ -5,7 +5,8 @@ import pytest from bluesky.protocols import Descriptor, Movable, Readable, Reading, SyncOrAsync -from dls_bluesky_core.core import MsgGenerator, PlanGenerator, inject +from bluesky.utils import MsgGenerator +from dodal.common import PlanGenerator, inject from ophyd.sim import SynAxis, SynGauss from pydantic import TypeAdapter, ValidationError from pytest import LogCaptureFixture @@ -147,7 +148,7 @@ def test_add_plan_from_module(empty_context: BlueskyContext) -> None: import tests.unit_tests.core.fake_plan_module as plan_module empty_context.with_plan_module(plan_module) - assert {"scan"} == empty_context.plans.keys() + assert {"spec_scan"} == empty_context.plans.keys() def test_add_named_device(empty_context: BlueskyContext, sim_motor: SynAxis) -> None: @@ -277,7 +278,7 @@ def test_add_devices_and_plans_from_modules_with_config( "motor_bundle_a", "motor_bundle_b", } == empty_context.devices.keys() - assert {"scan"} == empty_context.plans.keys() + assert {"spec_scan"} == empty_context.plans.keys() def test_function_spec(empty_context: BlueskyContext) -> None: diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index 5e2ec84f9..a5e0b32ad 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -204,8 +204,8 @@ def temp_yaml_config_file( "env": { "sources": [ {"kind": "dodal", "module": "dodal.adsim"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.plans"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.stubs"}, + {"kind": "planFunctions", "module": "dodal.plans"}, + {"kind": "planFunctions", "module": "dodal.plan_stubs.wrapped"}, ], }, "api": {"host": "0.0.0.0", "port": 8000}, @@ -215,8 +215,8 @@ def temp_yaml_config_file( "env": { "sources": [ {"kind": "dodal", "module": "dodal.adsim"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.plans"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.stubs"}, + {"kind": "planFunctions", "module": "dodal.plans"}, + {"kind": "planFunctions", "module": "dodal.plan_stubs.wrapped"}, ], "events": {"broadcast_status_events": True}, }, @@ -258,8 +258,8 @@ def test_config_yaml_parsed(temp_yaml_config_file): }, "sources": [ {"kind": "dodal", "module": "dodal.adsim"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.plans"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.stubs"}, + {"kind": "planFunctions", "module": "dodal.plans"}, + {"kind": "planFunctions", "module": "dodal.plan_stubs.wrapped"}, ], }, "api": { @@ -287,8 +287,8 @@ def test_config_yaml_parsed(temp_yaml_config_file): "env": { "sources": [ {"kind": "dodal", "module": "dodal.adsim"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.plans"}, - {"kind": "planFunctions", "module": "dls_bluesky_core.stubs"}, + {"kind": "planFunctions", "module": "dodal.plans"}, + {"kind": "planFunctions", "module": "dodal.plan_stubs.wrapped"}, ], "events": {"broadcast_status_events": True}, }, diff --git a/tests/unit_tests/worker/test_task_worker.py b/tests/unit_tests/worker/test_task_worker.py index 1b42af09e..1eae7280c 100644 --- a/tests/unit_tests/worker/test_task_worker.py +++ b/tests/unit_tests/worker/test_task_worker.py @@ -2,11 +2,13 @@ import threading from collections.abc import Callable, Iterable from concurrent.futures import Future +from pathlib import Path from queue import Full from typing import Any, TypeVar from unittest.mock import ANY, MagicMock, patch import pytest +from dodal.common.types import UpdatingPathProvider from observability_utils.tracing import ( JsonObjectSpanExporter, asserting_span_exporter, @@ -341,7 +343,18 @@ def begin_task_and_wait_until_complete( # -def test_worker_and_data_events_produce_in_order(worker: TaskWorker) -> None: +@pytest.fixture +def path_provider(tmp_path: Path): + # Prevents issue with leftover state from beamline tests + with patch("dodal.plan_stubs.data_session.get_path_provider") as mock: + mock.return_value = MagicMock(spec=UpdatingPathProvider, return_value=tmp_path) + mock.return_value.data_session.return_value = "foo" + yield + + +def test_worker_and_data_events_produce_in_order( + worker: TaskWorker, path_provider +) -> None: assert_running_count_plan_produces_ordered_worker_and_data_events( [ WorkerEvent( @@ -383,7 +396,7 @@ def assert_running_count_plan_produces_ordered_worker_and_data_events( task: Task | None = None, timeout: float = 5.0, ) -> None: - default_task = Task(name="count", params={"detectors": ["image_det"], "num": 1}) + default_task = Task(name="count", params={"detectors": {"image_det"}, "num": 1}) task = task or default_task event_streams: list[EventStream[Any, int]] = [ From 09391f7a8ee21359195a9c7d40deaeec62525873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Malinowski?= <56644812+stan-dot@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:48:48 +0000 Subject: [PATCH 2/5] amend the regex so that the `refs/tags/version` passes (#714) --- .github/workflows/_container.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index 70ce21742..2a11d16ba 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -19,7 +19,8 @@ jobs: - name: Validate SemVer2 version compliance if: startsWith(github.ref, 'refs/tags/') env: - SEMVER_REGEX: ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ + SEMVER_REGEX: ^refs/tags/(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ + run: | ref="${{ github.ref_name }}" my_regex="${{env.SEMVER_REGEX}}" From 86b56987d38aafcd2f47e3b2d2f3caef9597c1ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Malinowski?= <56644812+stan-dot@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:16:46 +0000 Subject: [PATCH 3/5] 713 quickfix too restrictive semver check - fixing the ref/version parsing (#715) Fixes #667 --------- Co-authored-by: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> --- .github/workflows/_container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index 2a11d16ba..b9f4ee13c 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -102,7 +102,7 @@ jobs: - name: package chart and push it run: | - sed -i "$ a appVersion: ${VERSION}" helm/blueapi/Chart.yaml + sed -i "$ a appVersion: ${GITHUB_REF##*/}" helm/blueapi/Chart.yaml helm dependencies update helm/blueapi - helm package helm/blueapi --version ${VERSION} -d /tmp/ - helm push /tmp/blueapi-${VERSION}.tgz oci://ghcr.io/diamondlightsource/charts + helm package helm/blueapi --version ${GITHUB_REF##*/} -d /tmp/ + helm push /tmp/blueapi-${GITHUB_REF##*/}.tgz oci://ghcr.io/diamondlightsource/charts From e51530c5069330c489b7390b282b36e6b234fda2 Mon Sep 17 00:00:00 2001 From: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:42:00 +0000 Subject: [PATCH 4/5] Fix regex semver matching (#716) --- .github/workflows/_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index b9f4ee13c..7833d83a1 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -19,7 +19,7 @@ jobs: - name: Validate SemVer2 version compliance if: startsWith(github.ref, 'refs/tags/') env: - SEMVER_REGEX: ^refs/tags/(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ + SEMVER_REGEX: ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ run: | ref="${{ github.ref_name }}" From fa202044dd9add61cdcc540a52521cf243a12f83 Mon Sep 17 00:00:00 2001 From: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:07:07 +0000 Subject: [PATCH 5/5] Update to copier-template 2.5.0 (#717) --- .copier-answers.yml | 4 +- .devcontainer/devcontainer.json | 2 +- .github/CONTRIBUTING.md | 2 +- .github/ISSUE_TEMPLATE/bug_report.md | 21 +++++++++++ .github/ISSUE_TEMPLATE/issue.md | 13 +++++++ .../pull_request_template.md | 8 ++++ .github/pages/index.html | 2 +- .github/pages/make_switcher.py | 5 +++ .github/workflows/_container.yml | 8 +++- .github/workflows/_docs.yml | 4 +- .github/workflows/_pypi.yml | 2 + .github/workflows/_release.yml | 2 +- .github/workflows/ci.yml | 3 +- .gitignore | 1 + .pre-commit-config.yaml | 1 + .vscode/extensions.json | 2 +- .vscode/launch.json | 2 +- .vscode/settings.json | 3 +- .vscode/tasks.json | 2 +- Dockerfile | 6 +-- README.md | 2 +- docs/_api.rst | 16 ++++++++ docs/_templates/custom-module-template.rst | 37 +++++++++++++++++++ docs/conf.py | 27 +++++++++----- docs/explanations/lifecycle.md | 2 - docs/how-to/add-plans-and-devices.md | 1 - docs/how-to/contribute.md | 2 +- docs/reference.md | 1 + docs/reference/api.md | 10 ----- docs/reference/asyncapi.yaml | 1 - docs/reference/messaging-spec.md | 1 - docs/reference/rest-spec.md | 2 - docs/tutorials/installation.md | 2 +- helm/blueapi/templates/configmap.yaml | 1 - pyproject.toml | 29 ++++++++++----- src/blueapi/__init__.py | 8 ++++ src/blueapi/startup/simmotor.py | 4 +- src/blueapi/worker/task_worker.py | 4 +- src/script/rabbitmq_setup/enabled_plugins | 2 +- tests/conftest.py | 17 ++++++++- tests/system_tests/devices.json | 2 +- tests/system_tests/plans.json | 2 +- 42 files changed, 200 insertions(+), 66 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/issue.md create mode 100644 .github/PULL_REQUEST_TEMPLATE/pull_request_template.md create mode 100644 docs/_api.rst create mode 100644 docs/_templates/custom-module-template.rst delete mode 100644 docs/reference/api.md diff --git a/.copier-answers.yml b/.copier-answers.yml index 8f7e8de45..9e8e4cf69 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,9 +1,11 @@ # Changes here will be overwritten by Copier -_commit: 2.0.1 +_commit: 2.5.0 _src_path: gh:DiamondLightSource/python-copier-template author_email: callum.forrester@diamond.ac.uk author_name: Callum Forrester +component_lifecycle: production component_owner: user:vid18871 +component_type: service description: Lightweight bluesky-as-a-service wrapper application. Also usable as a library. distribution_name: blueapi diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b66046bc2..d0fae1bda 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -47,4 +47,4 @@ "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind", // After the container is created, install the python project in editable form "postCreateCommand": "pip install $([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e '.[dev]' && pre-commit install" -} \ No newline at end of file +} diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 05162541c..2978529ba 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -24,4 +24,4 @@ It is recommended that developers use a [vscode devcontainer](https://code.visua This project was created using the [Diamond Light Source Copier Template](https://github.com/DiamondLightSource/python-copier-template) for Python projects. -For more information on common tasks like setting up a developer environment, running the tests, and setting a pre-commit hook, see the template's [How-to guides](https://diamondlightsource.github.io/python-copier-template/2.0.1/how-to.html). +For more information on common tasks like setting up a developer environment, running the tests, and setting a pre-commit hook, see the template's [How-to guides](https://diamondlightsource.github.io/python-copier-template/2.5.0/how-to.html). diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..aa65892f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,21 @@ +--- +name: Bug Report +about: The template to use for reporting bugs and usability issues +title: " " +labels: 'bug' +assignees: '' + +--- + +Describe the bug, including a clear and concise description of the expected behavior, the actual behavior and the context in which you encountered it (ideally include details of your environment). + +## Steps To Reproduce +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + + +## Acceptance Criteria +- Specific criteria that will be used to judge if the issue is fixed diff --git a/.github/ISSUE_TEMPLATE/issue.md b/.github/ISSUE_TEMPLATE/issue.md new file mode 100644 index 000000000..52c84dd85 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue.md @@ -0,0 +1,13 @@ +--- +name: Issue +about: The standard template to use for feature requests, design discussions and tasks +title: " " +labels: '' +assignees: '' + +--- + +A brief description of the issue, including specific stakeholders and the business case where appropriate + +## Acceptance Criteria +- Specific criteria that will be used to judge if the issue is fixed diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md new file mode 100644 index 000000000..8200afe5c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -0,0 +1,8 @@ +Fixes #ISSUE + +### Instructions to reviewer on how to test: +1. Do thing x +2. Confirm thing y happens + +### Checks for reviewer +- [ ] Would the PR title make sense to a user on a set of release notes diff --git a/.github/pages/index.html b/.github/pages/index.html index 80f0a0091..c495f39f2 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -8,4 +8,4 @@ - \ No newline at end of file + diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index 14577cce6..c06813afa 100755 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -1,3 +1,5 @@ +"""Make switcher.json to allow docs to switch between different versions.""" + import json import logging from argparse import ArgumentParser @@ -6,6 +8,7 @@ def report_output(stdout: bytes, label: str) -> list[str]: + """Print and return something received frm stdout.""" ret = stdout.decode().strip().split("\n") print(f"{label}: {ret}") return ret @@ -53,6 +56,7 @@ def get_versions(ref: str, add: str | None) -> list[str]: def write_json(path: Path, repository: str, versions: list[str]): + """Write the JSON switcher to path.""" org, repo_name = repository.split("/") struct = [ {"version": version, "url": f"https://{org}.github.io/{repo_name}/{version}/"} @@ -64,6 +68,7 @@ def write_json(path: Path, repository: str, versions: list[str]): def main(args=None): + """Parse args and write switcher.""" parser = ArgumentParser( description="Make a versions.json file from gh-pages directories" ) diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index 7833d83a1..8688011c3 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -43,7 +43,9 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and export to Docker local cache - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_RECORD_UPLOAD: false with: context: . # Need load and tags so we can test it below @@ -64,7 +66,9 @@ jobs: - name: Push cached image to container registry if: github.ref_type == 'tag' - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_RECORD_UPLOAD: false # This does not build the image again, it will find the image in the # Docker cache and publish it with: diff --git a/.github/workflows/_docs.yml b/.github/workflows/_docs.yml index 40446e332..a1cafcaed 100644 --- a/.github/workflows/_docs.yml +++ b/.github/workflows/_docs.yml @@ -47,8 +47,8 @@ jobs: if: github.ref_type == 'tag' || github.ref_name == 'main' # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3 + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: .github/pages - keep_files: true \ No newline at end of file + keep_files: true diff --git a/.github/workflows/_pypi.yml b/.github/workflows/_pypi.yml index 0c5258dbe..8032bbaac 100644 --- a/.github/workflows/_pypi.yml +++ b/.github/workflows/_pypi.yml @@ -15,3 +15,5 @@ jobs: - name: Publish to PyPI using trusted publishing uses: pypa/gh-action-pypi-publish@release/v1 + with: + attestations: false diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index b49fa7dca..81b626438 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -23,7 +23,7 @@ jobs: - name: Create GitHub Release # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 + uses: softprops/action-gh-release@e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8 # v2.0.9 with: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} files: "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f45e2aaad..485b82b66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] include: # Include one that runs in the dev environment - runs-on: "ubuntu-latest" @@ -39,6 +39,7 @@ jobs: if: needs.check.outputs.branch-pr == '' uses: ./.github/workflows/_container.yml permissions: + contents: read packages: write docs: diff --git a/.gitignore b/.gitignore index a05f166b6..472e30e96 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ cov.xml # Sphinx documentation docs/_build/ +docs/_api # PyBuilder target/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4af381a0c..8c709a4bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,6 +6,7 @@ repos: - id: check-yaml exclude: ^helm\/.*\/templates\/.*|catalog-info.yaml - id: check-merge-conflict + - id: end-of-file-fixer - repo: local hooks: diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 66ad6324d..933c580cd 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -2,4 +2,4 @@ "recommendations": [ "ms-vscode-remote.remote-containers", ] -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index 8291038c7..7e47213ba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -78,4 +78,4 @@ "default": "" }, ] -} \ No newline at end of file +} diff --git a/.vscode/settings.json b/.vscode/settings.json index c129d991b..101c75fa7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,7 +5,8 @@ "editor.codeActionsOnSave": { "source.organizeImports": "explicit" }, + "files.insertFinalNewline": true, "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", }, -} \ No newline at end of file +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 946e69d4b..c999e8646 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -13,4 +13,4 @@ "problemMatcher": [], } ] -} \ No newline at end of file +} diff --git a/Dockerfile b/Dockerfile index dd1a2398f..06b988263 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # The devcontainer should use the developer target and run as root with podman # or docker with user namespaces. ARG PYTHON_VERSION=3.11 -FROM python:${PYTHON_VERSION} as developer +FROM python:${PYTHON_VERSION} AS developer # Add any system dependencies for the developer/build environment here RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -13,13 +13,13 @@ RUN python -m venv /venv ENV PATH=/venv/bin:$PATH # The build stage installs the context into the venv -FROM developer as build +FROM developer AS build COPY . /context WORKDIR /context RUN touch dev-requirements.txt && pip install --upgrade pip && pip install -c dev-requirements.txt . # The runtime stage copies the built venv into a slim runtime container -FROM python:${PYTHON_VERSION}-slim as runtime +FROM python:${PYTHON_VERSION}-slim AS runtime # Add apt-get system dependecies for runtime here if needed RUN apt-get update && apt-get install -y --no-install-recommends \ # Git required for installing packages at runtime diff --git a/README.md b/README.md index 5e44be3d9..b026487e5 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![CI](https://github.com/DiamondLightSource/blueapi/actions/workflows/ci.yml/badge.svg)](https://github.com/DiamondLightSource/blueapi/actions/workflows/ci.yml) [![Coverage](https://codecov.io/gh/DiamondLightSource/blueapi/branch/main/graph/badge.svg)](https://codecov.io/gh/DiamondLightSource/blueapi) [![PyPI](https://img.shields.io/pypi/v/blueapi.svg)](https://pypi.org/project/blueapi) -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) # blueapi diff --git a/docs/_api.rst b/docs/_api.rst new file mode 100644 index 000000000..d0f030ac4 --- /dev/null +++ b/docs/_api.rst @@ -0,0 +1,16 @@ +:orphan: + +.. + This page is not included in the TOC tree, but must exist so that the + autosummary pages are generated for blueapi and all its + subpackages + +API +=== + +.. autosummary:: + :toctree: _api + :template: custom-module-template.rst + :recursive: + + blueapi diff --git a/docs/_templates/custom-module-template.rst b/docs/_templates/custom-module-template.rst new file mode 100644 index 000000000..9aeca5401 --- /dev/null +++ b/docs/_templates/custom-module-template.rst @@ -0,0 +1,37 @@ +{{ ('``' + fullname + '``') | underline }} + +{%- set filtered_members = [] %} +{%- for item in members %} + {%- if item in functions + classes + exceptions + attributes %} + {% set _ = filtered_members.append(item) %} + {%- endif %} +{%- endfor %} + +.. automodule:: {{ fullname }} + :members: + + {% block modules %} + {% if modules %} + .. rubric:: Submodules + + .. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: + {% for item in modules %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block members %} + {% if filtered_members %} + .. rubric:: Members + + .. autosummary:: + :nosignatures: + {% for item in filtered_members %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 23fb13a2b..0e96d9dea 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,8 +1,9 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +"""Configuration file for the Sphinx documentation builder. + +This file only contains a selection of the most common options. For a full +list see the documentation: +https://www.sphinx-doc.org/en/master/usage/configuration.html +""" import sys from pathlib import Path @@ -32,6 +33,8 @@ extensions = [ # Use this for generating API docs "sphinx.ext.autodoc", + # and making summary tables at the top of API docs + "sphinx.ext.autosummary", # This can parse google style docstrings "sphinx.ext.napoleon", # For linking to external sphinx documentation @@ -84,6 +87,12 @@ # Don't inherit docstrings from baseclasses autodoc_inherit_docstrings = False +# Document only what is in __all__ +autosummary_ignore_module_all = False + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + # Output graphviz directive produced images in a scalable format graphviz_output_format = "svg" @@ -139,10 +148,10 @@ # Theme options for pydata_sphinx_theme # We don't check switcher because there are 3 possible states for a repo: # 1. New project, docs are not published so there is no switcher -# 2. Existing project with latest skeleton, switcher exists and works -# 3. Existing project with old skeleton that makes broken switcher, +# 2. Existing project with latest copier template, switcher exists and works +# 3. Existing project with old copier template that makes broken switcher, # switcher exists but is broken -# Point 3 makes checking switcher difficult, because the updated skeleton +# Point 3 makes checking switcher difficult, because the updated copier template # will fix the switcher at the end of the docs workflow, but never gets a chance # to complete as the docs build warns and fails. html_theme_options = { @@ -170,7 +179,7 @@ # A dictionary of values to pass into the template engine’s context for all pages html_context = { "github_user": github_user, - "github_repo": project, + "github_repo": github_repo, "github_version": version, "doc_path": "docs", } diff --git a/docs/explanations/lifecycle.md b/docs/explanations/lifecycle.md index f0455d28b..78018f1f3 100644 --- a/docs/explanations/lifecycle.md +++ b/docs/explanations/lifecycle.md @@ -124,5 +124,3 @@ The plan is executed. While it is running, the `Worker` will publish If an error occurs during any of the stages from "Request" onwards it is sent back to the user over the message bus. - - diff --git a/docs/how-to/add-plans-and-devices.md b/docs/how-to/add-plans-and-devices.md index a2c12e0b2..28aca4df3 100644 --- a/docs/how-to/add-plans-and-devices.md +++ b/docs/how-to/add-plans-and-devices.md @@ -100,4 +100,3 @@ host machine, include the following in your `values.yaml`: ``` You can then clone projects into the scratch directory and blueapi will automatically incorporate them on startup. You must still include configuration to load the plans and devices from specific modules within those packages, see above. - diff --git a/docs/how-to/contribute.md b/docs/how-to/contribute.md index f9c4ca1d7..6e4197970 100644 --- a/docs/how-to/contribute.md +++ b/docs/how-to/contribute.md @@ -1,2 +1,2 @@ ```{include} ../../.github/CONTRIBUTING.md -``` \ No newline at end of file +``` diff --git a/docs/reference.md b/docs/reference.md index 3be116edd..3b6db4d05 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -7,6 +7,7 @@ Technical reference material including APIs and release notes. :glob: reference/* +API <_api/blueapi> genindex Release Notes ``` diff --git a/docs/reference/api.md b/docs/reference/api.md deleted file mode 100644 index 1d29fbcd7..000000000 --- a/docs/reference/api.md +++ /dev/null @@ -1,10 +0,0 @@ -# API - -This is the internal API reference for blueapi - -```python -import blueapi -blueapi.__version__ -``` - -Version number as calculated by https://github.com/pypa/setuptools_scm diff --git a/docs/reference/asyncapi.yaml b/docs/reference/asyncapi.yaml index 96d314297..f78394420 100644 --- a/docs/reference/asyncapi.yaml +++ b/docs/reference/asyncapi.yaml @@ -263,4 +263,3 @@ components: description: Estimated time remaining until operation completion, if known type: number format: float - diff --git a/docs/reference/messaging-spec.md b/docs/reference/messaging-spec.md index 363577c80..d5c52e5d6 100644 --- a/docs/reference/messaging-spec.md +++ b/docs/reference/messaging-spec.md @@ -4,4 +4,3 @@ The Blueapi service publishes Bluesky documents and other events to the message bus, allowing subscribers to keep track of the status of plans, as well as other status changes. This page documents the channels to which clients can subscribe to receive these messages and their structure. - diff --git a/docs/reference/rest-spec.md b/docs/reference/rest-spec.md index 8db73b848..7a16c50b9 100644 --- a/docs/reference/rest-spec.md +++ b/docs/reference/rest-spec.md @@ -2,5 +2,3 @@ Blueapi runs a FastAPI server through which the blueapi worker can be interacted with. Here the [openapi docs page](./openapi.yaml) documents all possible endpoints of this server. - - diff --git a/docs/tutorials/installation.md b/docs/tutorials/installation.md index effcec222..bd9cfb2e2 100644 --- a/docs/tutorials/installation.md +++ b/docs/tutorials/installation.md @@ -2,7 +2,7 @@ ## Check your version of python -You will need python 3.8 or later. You can check your version of python by +You will need python 3.10 or later. You can check your version of python by typing into a terminal: ``` python3 --version diff --git a/helm/blueapi/templates/configmap.yaml b/helm/blueapi/templates/configmap.yaml index 868a78279..43ea33b03 100644 --- a/helm/blueapi/templates/configmap.yaml +++ b/helm/blueapi/templates/configmap.yaml @@ -5,4 +5,3 @@ metadata: data: config.yaml: |- {{- toYaml .Values.worker | nindent 4 }} - diff --git a/pyproject.toml b/pyproject.toml index a8a22e2d4..8658d979a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2"] +requires = ["setuptools>=64", "setuptools_scm[toml]>=8"] build-backend = "setuptools.build_meta" [project] @@ -9,6 +9,7 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] description = "Lightweight bluesky-as-a-service wrapper application. Also usable as a library." dependencies = [ @@ -49,6 +50,7 @@ dev = [ "pre-commit>=3.8.0", "pydata-sphinx-theme>=0.15.4", "mypy", + "pytest", "pytest-cov", "pytest-asyncio", "responses", @@ -78,7 +80,7 @@ name = "Callum Forrester" [tool.setuptools_scm] -write_to = "src/blueapi/_version.py" +version_file = "src/blueapi/_version.py" [tool.mypy] ignore_missing_imports = true # Ignore missing stubs in imported modules @@ -124,7 +126,7 @@ allowlist_externals = sphinx-build sphinx-autobuild commands = - pre-commit: pre-commit run --all-files {posargs} + pre-commit: pre-commit run --all-files --show-diff-on-failure {posargs} type-checking: mypy src tests {posargs} tests: pytest --cov=blueapi --cov-report term --cov-report xml:cov.xml tests/unit_tests {posargs} docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html @@ -135,13 +137,14 @@ commands = src = ["src", "tests"] line-length = 88 lint.select = [ - "B", # flake8-bugbear - https://docs.astral.sh/ruff/rules/#flake8-bugbear-b - "C4", # flake8-comprehensions - https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4 - "E", # pycodestyle errors - https://docs.astral.sh/ruff/rules/#error-e - "F", # pyflakes rules - https://docs.astral.sh/ruff/rules/#pyflakes-f - "W", # pycodestyle warnings - https://docs.astral.sh/ruff/rules/#warning-w - "I", # isort - https://docs.astral.sh/ruff/rules/#isort-i - "UP", # pyupgrade - https://docs.astral.sh/ruff/rules/#pyupgrade-up + "B", # flake8-bugbear - https://docs.astral.sh/ruff/rules/#flake8-bugbear-b + "C4", # flake8-comprehensions - https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4 + "E", # pycodestyle errors - https://docs.astral.sh/ruff/rules/#error-e + "F", # pyflakes rules - https://docs.astral.sh/ruff/rules/#pyflakes-f + "W", # pycodestyle warnings - https://docs.astral.sh/ruff/rules/#warning-w + "I", # isort - https://docs.astral.sh/ruff/rules/#isort-i + "UP", # pyupgrade - https://docs.astral.sh/ruff/rules/#pyupgrade-up + "SLF", # self - https://docs.astral.sh/ruff/settings/#lintflake8-self ] [tool.ruff.lint.flake8-bugbear] @@ -151,3 +154,9 @@ extend-immutable-calls = [ "fastapi.Task", "dodal.common.inject", ] + +[tool.ruff.lint.per-file-ignores] +# By default, private member access is allowed in tests +# See https://github.com/DiamondLightSource/python-copier-template/issues/154 +# Remove this line to forbid private member access in tests +"tests/**/*" = ["SLF001"] diff --git a/src/blueapi/__init__.py b/src/blueapi/__init__.py index 26d23badb..a2ffbf369 100644 --- a/src/blueapi/__init__.py +++ b/src/blueapi/__init__.py @@ -1,3 +1,11 @@ +"""Top level API. + +.. data:: __version__ + :type: str + + Version number as calculated by https://github.com/pypa/setuptools_scm +""" + from ._version import __version__ __all__ = ["__version__"] diff --git a/src/blueapi/startup/simmotor.py b/src/blueapi/startup/simmotor.py index 84067ba21..1d8f9c4a6 100644 --- a/src/blueapi/startup/simmotor.py +++ b/src/blueapi/startup/simmotor.py @@ -43,7 +43,7 @@ def set(self, value: float) -> None: distance = value - old_setpoint self.sim_state["setpoint"] = value self.sim_state["setpoint_ts"] = ttime.time() - self.setpoint._run_subs( + self.setpoint._run_subs( # noqa sub_type=self.setpoint.SUB_VALUE, old_value=old_setpoint, value=self.sim_state["setpoint"], @@ -54,7 +54,7 @@ def update_state(position: float) -> None: old_readback = self.sim_state["readback"] self.sim_state["readback"] = self._readback_func(position) self.sim_state["readback_ts"] = ttime.time() - self.readback._run_subs( + self.readback._run_subs( # noqa sub_type=self.readback.SUB_VALUE, old_value=old_readback, value=self.sim_state["readback"], diff --git a/src/blueapi/worker/task_worker.py b/src/blueapi/worker/task_worker.py index d9aa38e86..db760f421 100644 --- a/src/blueapi/worker/task_worker.py +++ b/src/blueapi/worker/task_worker.py @@ -239,11 +239,9 @@ def start(self) -> None: if self._started.is_set(): raise WorkerAlreadyStartedError("Worker is already running") self._wait_until_stopped() - fut = run_worker_in_own_thread(self) + run_worker_in_own_thread(self) self._wait_until_started() - add_span_attributes({WORKER_THREAD_STATE: fut._state}) - @start_as_current_span(TRACER) def stop(self) -> None: LOGGER.info("Attempting to stop worker") diff --git a/src/script/rabbitmq_setup/enabled_plugins b/src/script/rabbitmq_setup/enabled_plugins index 454685a05..37920ac73 100644 --- a/src/script/rabbitmq_setup/enabled_plugins +++ b/src/script/rabbitmq_setup/enabled_plugins @@ -1 +1 @@ -[rabbitmq_management,rabbitmq_prometheus,rabbitmq_stomp]. \ No newline at end of file +[rabbitmq_management,rabbitmq_prometheus,rabbitmq_stomp]. diff --git a/tests/conftest.py b/tests/conftest.py index 8f311754b..b61c09e9d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,5 @@ import asyncio -from typing import cast +from typing import Any, cast # Based on https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option # noqa: E501 import pytest @@ -39,3 +39,18 @@ def exporter() -> TracerProvider: # Use SimpleSpanProcessor to keep tests quick provider.add_span_processor(SimpleSpanProcessor(exporter)) return exporter + + +@pytest.hookimpl(tryfirst=True) +def pytest_exception_interact(call: pytest.CallInfo[Any]): + if call.excinfo is not None: + raise call.excinfo.value + else: + raise RuntimeError( + f"{call} has no exception data, an unknown error has occurred" + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_internalerror(excinfo: pytest.ExceptionInfo[Any]): + raise excinfo.value diff --git a/tests/system_tests/devices.json b/tests/system_tests/devices.json index b13566d48..c6c153503 100644 --- a/tests/system_tests/devices.json +++ b/tests/system_tests/devices.json @@ -149,4 +149,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/tests/system_tests/plans.json b/tests/system_tests/plans.json index 76d17457a..d9a8b551e 100644 --- a/tests/system_tests/plans.json +++ b/tests/system_tests/plans.json @@ -1165,4 +1165,4 @@ } } ] -} \ No newline at end of file +}