From a0ca2c06e842d1e761e9fda5ae93ff90be80da85 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 20 Apr 2023 14:26:37 +0000 Subject: [PATCH 01/12] rebased --- src/blueapi/__init__.py | 3 + src/blueapi/rest/__init__.py | 0 src/blueapi/rest/app.py | 58 +++++++++++++ src/blueapi/service/app.py | 151 ++++++++++++++++++++++++++++++++++ src/blueapi/service/rest.py | 7 ++ src/blueapi/service/routes.py | 38 +++++++++ 6 files changed, 257 insertions(+) create mode 100644 src/blueapi/rest/__init__.py create mode 100644 src/blueapi/rest/app.py create mode 100644 src/blueapi/service/app.py create mode 100644 src/blueapi/service/rest.py create mode 100644 src/blueapi/service/routes.py diff --git a/src/blueapi/__init__.py b/src/blueapi/__init__.py index bdccda11d..c701957c5 100644 --- a/src/blueapi/__init__.py +++ b/src/blueapi/__init__.py @@ -1,4 +1,7 @@ from importlib.metadata import version +from blueapi.core.context import BlueskyContext + +from blueapi.worker.reworker import RunEngineWorker __version__ = version("blueapi") del version diff --git a/src/blueapi/rest/__init__.py b/src/blueapi/rest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/blueapi/rest/app.py b/src/blueapi/rest/app.py new file mode 100644 index 000000000..f0026d96e --- /dev/null +++ b/src/blueapi/rest/app.py @@ -0,0 +1,58 @@ +from pathlib import Path +from typing import Mapping, Optional +from fastapi import FastAPI +from blueapi.core.context import BlueskyContext +from blueapi.core.event import EventStream +from blueapi.messaging.stomptemplate import StompMessagingTemplate, MessagingTemplate +from blueapi.utils.config import ConfigLoader +from blueapi.worker import run_worker_in_own_thread +from blueapi.worker.reworker import RunEngineWorker +from blueapi.config import ApplicationConfig +from blueapi.worker.worker import Worker +import logging + +app = () + + +class RestApi: + _config: ApplicationConfig + _message_bus: MessagingTemplate + _ctx: BlueskyContext + _worker: Worker + _app: FastAPI + + def __init__(self, config: ApplicationConfig) -> None: + self._config = config + self._ctx = BlueskyContext() + self._ctx.with_startup_script(self._config.env.startup_script) + self._worker = RunEngineWorker(self._ctx) + self._worker_future = run_worker_in_own_thread(self._worker) + self._message_bus = StompMessagingTemplate.autoconfigured(config.stomp) + + def run(self) -> None: + logging.basicConfig(level=self._config.logging.level) + + self._worker.data_events.subscribe( + lambda event, corr_id: self._message_bus.send( + "public.worker.event.data", event, None, corr_id + ) + ) + self._worker.progress_events.subscribe( + lambda event, corr_id: self._message_bus.send( + "public.worker.event.progress", event, None, corr_id + ) + ) + + self._message_bus.connect() + self._app = FastAPI() + + self._worker.run_forever() + + +def start(config_path: Optional[Path] = None): + loader = ConfigLoader(ApplicationConfig) + if config_path is not None: + loader.use_yaml_or_json_file(config_path) + config = loader.load() + + RestApi(config).run() diff --git a/src/blueapi/service/app.py b/src/blueapi/service/app.py new file mode 100644 index 000000000..d9002e1cf --- /dev/null +++ b/src/blueapi/service/app.py @@ -0,0 +1,151 @@ +import logging +import uuid +from pathlib import Path +from typing import Mapping, Optional + +from fastapi import FastAPI + +from blueapi.config import ApplicationConfig +from blueapi.core import BlueskyContext, EventStream +from blueapi.messaging import MessageContext, MessagingTemplate, StompMessagingTemplate +from blueapi.utils import ConfigLoader +from blueapi.worker import RunEngineWorker, RunPlan, Worker +from blueapi import context, worker +from blueapi.worker.multithread import run_worker_in_own_thread + +from .routes import router + +from .model import ( + DeviceModel, + DeviceRequest, + DeviceResponse, + PlanModel, + PlanRequest, + PlanResponse, + TaskResponse, +) + + +class Service: + _config: ApplicationConfig + _ctx: BlueskyContext + _worker: Worker + _template: MessagingTemplate + + def __init__(self, config: ApplicationConfig) -> None: + self._config = config + self._ctx = BlueskyContext() + self._ctx.with_startup_script(self._config.env.startup_script) + self._worker = RunEngineWorker(self._ctx) + self._template = StompMessagingTemplate.autoconfigured(config.stomp) + + def run(self) -> None: + logging.basicConfig(level=self._config.logging.level) + + self._publish_event_streams( + { + self._worker.worker_events: self._template.destinations.topic( + "public.worker.event" + ), + self._worker.progress_events: self._template.destinations.topic( + "public.worker.event.progress" + ), + self._worker.data_events: self._template.destinations.topic( + "public.worker.event.data" + ), + } + ) + + self._template.subscribe(" ", self._on_run_request) + self._template.subscribe("worker.plans", self._get_plans) + self._template.subscribe("worker.devices", self._get_devices) + + self._template.connect() + + self._worker.run_forever() + + def _publish_event_streams( + self, streams_to_destinations: Mapping[EventStream, str] + ) -> None: + for stream, destination in streams_to_destinations.items(): + self._publish_event_stream(stream, destination) + + def _publish_event_stream(self, stream: EventStream, destination: str) -> None: + stream.subscribe( + lambda event, correlation_id: self._template.send( + destination, event, None, correlation_id + ) + ) + + def _on_run_request(self, message_context: MessageContext, task: RunPlan) -> None: + correlation_id = message_context.correlation_id or str(uuid.uuid1()) + self._worker.submit_task(correlation_id, task) + + reply_queue = message_context.reply_destination + if reply_queue is not None: + response = TaskResponse(task_name=correlation_id) + self._template.send(reply_queue, response) + + def _get_plans(self, message_context: MessageContext, message: PlanRequest) -> None: + plans = list(map(PlanModel.from_plan, self._ctx.plans.values())) + response = PlanResponse(plans=plans) + + assert message_context.reply_destination is not None + self._template.send(message_context.reply_destination, response) + + def _get_devices( + self, message_context: MessageContext, message: DeviceRequest + ) -> None: + devices = list(map(DeviceModel.from_device, self._ctx.devices.values())) + response = DeviceResponse(devices=devices) + + assert message_context.reply_destination is not None + self._template.send(message_context.reply_destination, response) + + +##need to globally, start the worker and message bus. +## message bus needs a config file, +## worker needs a context, +## context needs a config file. + +## so how about, we set up a context somewhere (in context module), +## we start up the worker with the context, +# THEN in this start we load config into the context and load the message bus from the config. + +## the rest api never needs to interact with the message bus anyways... it only interacts with context or worker. + + +def start(config_path: Optional[Path] = None): + # 1. load config and setup logging + loader = ConfigLoader(ApplicationConfig) + if config_path is not None: + loader.use_yaml_or_json_file(config_path) + config = loader.load() + logging.basicConfig(level=config.logging.level) + + # 2. set context with startup script + context.with_startup_script(config.env.startup_script) + + # 3. run the worker in it's own thread + worker_future = run_worker_in_own_thread(worker) + + # 4. create a message bus and subscribe all relevant worker docs to it + message_bus = StompMessagingTemplate.autoconfigured(config.stomp) + worker.data_events.subscribe( + lambda event, corr_id: message_bus.send( + "public.worker.event.data", event, None, corr_id + ) + ) + worker.progress_events.subscribe( + lambda event, corr_id: message_bus.send( + "public.worker.event.progress", event, None, corr_id + ) + ) + + # 5. start the message bus + message_bus.connect() + + # 7. run the worker forever + worker.run_forever() + + # Service(config).run() diff --git a/src/blueapi/service/rest.py b/src/blueapi/service/rest.py new file mode 100644 index 000000000..905e0e8f3 --- /dev/null +++ b/src/blueapi/service/rest.py @@ -0,0 +1,7 @@ +from blueapi.service.routes import router +from fastapi import FastAPI + +app = FastAPI() + +# here, do app.include_router from all the other routes you want. +app.include_router(router) diff --git a/src/blueapi/service/routes.py b/src/blueapi/service/routes.py new file mode 100644 index 000000000..16e41aacc --- /dev/null +++ b/src/blueapi/service/routes.py @@ -0,0 +1,38 @@ +from fastapi import APIRouter +from blueapi import context, worker + +router = APIRouter() + + +@router.get("/plans") +async def get_plans(): + context.plans + ... + + +@router.get("/plan/{name}") +async def get_plan_by_name(name: str): + try: + context.plans[name] + except IndexError: + raise Exception() # really, return a 404. + + +@router.get("/devices") +async def get_devices(): + context.devices + + +@router.get("/device/{name}") +async def get_device_by_name(name: str): + try: + context.plans[name] + except IndexError: + raise Exception() # really, return a 404. + + +@router.put("task/{name}") +async def execute_task(name: str): + ##basically in here, do the same thing the service once did... + # worker.submit_task(name, task) + pass From 633283f6ac02ee393479082aef1ef259ffffd498 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 10:27:39 +0000 Subject: [PATCH 02/12] added a catalog info and openapi schema --- catalog-info.yaml | 28 +++ openapi.json | 323 +++++++++++++++++++++++++++++++++ src/blueapi/__init__.py | 3 - src/blueapi/rest/__init__.py | 0 src/blueapi/rest/app.py | 58 ------ src/blueapi/service/app.py | 151 --------------- src/blueapi/service/openapi.py | 23 +++ src/blueapi/service/rest.py | 7 - src/blueapi/service/routes.py | 38 ---- tests/rest/test_rest_api.py | 116 ++++++++++++ 10 files changed, 490 insertions(+), 257 deletions(-) create mode 100644 catalog-info.yaml create mode 100644 openapi.json delete mode 100644 src/blueapi/rest/__init__.py delete mode 100644 src/blueapi/rest/app.py delete mode 100644 src/blueapi/service/app.py create mode 100644 src/blueapi/service/openapi.py delete mode 100644 src/blueapi/service/rest.py delete mode 100644 src/blueapi/service/routes.py create mode 100644 tests/rest/test_rest_api.py diff --git a/catalog-info.yaml b/catalog-info.yaml new file mode 100644 index 000000000..230e7305e --- /dev/null +++ b/catalog-info.yaml @@ -0,0 +1,28 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: bluesky-worker + title: bluesky-worker + description: Lightweight wrapper around bluesky services +spec: + type: service + lifecycle: production + owner: user:vid18871 # TODO: owner: DAQ-Core + # system: Athena # TODO: Define Athena system: presumably same location as DAQ-Core/DAQ? + providesApis: + - blueapi + - blueskydocument-to-scanmessage + +--- +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: blueapi + title: blueapi + description: REST API for getting plans/devices from the worker (and running tasks) +spec: + type: openapi + lifecycle: production + owner: user:vid18871 + definition: + $text: ./openapi.json diff --git a/openapi.json b/openapi.json new file mode 100644 index 000000000..0d1ee40e9 --- /dev/null +++ b/openapi.json @@ -0,0 +1,323 @@ +{ + "openapi": "3.0.2", + "info": { + "title": "FastAPI", + "version": "0.1.0" + }, + "paths": { + "/plans": { + "get": { + "summary": "Get Plans", + "operationId": "get_plans_plans_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PlanResponse" + } + } + } + } + } + } + }, + "/plan/{name}": { + "get": { + "summary": "Get Plan By Name", + "operationId": "get_plan_by_name_plan__name__get", + "parameters": [ + { + "required": true, + "schema": { + "title": "Name", + "type": "string" + }, + "name": "name", + "in": "path" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PlanModel" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/devices": { + "get": { + "summary": "Get Devices", + "operationId": "get_devices_devices_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeviceResponse" + } + } + } + } + } + } + }, + "/device/{name}": { + "get": { + "summary": "Get Device By Name", + "operationId": "get_device_by_name_device__name__get", + "parameters": [ + { + "required": true, + "schema": { + "title": "Name", + "type": "string" + }, + "name": "name", + "in": "path" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeviceModel" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/task/{name}": { + "put": { + "summary": "Execute Task", + "operationId": "execute_task_task__name__put", + "parameters": [ + { + "required": true, + "schema": { + "title": "Name", + "type": "string" + }, + "name": "name", + "in": "path" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RunPlan" + }, + "example": { + "name": "count", + "params": { + "detectors": [ + "x" + ] + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "DeviceModel": { + "title": "DeviceModel", + "required": [ + "name", + "protocols" + ], + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string", + "description": "Name of the device" + }, + "protocols": { + "title": "Protocols", + "type": "array", + "items": { + "type": "string" + }, + "description": "Protocols that a device conforms to, indicating its capabilities" + } + }, + "description": "Representation of a device" + }, + "DeviceResponse": { + "title": "DeviceResponse", + "required": [ + "devices" + ], + "type": "object", + "properties": { + "devices": { + "title": "Devices", + "type": "array", + "items": { + "$ref": "#/components/schemas/DeviceModel" + }, + "description": "Devices available to use in plans" + } + }, + "description": "Response to a query for devices" + }, + "HTTPValidationError": { + "title": "HTTPValidationError", + "type": "object", + "properties": { + "detail": { + "title": "Detail", + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + } + } + } + }, + "PlanModel": { + "title": "PlanModel", + "required": [ + "name" + ], + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string", + "description": "Name of the plan" + } + }, + "description": "Representation of a plan" + }, + "PlanResponse": { + "title": "PlanResponse", + "required": [ + "plans" + ], + "type": "object", + "properties": { + "plans": { + "title": "Plans", + "type": "array", + "items": { + "$ref": "#/components/schemas/PlanModel" + }, + "description": "Plans available to use by a worker" + } + }, + "description": "Response to a query for plans" + }, + "RunPlan": { + "title": "RunPlan", + "required": [ + "name" + ], + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string", + "description": "Name of plan to run" + }, + "params": { + "title": "Params", + "type": "object", + "description": "Values for parameters to plan, if any" + } + }, + "description": "Task that will run a plan" + }, + "ValidationError": { + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "type": "object", + "properties": { + "loc": { + "title": "Location", + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + } + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/src/blueapi/__init__.py b/src/blueapi/__init__.py index c701957c5..bdccda11d 100644 --- a/src/blueapi/__init__.py +++ b/src/blueapi/__init__.py @@ -1,7 +1,4 @@ from importlib.metadata import version -from blueapi.core.context import BlueskyContext - -from blueapi.worker.reworker import RunEngineWorker __version__ = version("blueapi") del version diff --git a/src/blueapi/rest/__init__.py b/src/blueapi/rest/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/blueapi/rest/app.py b/src/blueapi/rest/app.py deleted file mode 100644 index f0026d96e..000000000 --- a/src/blueapi/rest/app.py +++ /dev/null @@ -1,58 +0,0 @@ -from pathlib import Path -from typing import Mapping, Optional -from fastapi import FastAPI -from blueapi.core.context import BlueskyContext -from blueapi.core.event import EventStream -from blueapi.messaging.stomptemplate import StompMessagingTemplate, MessagingTemplate -from blueapi.utils.config import ConfigLoader -from blueapi.worker import run_worker_in_own_thread -from blueapi.worker.reworker import RunEngineWorker -from blueapi.config import ApplicationConfig -from blueapi.worker.worker import Worker -import logging - -app = () - - -class RestApi: - _config: ApplicationConfig - _message_bus: MessagingTemplate - _ctx: BlueskyContext - _worker: Worker - _app: FastAPI - - def __init__(self, config: ApplicationConfig) -> None: - self._config = config - self._ctx = BlueskyContext() - self._ctx.with_startup_script(self._config.env.startup_script) - self._worker = RunEngineWorker(self._ctx) - self._worker_future = run_worker_in_own_thread(self._worker) - self._message_bus = StompMessagingTemplate.autoconfigured(config.stomp) - - def run(self) -> None: - logging.basicConfig(level=self._config.logging.level) - - self._worker.data_events.subscribe( - lambda event, corr_id: self._message_bus.send( - "public.worker.event.data", event, None, corr_id - ) - ) - self._worker.progress_events.subscribe( - lambda event, corr_id: self._message_bus.send( - "public.worker.event.progress", event, None, corr_id - ) - ) - - self._message_bus.connect() - self._app = FastAPI() - - self._worker.run_forever() - - -def start(config_path: Optional[Path] = None): - loader = ConfigLoader(ApplicationConfig) - if config_path is not None: - loader.use_yaml_or_json_file(config_path) - config = loader.load() - - RestApi(config).run() diff --git a/src/blueapi/service/app.py b/src/blueapi/service/app.py deleted file mode 100644 index d9002e1cf..000000000 --- a/src/blueapi/service/app.py +++ /dev/null @@ -1,151 +0,0 @@ -import logging -import uuid -from pathlib import Path -from typing import Mapping, Optional - -from fastapi import FastAPI - -from blueapi.config import ApplicationConfig -from blueapi.core import BlueskyContext, EventStream -from blueapi.messaging import MessageContext, MessagingTemplate, StompMessagingTemplate -from blueapi.utils import ConfigLoader -from blueapi.worker import RunEngineWorker, RunPlan, Worker -from blueapi import context, worker -from blueapi.worker.multithread import run_worker_in_own_thread - -from .routes import router - -from .model import ( - DeviceModel, - DeviceRequest, - DeviceResponse, - PlanModel, - PlanRequest, - PlanResponse, - TaskResponse, -) - - -class Service: - _config: ApplicationConfig - _ctx: BlueskyContext - _worker: Worker - _template: MessagingTemplate - - def __init__(self, config: ApplicationConfig) -> None: - self._config = config - self._ctx = BlueskyContext() - self._ctx.with_startup_script(self._config.env.startup_script) - self._worker = RunEngineWorker(self._ctx) - self._template = StompMessagingTemplate.autoconfigured(config.stomp) - - def run(self) -> None: - logging.basicConfig(level=self._config.logging.level) - - self._publish_event_streams( - { - self._worker.worker_events: self._template.destinations.topic( - "public.worker.event" - ), - self._worker.progress_events: self._template.destinations.topic( - "public.worker.event.progress" - ), - self._worker.data_events: self._template.destinations.topic( - "public.worker.event.data" - ), - } - ) - - self._template.subscribe(" ", self._on_run_request) - self._template.subscribe("worker.plans", self._get_plans) - self._template.subscribe("worker.devices", self._get_devices) - - self._template.connect() - - self._worker.run_forever() - - def _publish_event_streams( - self, streams_to_destinations: Mapping[EventStream, str] - ) -> None: - for stream, destination in streams_to_destinations.items(): - self._publish_event_stream(stream, destination) - - def _publish_event_stream(self, stream: EventStream, destination: str) -> None: - stream.subscribe( - lambda event, correlation_id: self._template.send( - destination, event, None, correlation_id - ) - ) - - def _on_run_request(self, message_context: MessageContext, task: RunPlan) -> None: - correlation_id = message_context.correlation_id or str(uuid.uuid1()) - self._worker.submit_task(correlation_id, task) - - reply_queue = message_context.reply_destination - if reply_queue is not None: - response = TaskResponse(task_name=correlation_id) - self._template.send(reply_queue, response) - - def _get_plans(self, message_context: MessageContext, message: PlanRequest) -> None: - plans = list(map(PlanModel.from_plan, self._ctx.plans.values())) - response = PlanResponse(plans=plans) - - assert message_context.reply_destination is not None - self._template.send(message_context.reply_destination, response) - - def _get_devices( - self, message_context: MessageContext, message: DeviceRequest - ) -> None: - devices = list(map(DeviceModel.from_device, self._ctx.devices.values())) - response = DeviceResponse(devices=devices) - - assert message_context.reply_destination is not None - self._template.send(message_context.reply_destination, response) - - -##need to globally, start the worker and message bus. -## message bus needs a config file, -## worker needs a context, -## context needs a config file. - -## so how about, we set up a context somewhere (in context module), -## we start up the worker with the context, -# THEN in this start we load config into the context and load the message bus from the config. - -## the rest api never needs to interact with the message bus anyways... it only interacts with context or worker. - - -def start(config_path: Optional[Path] = None): - # 1. load config and setup logging - loader = ConfigLoader(ApplicationConfig) - if config_path is not None: - loader.use_yaml_or_json_file(config_path) - config = loader.load() - logging.basicConfig(level=config.logging.level) - - # 2. set context with startup script - context.with_startup_script(config.env.startup_script) - - # 3. run the worker in it's own thread - worker_future = run_worker_in_own_thread(worker) - - # 4. create a message bus and subscribe all relevant worker docs to it - message_bus = StompMessagingTemplate.autoconfigured(config.stomp) - worker.data_events.subscribe( - lambda event, corr_id: message_bus.send( - "public.worker.event.data", event, None, corr_id - ) - ) - worker.progress_events.subscribe( - lambda event, corr_id: message_bus.send( - "public.worker.event.progress", event, None, corr_id - ) - ) - - # 5. start the message bus - message_bus.connect() - - # 7. run the worker forever - worker.run_forever() - - # Service(config).run() diff --git a/src/blueapi/service/openapi.py b/src/blueapi/service/openapi.py new file mode 100644 index 000000000..e537f1718 --- /dev/null +++ b/src/blueapi/service/openapi.py @@ -0,0 +1,23 @@ +"""Generate openapi.json.""" + +import json + +from fastapi.openapi.utils import get_openapi + +from blueapi.rest.main import app + +if __name__ == "__main__": + with open("openapi.json", "w") as f: + json.dump( + get_openapi( + title=app.title, + version=app.version, + openapi_version=app.openapi_version, + description=app.description, + routes=app.routes, + ), + f, + indent=4, + ) + + print("ah") diff --git a/src/blueapi/service/rest.py b/src/blueapi/service/rest.py deleted file mode 100644 index 905e0e8f3..000000000 --- a/src/blueapi/service/rest.py +++ /dev/null @@ -1,7 +0,0 @@ -from blueapi.service.routes import router -from fastapi import FastAPI - -app = FastAPI() - -# here, do app.include_router from all the other routes you want. -app.include_router(router) diff --git a/src/blueapi/service/routes.py b/src/blueapi/service/routes.py deleted file mode 100644 index 16e41aacc..000000000 --- a/src/blueapi/service/routes.py +++ /dev/null @@ -1,38 +0,0 @@ -from fastapi import APIRouter -from blueapi import context, worker - -router = APIRouter() - - -@router.get("/plans") -async def get_plans(): - context.plans - ... - - -@router.get("/plan/{name}") -async def get_plan_by_name(name: str): - try: - context.plans[name] - except IndexError: - raise Exception() # really, return a 404. - - -@router.get("/devices") -async def get_devices(): - context.devices - - -@router.get("/device/{name}") -async def get_device_by_name(name: str): - try: - context.plans[name] - except IndexError: - raise Exception() # really, return a 404. - - -@router.put("task/{name}") -async def execute_task(name: str): - ##basically in here, do the same thing the service once did... - # worker.submit_task(name, task) - pass diff --git a/tests/rest/test_rest_api.py b/tests/rest/test_rest_api.py new file mode 100644 index 000000000..27950dab9 --- /dev/null +++ b/tests/rest/test_rest_api.py @@ -0,0 +1,116 @@ +from ast import literal_eval +from dataclasses import dataclass + +from fastapi.testclient import TestClient +from pydantic import BaseModel + +from blueapi.core.bluesky_types import Plan +from blueapi.core.context import BlueskyContext +from blueapi.rest.handler import get_handler +from blueapi.rest.main import app +from blueapi.worker import RunEngineWorker +from blueapi.worker.task import ActiveTask + +# client = TestClient(app) + + +class MockHandler: + context: BlueskyContext + worker: RunEngineWorker + + def __init__(self) -> None: + self.context = BlueskyContext() + self.worker = RunEngineWorker(self.context) + + +class Client: + def __init__(self, handler: MockHandler) -> None: + """Create tester object""" + self.handler = handler + + @property + def client(self) -> TestClient: + app.dependency_overrides[get_handler] = lambda: self.handler + return TestClient(app) + + +def test_get_plans() -> None: + handler = MockHandler() + client = Client(handler).client + + class MyModel(BaseModel): + id: str + + plan = Plan(name="my-plan", model=MyModel) + + handler.context.plans = {"my-plan": plan} + response = client.get("/plans") + + assert response.status_code == 200 + assert literal_eval(response.content.decode())["plans"][0] == {"name": "my-plan"} + + +def test_get_plan_by_name() -> None: + handler = MockHandler() + client = Client(handler).client + + class MyModel(BaseModel): + id: str + + plan = Plan(name="my-plan", model=MyModel) + + handler.context.plans = {"my-plan": plan} + response = client.get("/plan/my-plan") + + assert response.status_code == 200 + assert literal_eval(response.content.decode()) == {"name": "my-plan"} + + +def test_get_devices() -> None: + handler = MockHandler() + client = Client(handler).client + + @dataclass + class MyDevice: + name: str + + device = MyDevice("my-device") + + handler.context.devices = {"my-device": device} + response = client.get("/devices") + + assert response.status_code == 200 + assert literal_eval(response.content.decode())["devices"][0] == { + "name": "my-device", + "protocols": ["HasName"], + } + + +def test_get_device_by_name() -> None: + handler = MockHandler() + client = Client(handler).client + + @dataclass + class MyDevice: + name: str + + device = MyDevice("my-device") + + handler.context.devices = {"my-device": device} + response = client.get("/device/my-device") + + assert response.status_code == 200 + assert literal_eval(response.content.decode()) == { + "name": "my-device", + "protocols": ["HasName"], + } + + +def test_put_plan_on_queue() -> None: + handler = MockHandler() + client = Client(handler).client + + client.put("/task/my-task", json={"name": "count", "params": {"detectors": ["x"]}}) + next_task: ActiveTask = handler.worker._task_queue.get(timeout=1.0) + + assert next_task From 3be91b25bf383581ed3ba2a592bc7300306b23d7 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 10:42:59 +0000 Subject: [PATCH 03/12] renamed rest -> service as files have changed location --- src/blueapi/service/openapi.py | 2 +- tests/rest/test_rest_api.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/blueapi/service/openapi.py b/src/blueapi/service/openapi.py index e537f1718..010f03815 100644 --- a/src/blueapi/service/openapi.py +++ b/src/blueapi/service/openapi.py @@ -4,7 +4,7 @@ from fastapi.openapi.utils import get_openapi -from blueapi.rest.main import app +from blueapi.service.main import app if __name__ == "__main__": with open("openapi.json", "w") as f: diff --git a/tests/rest/test_rest_api.py b/tests/rest/test_rest_api.py index 27950dab9..8607c7c39 100644 --- a/tests/rest/test_rest_api.py +++ b/tests/rest/test_rest_api.py @@ -6,8 +6,8 @@ from blueapi.core.bluesky_types import Plan from blueapi.core.context import BlueskyContext -from blueapi.rest.handler import get_handler -from blueapi.rest.main import app +from blueapi.service.handler import get_handler +from blueapi.service.main import app from blueapi.worker import RunEngineWorker from blueapi.worker.task import ActiveTask From 680ae4a351ab066edba50831ed5f403292f9e1a5 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 10:44:57 +0000 Subject: [PATCH 04/12] removed outdated test_rest_api.py file --- tests/rest/test_rest_api.py | 116 ------------------------------------ 1 file changed, 116 deletions(-) delete mode 100644 tests/rest/test_rest_api.py diff --git a/tests/rest/test_rest_api.py b/tests/rest/test_rest_api.py deleted file mode 100644 index 8607c7c39..000000000 --- a/tests/rest/test_rest_api.py +++ /dev/null @@ -1,116 +0,0 @@ -from ast import literal_eval -from dataclasses import dataclass - -from fastapi.testclient import TestClient -from pydantic import BaseModel - -from blueapi.core.bluesky_types import Plan -from blueapi.core.context import BlueskyContext -from blueapi.service.handler import get_handler -from blueapi.service.main import app -from blueapi.worker import RunEngineWorker -from blueapi.worker.task import ActiveTask - -# client = TestClient(app) - - -class MockHandler: - context: BlueskyContext - worker: RunEngineWorker - - def __init__(self) -> None: - self.context = BlueskyContext() - self.worker = RunEngineWorker(self.context) - - -class Client: - def __init__(self, handler: MockHandler) -> None: - """Create tester object""" - self.handler = handler - - @property - def client(self) -> TestClient: - app.dependency_overrides[get_handler] = lambda: self.handler - return TestClient(app) - - -def test_get_plans() -> None: - handler = MockHandler() - client = Client(handler).client - - class MyModel(BaseModel): - id: str - - plan = Plan(name="my-plan", model=MyModel) - - handler.context.plans = {"my-plan": plan} - response = client.get("/plans") - - assert response.status_code == 200 - assert literal_eval(response.content.decode())["plans"][0] == {"name": "my-plan"} - - -def test_get_plan_by_name() -> None: - handler = MockHandler() - client = Client(handler).client - - class MyModel(BaseModel): - id: str - - plan = Plan(name="my-plan", model=MyModel) - - handler.context.plans = {"my-plan": plan} - response = client.get("/plan/my-plan") - - assert response.status_code == 200 - assert literal_eval(response.content.decode()) == {"name": "my-plan"} - - -def test_get_devices() -> None: - handler = MockHandler() - client = Client(handler).client - - @dataclass - class MyDevice: - name: str - - device = MyDevice("my-device") - - handler.context.devices = {"my-device": device} - response = client.get("/devices") - - assert response.status_code == 200 - assert literal_eval(response.content.decode())["devices"][0] == { - "name": "my-device", - "protocols": ["HasName"], - } - - -def test_get_device_by_name() -> None: - handler = MockHandler() - client = Client(handler).client - - @dataclass - class MyDevice: - name: str - - device = MyDevice("my-device") - - handler.context.devices = {"my-device": device} - response = client.get("/device/my-device") - - assert response.status_code == 200 - assert literal_eval(response.content.decode()) == { - "name": "my-device", - "protocols": ["HasName"], - } - - -def test_put_plan_on_queue() -> None: - handler = MockHandler() - client = Client(handler).client - - client.put("/task/my-task", json={"name": "count", "params": {"detectors": ["x"]}}) - next_task: ActiveTask = handler.worker._task_queue.get(timeout=1.0) - - assert next_task From 3893419844c89f3d8c096628fa60a844ce60ba0c Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 10:58:01 +0000 Subject: [PATCH 05/12] modified catalog-info to include asyncapi, and made openapi.py output openapi.json into the same location as asyncapi.yaml --- catalog-info.yaml | 25 ++++++++-- .../user/reference/openapi.json | 49 ++++++++++--------- src/blueapi/service/openapi.py | 8 +-- 3 files changed, 52 insertions(+), 30 deletions(-) rename openapi.json => docs/user/reference/openapi.json (87%) diff --git a/catalog-info.yaml b/catalog-info.yaml index 230e7305e..44b9b86c0 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -4,25 +4,40 @@ metadata: name: bluesky-worker title: bluesky-worker description: Lightweight wrapper around bluesky services + annotations: + github.com/project-slug: DiamondLightSouce/blueapi spec: type: service lifecycle: production owner: user:vid18871 # TODO: owner: DAQ-Core # system: Athena # TODO: Define Athena system: presumably same location as DAQ-Core/DAQ? providesApis: - - blueapi + - message-topics + - restapi - blueskydocument-to-scanmessage - --- apiVersion: backstage.io/v1alpha1 kind: API metadata: - name: blueapi - title: blueapi + name: restapi + title: restapi description: REST API for getting plans/devices from the worker (and running tasks) spec: type: openapi lifecycle: production owner: user:vid18871 definition: - $text: ./openapi.json + $text: ./docs/user/reference/openapi.json +--- +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: message-topics + title: message-topics + description: Message topics which can be listened to over an activeMQ message bus +spec: + type: asyncapi + lifecycle: production + owner: user:vid18871 + definition: + $text: ./docs/user/reference/asyncapi.yaml diff --git a/openapi.json b/docs/user/reference/openapi.json similarity index 87% rename from openapi.json rename to docs/user/reference/openapi.json index 0d1ee40e9..e3059182b 100644 --- a/openapi.json +++ b/docs/user/reference/openapi.json @@ -8,6 +8,7 @@ "/plans": { "get": { "summary": "Get Plans", + "description": "Retrieve information about all available plans.", "operationId": "get_plans_plans_get", "responses": { "200": { @@ -26,6 +27,7 @@ "/plan/{name}": { "get": { "summary": "Get Plan By Name", + "description": "Retrieve information about a plan by its (unique) name.", "operationId": "get_plan_by_name_plan__name__get", "parameters": [ { @@ -65,6 +67,7 @@ "/devices": { "get": { "summary": "Get Devices", + "description": "Retrieve information about all available devices.", "operationId": "get_devices_devices_get", "responses": { "200": { @@ -83,6 +86,7 @@ "/device/{name}": { "get": { "summary": "Get Device By Name", + "description": "Retrieve information about a devices by its (unique) name.", "operationId": "get_device_by_name_device__name__get", "parameters": [ { @@ -121,8 +125,9 @@ }, "/task/{name}": { "put": { - "summary": "Execute Task", - "operationId": "execute_task_task__name__put", + "summary": "Submit Task", + "description": "Submit a task onto the worker queue.", + "operationId": "submit_task_task__name__put", "parameters": [ { "required": true, @@ -138,15 +143,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RunPlan" + "title": "Task", + "type": "object" }, "example": { - "name": "count", - "params": { - "detectors": [ - "x" - ] - } + "detectors": [ + "x" + ] } } }, @@ -157,7 +160,9 @@ "description": "Successful Response", "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/TaskResponse" + } } } }, @@ -199,6 +204,7 @@ "description": "Protocols that a device conforms to, indicating its capabilities" } }, + "additionalProperties": false, "description": "Representation of a device" }, "DeviceResponse": { @@ -217,6 +223,7 @@ "description": "Devices available to use in plans" } }, + "additionalProperties": false, "description": "Response to a query for devices" }, "HTTPValidationError": { @@ -245,6 +252,7 @@ "description": "Name of the plan" } }, + "additionalProperties": false, "description": "Representation of a plan" }, "PlanResponse": { @@ -263,27 +271,24 @@ "description": "Plans available to use by a worker" } }, + "additionalProperties": false, "description": "Response to a query for plans" }, - "RunPlan": { - "title": "RunPlan", + "TaskResponse": { + "title": "TaskResponse", "required": [ - "name" + "taskName" ], "type": "object", "properties": { - "name": { - "title": "Name", + "taskName": { + "title": "Taskname", "type": "string", - "description": "Name of plan to run" - }, - "params": { - "title": "Params", - "type": "object", - "description": "Values for parameters to plan, if any" + "description": "Unique identifier for the task" } }, - "description": "Task that will run a plan" + "additionalProperties": false, + "description": "Acknowledgement that a task has started, includes its ID" }, "ValidationError": { "title": "ValidationError", diff --git a/src/blueapi/service/openapi.py b/src/blueapi/service/openapi.py index 010f03815..644cba4bf 100644 --- a/src/blueapi/service/openapi.py +++ b/src/blueapi/service/openapi.py @@ -1,13 +1,17 @@ """Generate openapi.json.""" import json +from pathlib import Path from fastapi.openapi.utils import get_openapi from blueapi.service.main import app if __name__ == "__main__": - with open("openapi.json", "w") as f: + location = ( + Path(__file__).parents[3] / "docs" / "user" / "reference" / "openapi.json" + ) + with open(location, "w") as f: json.dump( get_openapi( title=app.title, @@ -19,5 +23,3 @@ f, indent=4, ) - - print("ah") From 100636001fb500fbb34c0b0ca17b98269886f757 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 10:59:05 +0000 Subject: [PATCH 06/12] removed catalog-info.yaml from pre commit checks to pass linting --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bdce7ee1..3667d7f3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: check-added-large-files - id: check-yaml - exclude: ^helm\/.*\/templates\/.* + exclude: ^helm\/.*\/templates\/.*|catalog-info.yaml - id: check-merge-conflict - repo: local From 8bee114a9e127084dda21e628a6547960b7012d2 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 11:05:38 +0000 Subject: [PATCH 07/12] added CI job to automatically run openapi.py script to generate openapi schema --- .github/workflows/openapi.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/openapi.yml diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml new file mode 100644 index 000000000..2030f2b99 --- /dev/null +++ b/.github/workflows/openapi.yml @@ -0,0 +1,30 @@ +name: Code CI + +on: + push: + pull_request: + schedule: + # Run weekly to check latest versions of dependencies + - cron: "0 8 * * WED" +env: + # The target python version, which must match the Dockerfile version + CONTAINER_PYTHON: "3.11" + +jobs: + publish: + # pull requests are a duplicate of a branch push if within the same repo. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install python packages + uses: ./.github/actions/install_requirements + with: + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] + + - name: generate openapi schema + run: python src/blueapi/service/openapi.py From 4f169433c93670d22528d74d6441b907f1184bd7 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 11:06:35 +0000 Subject: [PATCH 08/12] changed name of CI job for openapi --- .github/workflows/openapi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index 2030f2b99..20476441c 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -1,4 +1,4 @@ -name: Code CI +name: Openapi on: push: From ae4556aa098794b63fcf43aec3ecad3b535a02bd Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 13:17:38 +0000 Subject: [PATCH 09/12] added test for openapi.py --- src/blueapi/service/openapi.py | 15 ++++++++--- tests/service/test_openapi.py | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 tests/service/test_openapi.py diff --git a/src/blueapi/service/openapi.py b/src/blueapi/service/openapi.py index 644cba4bf..e279bc19d 100644 --- a/src/blueapi/service/openapi.py +++ b/src/blueapi/service/openapi.py @@ -7,10 +7,8 @@ from blueapi.service.main import app -if __name__ == "__main__": - location = ( - Path(__file__).parents[3] / "docs" / "user" / "reference" / "openapi.json" - ) + +def write_openapi_file(location: Path): with open(location, "w") as f: json.dump( get_openapi( @@ -23,3 +21,12 @@ f, indent=4, ) + + +def init(location: Path): + if __name__ == "__main__": + write_openapi_file(location) + + +location = Path(__file__).parents[3] / "docs" / "user" / "reference" / "openapi.json" +init(location) diff --git a/tests/service/test_openapi.py b/tests/service/test_openapi.py new file mode 100644 index 000000000..ae247be02 --- /dev/null +++ b/tests/service/test_openapi.py @@ -0,0 +1,48 @@ +# this should test if we change app, what openapi is generated. + +import json + +# i.e.checking that the openapi generation actually works. +from pathlib import Path + +import mock +from mock import Mock, PropertyMock + + +@mock.patch("blueapi.service.openapi.app") +def test_init(mock_app: Mock): + from blueapi.service.main import app + + title = PropertyMock(return_value="title") + version = PropertyMock(return_value=app.version) + openapi_version = PropertyMock(return_value=app.openapi_version) + description = PropertyMock(return_value="description") + routes = PropertyMock(return_value=[app.routes[0]]) + + type(mock_app).title = title + type(mock_app).version = version + type(mock_app).openapi_version = openapi_version + type(mock_app).description = description + type(mock_app).routes = routes + + from blueapi.service import openapi + + with mock.patch.object(openapi, "__name__", "__main__"): + location = Path(__file__).parent / "test_file.json" + openapi.init(location) + print("ah") + + with open(location, "r") as f: + result = json.load(f) + + assert result == { + "openapi": openapi_version(), + "info": { + "title": title(), + "description": description(), + "version": version(), + }, + "paths": {}, + } + + location.unlink() From f17ef5055ee37fa8496c9a535236259d5e86b0b5 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Thu, 11 May 2023 13:28:31 +0000 Subject: [PATCH 10/12] added openapi schema in docs --- docs/user/index.rst | 1 + docs/user/reference/openapi.rst | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 docs/user/reference/openapi.rst diff --git a/docs/user/index.rst b/docs/user/index.rst index e3ba1aa4d..a36bc9f42 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -52,6 +52,7 @@ side-bar. reference/api reference/asyncapi + reference/openapi ../genindex +++ diff --git a/docs/user/reference/openapi.rst b/docs/user/reference/openapi.rst new file mode 100644 index 000000000..426d251b1 --- /dev/null +++ b/docs/user/reference/openapi.rst @@ -0,0 +1,5 @@ +OpenAPI Specification +====================== + +.. literalinclude:: ./openapi.json + :language: json From 64723115d3ad1771030917c4876b34330f94b2d6 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Fri, 12 May 2023 08:44:46 +0000 Subject: [PATCH 11/12] using sphinxcontrib.openapi to autogenerate open api documentation --- docs/conf.py | 2 ++ docs/user/reference/openapi.rst | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7b1a737e8..c4b9364d2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,8 @@ "sphinx_copybutton", # For the card element "sphinx_design", + # OpenAPI directive + "sphinxcontrib.openapi", ] # If true, Sphinx will warn about all references where the target cannot diff --git a/docs/user/reference/openapi.rst b/docs/user/reference/openapi.rst index 426d251b1..b059ed2c5 100644 --- a/docs/user/reference/openapi.rst +++ b/docs/user/reference/openapi.rst @@ -1,5 +1,6 @@ -OpenAPI Specification -====================== +REST API +======== -.. literalinclude:: ./openapi.json - :language: json +The endpoints of the REST service are documented below. + +.. openapi:: ./openapi.json From 9c035fbeaac1217040a504371071140b141589fb Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Fri, 12 May 2023 09:14:05 +0000 Subject: [PATCH 12/12] added dependency to pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3f00faea0..621f6e24d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dev = [ "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", + "sphinxcontrib-openapi", "tox-direct", "types-mock", "types-PyYAML",