diff --git a/api/src/opentrons/config/__init__.py b/api/src/opentrons/config/__init__.py index a4571521211..71ba78d39b0 100644 --- a/api/src/opentrons/config/__init__.py +++ b/api/src/opentrons/config/__init__.py @@ -202,6 +202,15 @@ class ConfigElement(NamedTuple): " absolute path, it will be used directly. If it is a " "relative path it will be relative to log_dir", ), + ConfigElement( + "sensor_log_file", + "Sensor Log File", + Path("logs") / "sensor.log", + ConfigElementType.FILE, + "The location of the file to save sensor logs to. If this is an" + " absolute path, it will be used directly. If it is a " + "relative path it will be relative to log_dir", + ), ConfigElement( "serial_log_file", "Serial Log File", diff --git a/api/src/opentrons/config/defaults_ot3.py b/api/src/opentrons/config/defaults_ot3.py index 08b86f16c95..55565745d3a 100644 --- a/api/src/opentrons/config/defaults_ot3.py +++ b/api/src/opentrons/config/defaults_ot3.py @@ -15,7 +15,6 @@ LiquidProbeSettings, ZSenseSettings, EdgeSenseSettings, - OutputOptions, ) @@ -27,13 +26,11 @@ plunger_speed=15, plunger_impulse_time=0.2, sensor_threshold_pascals=15, - output_option=OutputOptions.sync_buffer_to_csv, aspirate_while_sensing=False, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files={InstrumentProbeType.PRIMARY: "/data/pressure_sensor_data.csv"}, ) DEFAULT_CALIBRATION_SETTINGS: Final[OT3CalibrationSettings] = OT3CalibrationSettings( @@ -43,7 +40,6 @@ max_overrun_distance_mm=5.0, speed_mm_per_s=1.0, sensor_threshold_pf=3.0, - output_option=OutputOptions.sync_only, ), ), edge_sense=EdgeSenseSettings( @@ -54,7 +50,6 @@ max_overrun_distance_mm=0.5, speed_mm_per_s=1, sensor_threshold_pf=3.0, - output_option=OutputOptions.sync_only, ), search_initial_tolerance_mm=12.0, search_iteration_limit=8, @@ -195,23 +190,6 @@ ) -def _build_output_option_with_default( - from_conf: Any, default: OutputOptions -) -> OutputOptions: - if from_conf is None: - return default - else: - if isinstance(from_conf, OutputOptions): - return from_conf - else: - try: - enumval = OutputOptions[from_conf] - except KeyError: # not an enum entry - return default - else: - return enumval - - def _build_log_files_with_default( from_conf: Any, default: Optional[Dict[InstrumentProbeType, str]], @@ -316,24 +294,12 @@ def _build_default_cap_pass( sensor_threshold_pf=from_conf.get( "sensor_threshold_pf", default.sensor_threshold_pf ), - output_option=from_conf.get("output_option", default.output_option), ) def _build_default_liquid_probe( from_conf: Any, default: LiquidProbeSettings ) -> LiquidProbeSettings: - output_option = _build_output_option_with_default( - from_conf.get("output_option", None), default.output_option - ) - data_files: Optional[Dict[InstrumentProbeType, str]] = None - if ( - output_option is OutputOptions.sync_buffer_to_csv - or output_option is OutputOptions.stream_to_csv - ): - data_files = _build_log_files_with_default( - from_conf.get("data_files", None), default.data_files - ) return LiquidProbeSettings( mount_speed=from_conf.get("mount_speed", default.mount_speed), plunger_speed=from_conf.get("plunger_speed", default.plunger_speed), @@ -343,7 +309,6 @@ def _build_default_liquid_probe( sensor_threshold_pascals=from_conf.get( "sensor_threshold_pascals", default.sensor_threshold_pascals ), - output_option=from_conf.get("output_option", default.output_option), aspirate_while_sensing=from_conf.get( "aspirate_while_sensing", default.aspirate_while_sensing ), @@ -357,7 +322,6 @@ def _build_default_liquid_probe( "samples_for_baselining", default.samples_for_baselining ), sample_time_sec=from_conf.get("sample_time_sec", default.sample_time_sec), - data_files=data_files, ) diff --git a/api/src/opentrons/config/types.py b/api/src/opentrons/config/types.py index 5a6c67725d0..d35b58578ca 100644 --- a/api/src/opentrons/config/types.py +++ b/api/src/opentrons/config/types.py @@ -1,8 +1,8 @@ from enum import Enum from dataclasses import dataclass, asdict, fields -from typing import Dict, Tuple, TypeVar, Generic, List, cast, Optional +from typing import Dict, Tuple, TypeVar, Generic, List, cast from typing_extensions import TypedDict, Literal -from opentrons.hardware_control.types import OT3AxisKind, InstrumentProbeType +from opentrons.hardware_control.types import OT3AxisKind class AxisDict(TypedDict): @@ -103,25 +103,12 @@ def by_gantry_load( ) -class OutputOptions(int, Enum): - """Specifies where we should report sensor data to during a sensor pass.""" - - stream_to_csv = 0x1 # compile sensor data stream into a csv file, in addition to can_bus_only behavior - sync_buffer_to_csv = 0x2 # collect sensor data on pipette mcu, then stream to robot server and compile into a csv file, in addition to can_bus_only behavior - can_bus_only = ( - 0x4 # stream sensor data over CAN bus, in addition to sync_only behavior - ) - sync_only = 0x8 # trigger pipette sync line upon sensor's detection of something - - @dataclass(frozen=True) class CapacitivePassSettings: prep_distance_mm: float max_overrun_distance_mm: float speed_mm_per_s: float sensor_threshold_pf: float - output_option: OutputOptions - data_files: Optional[Dict[InstrumentProbeType, str]] = None @dataclass(frozen=True) @@ -135,13 +122,11 @@ class LiquidProbeSettings: plunger_speed: float plunger_impulse_time: float sensor_threshold_pascals: float - output_option: OutputOptions aspirate_while_sensing: bool z_overlap_between_passes_mm: float plunger_reset_offset: float samples_for_baselining: int sample_time_sec: float - data_files: Optional[Dict[InstrumentProbeType, str]] @dataclass(frozen=True) diff --git a/api/src/opentrons/hardware_control/backends/flex_protocol.py b/api/src/opentrons/hardware_control/backends/flex_protocol.py index 6f3299cf92d..466e7890026 100644 --- a/api/src/opentrons/hardware_control/backends/flex_protocol.py +++ b/api/src/opentrons/hardware_control/backends/flex_protocol.py @@ -15,7 +15,7 @@ from opentrons_shared_data.pipette.types import ( PipetteName, ) -from opentrons.config.types import GantryLoad, OutputOptions +from opentrons.config.types import GantryLoad from opentrons.hardware_control.types import ( BoardRevision, Axis, @@ -38,6 +38,8 @@ StatusBarState, ) from opentrons.hardware_control.module_control import AttachedModulesControl +from opentrons_hardware.firmware_bindings.constants import SensorId +from opentrons_hardware.sensors.types import SensorDataType from ..dev_types import OT3AttachedInstruments from .types import HWStopCondition @@ -152,10 +154,11 @@ async def liquid_probe( threshold_pascals: float, plunger_impulse_time: float, num_baseline_reads: int, - output_format: OutputOptions = OutputOptions.can_bus_only, - data_files: Optional[Dict[InstrumentProbeType, str]] = None, probe: InstrumentProbeType = InstrumentProbeType.PRIMARY, force_both_sensors: bool = False, + response_queue: Optional[ + asyncio.Queue[Dict[SensorId, List[SensorDataType]]] + ] = None, ) -> float: ... @@ -371,8 +374,6 @@ async def capacitive_probe( speed_mm_per_s: float, sensor_threshold_pf: float, probe: InstrumentProbeType = InstrumentProbeType.PRIMARY, - output_format: OutputOptions = OutputOptions.sync_only, - data_files: Optional[Dict[InstrumentProbeType, str]] = None, ) -> bool: ... diff --git a/api/src/opentrons/hardware_control/backends/ot3controller.py b/api/src/opentrons/hardware_control/backends/ot3controller.py index 84c95c8fbc4..48787e86933 100644 --- a/api/src/opentrons/hardware_control/backends/ot3controller.py +++ b/api/src/opentrons/hardware_control/backends/ot3controller.py @@ -25,7 +25,7 @@ Union, Mapping, ) -from opentrons.config.types import OT3Config, GantryLoad, OutputOptions +from opentrons.config.types import OT3Config, GantryLoad from opentrons.config import gripper_config from .ot3utils import ( axis_convert, @@ -102,7 +102,9 @@ NodeId, PipetteName as FirmwarePipetteName, ErrorCode, + SensorId, ) +from opentrons_hardware.sensors.types import SensorDataType from opentrons_hardware.firmware_bindings.messages.message_definitions import ( StopRequest, ) @@ -1368,28 +1370,14 @@ async def liquid_probe( threshold_pascals: float, plunger_impulse_time: float, num_baseline_reads: int, - output_option: OutputOptions = OutputOptions.can_bus_only, - data_files: Optional[Dict[InstrumentProbeType, str]] = None, probe: InstrumentProbeType = InstrumentProbeType.PRIMARY, force_both_sensors: bool = False, + response_queue: Optional[ + asyncio.Queue[Dict[SensorId, List[SensorDataType]]] + ] = None, ) -> float: head_node = axis_to_node(Axis.by_mount(mount)) tool = sensor_node_for_pipette(OT3Mount(mount.value)) - csv_output = bool(output_option.value & OutputOptions.stream_to_csv.value) - sync_buffer_output = bool( - output_option.value & OutputOptions.sync_buffer_to_csv.value - ) - can_bus_only_output = bool( - output_option.value & OutputOptions.can_bus_only.value - ) - data_files_transposed = ( - None - if data_files is None - else { - sensor_id_for_instrument(probe): data_files[probe] - for probe in data_files.keys() - } - ) positions = await liquid_probe( messenger=self._messenger, tool=tool, @@ -1400,12 +1388,9 @@ async def liquid_probe( threshold_pascals=threshold_pascals, plunger_impulse_time=plunger_impulse_time, num_baseline_reads=num_baseline_reads, - csv_output=csv_output, - sync_buffer_output=sync_buffer_output, - can_bus_only_output=can_bus_only_output, - data_files=data_files_transposed, sensor_id=sensor_id_for_instrument(probe), force_both_sensors=force_both_sensors, + response_queue=response_queue, ) for node, point in positions.items(): self._position.update({node: point.motor_position}) @@ -1432,41 +1417,13 @@ async def capacitive_probe( speed_mm_per_s: float, sensor_threshold_pf: float, probe: InstrumentProbeType = InstrumentProbeType.PRIMARY, - output_option: OutputOptions = OutputOptions.sync_only, - data_files: Optional[Dict[InstrumentProbeType, str]] = None, ) -> bool: - if output_option == OutputOptions.sync_buffer_to_csv: - assert ( - self._subsystem_manager.device_info[ - SubSystem.of_mount(mount) - ].revision.tertiary - == "1" - ) - csv_output = bool(output_option.value & OutputOptions.stream_to_csv.value) - sync_buffer_output = bool( - output_option.value & OutputOptions.sync_buffer_to_csv.value - ) - can_bus_only_output = bool( - output_option.value & OutputOptions.can_bus_only.value - ) - data_files_transposed = ( - None - if data_files is None - else { - sensor_id_for_instrument(probe): data_files[probe] - for probe in data_files.keys() - } - ) status = await capacitive_probe( messenger=self._messenger, tool=sensor_node_for_mount(mount), mover=axis_to_node(moving), distance=distance_mm, mount_speed=speed_mm_per_s, - csv_output=csv_output, - sync_buffer_output=sync_buffer_output, - can_bus_only_output=can_bus_only_output, - data_files=data_files_transposed, sensor_id=sensor_id_for_instrument(probe), relative_threshold_pf=sensor_threshold_pf, ) diff --git a/api/src/opentrons/hardware_control/backends/ot3simulator.py b/api/src/opentrons/hardware_control/backends/ot3simulator.py index 034531892d8..017c90c45b3 100644 --- a/api/src/opentrons/hardware_control/backends/ot3simulator.py +++ b/api/src/opentrons/hardware_control/backends/ot3simulator.py @@ -17,7 +17,7 @@ Mapping, ) -from opentrons.config.types import OT3Config, GantryLoad, OutputOptions +from opentrons.config.types import OT3Config, GantryLoad from opentrons.config import gripper_config from opentrons.hardware_control.module_control import AttachedModulesControl @@ -63,7 +63,8 @@ from opentrons.util.async_helpers import ensure_yield from .types import HWStopCondition from .flex_protocol import FlexBackend - +from opentrons_hardware.firmware_bindings.constants import SensorId +from opentrons_hardware.sensors.types import SensorDataType log = logging.getLogger(__name__) @@ -347,10 +348,11 @@ async def liquid_probe( threshold_pascals: float, plunger_impulse_time: float, num_baseline_reads: int, - output_format: OutputOptions = OutputOptions.can_bus_only, - data_files: Optional[Dict[InstrumentProbeType, str]] = None, probe: InstrumentProbeType = InstrumentProbeType.PRIMARY, force_both_sensors: bool = False, + response_queue: Optional[ + asyncio.Queue[Dict[SensorId, List[SensorDataType]]] + ] = None, ) -> float: z_axis = Axis.by_mount(mount) pos = self._position @@ -750,8 +752,6 @@ async def capacitive_probe( speed_mm_per_s: float, sensor_threshold_pf: float, probe: InstrumentProbeType = InstrumentProbeType.PRIMARY, - output_format: OutputOptions = OutputOptions.sync_only, - data_files: Optional[Dict[InstrumentProbeType, str]] = None, ) -> bool: self._position[moving] += distance_mm return True diff --git a/api/src/opentrons/hardware_control/ot3api.py b/api/src/opentrons/hardware_control/ot3api.py index cdd69fc2f90..ebd56d106d1 100644 --- a/api/src/opentrons/hardware_control/ot3api.py +++ b/api/src/opentrons/hardware_control/ot3api.py @@ -143,7 +143,8 @@ from .backends.flex_protocol import FlexBackend from .backends.ot3simulator import OT3Simulator from .backends.errors import SubsystemUpdating - +from opentrons_hardware.firmware_bindings.constants import SensorId +from opentrons_hardware.sensors.types import SensorDataType mod_log = logging.getLogger(__name__) @@ -2634,6 +2635,9 @@ async def _liquid_probe_pass( probe: InstrumentProbeType, p_travel: float, force_both_sensors: bool = False, + response_queue: Optional[ + asyncio.Queue[Dict[SensorId, List[SensorDataType]]] + ] = None, ) -> float: plunger_direction = -1 if probe_settings.aspirate_while_sensing else 1 end_z = await self._backend.liquid_probe( @@ -2644,10 +2648,9 @@ async def _liquid_probe_pass( probe_settings.sensor_threshold_pascals, probe_settings.plunger_impulse_time, probe_settings.samples_for_baselining, - probe_settings.output_option, - probe_settings.data_files, probe=probe, force_both_sensors=force_both_sensors, + response_queue=response_queue, ) machine_pos = await self._backend.update_position() machine_pos[Axis.by_mount(mount)] = end_z @@ -2668,6 +2671,9 @@ async def liquid_probe( # noqa: C901 probe_settings: Optional[LiquidProbeSettings] = None, probe: Optional[InstrumentProbeType] = None, force_both_sensors: bool = False, + response_queue: Optional[ + asyncio.Queue[Dict[SensorId, List[SensorDataType]]] + ] = None, ) -> float: """Search for and return liquid level height. @@ -2793,6 +2799,8 @@ async def prep_plunger_for_probe_move( probe_settings, checked_probe, plunger_travel_mm + sensor_baseline_plunger_move_mm, + force_both_sensors, + response_queue, ) # if we made it here without an error we found the liquid error = None @@ -2861,8 +2869,6 @@ async def capacitive_probe( pass_settings.speed_mm_per_s, pass_settings.sensor_threshold_pf, probe, - pass_settings.output_option, - pass_settings.data_files, ) end_pos = await self.gantry_position(mount, refresh=True) if retract_after: diff --git a/api/src/opentrons/util/logging_config.py b/api/src/opentrons/util/logging_config.py index e9a4d2042a2..944f4d3d5ed 100644 --- a/api/src/opentrons/util/logging_config.py +++ b/api/src/opentrons/util/logging_config.py @@ -5,10 +5,13 @@ from opentrons.config import CONFIG, ARCHITECTURE, SystemArchitecture +from opentrons_hardware.sensors import SENSOR_LOG_NAME + def _host_config(level_value: int) -> Dict[str, Any]: serial_log_filename = CONFIG["serial_log_file"] api_log_filename = CONFIG["api_log_file"] + sensor_log_filename = CONFIG["sensor_log_file"] return { "version": 1, "disable_existing_loggers": False, @@ -41,6 +44,14 @@ def _host_config(level_value: int) -> Dict[str, Any]: "level": logging.DEBUG, "backupCount": 5, }, + "sensor": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "basic", + "filename": sensor_log_filename, + "maxBytes": 1000000, + "level": logging.DEBUG, + "backupCount": 5, + }, }, "loggers": { "opentrons": { @@ -66,6 +77,11 @@ def _host_config(level_value: int) -> Dict[str, Any]: "level": logging.DEBUG, "propagate": False, }, + SENSOR_LOG_NAME: { + "handlers": ["sensor"], + "level": logging.DEBUG, + "propagate": False, + }, "__main__": {"handlers": ["api"], "level": level_value}, }, } @@ -75,6 +91,7 @@ def _buildroot_config(level_value: int) -> Dict[str, Any]: # Import systemd.journald here since it is generally unavailble on non # linux systems and we probably don't want to use it on linux desktops # either + sensor_log_filename = CONFIG["sensor_log_file"] return { "version": 1, "disable_existing_loggers": False, @@ -106,6 +123,14 @@ def _buildroot_config(level_value: int) -> Dict[str, Any]: "formatter": "message_only", "SYSLOG_IDENTIFIER": "opentrons-api-serial-usbbin", }, + "sensor": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "basic", + "filename": sensor_log_filename, + "maxBytes": 1000000, + "level": logging.DEBUG, + "backupCount": 3, + }, }, "loggers": { "opentrons.drivers.asyncio.communication.serial_connection": { @@ -131,6 +156,11 @@ def _buildroot_config(level_value: int) -> Dict[str, Any]: "level": logging.DEBUG, "propagate": False, }, + SENSOR_LOG_NAME: { + "handlers": ["sensor"], + "level": logging.DEBUG, + "propagate": False, + }, "__main__": {"handlers": ["api"], "level": level_value}, }, } diff --git a/api/tests/opentrons/config/ot3_settings.py b/api/tests/opentrons/config/ot3_settings.py index 38353c05a3c..04370fd6c09 100644 --- a/api/tests/opentrons/config/ot3_settings.py +++ b/api/tests/opentrons/config/ot3_settings.py @@ -1,5 +1,3 @@ -from opentrons.config.types import OutputOptions - ot3_dummy_settings = { "name": "Marie Curie", "model": "OT-3 Standard", @@ -122,13 +120,11 @@ "plunger_speed": 10, "plunger_impulse_time": 0.2, "sensor_threshold_pascals": 17, - "output_option": OutputOptions.stream_to_csv, "aspirate_while_sensing": False, "z_overlap_between_passes_mm": 0.1, "plunger_reset_offset": 2.0, "samples_for_baselining": 20, "sample_time_sec": 0.004, - "data_files": {"PRIMARY": "/data/pressure_sensor_data.csv"}, }, "calibration": { "z_offset": { @@ -137,8 +133,6 @@ "max_overrun_distance_mm": 2, "speed_mm_per_s": 3, "sensor_threshold_pf": 4, - "output_option": OutputOptions.sync_only, - "data_files": None, }, }, "edge_sense": { @@ -149,8 +143,6 @@ "max_overrun_distance_mm": 5, "speed_mm_per_s": 6, "sensor_threshold_pf": 7, - "output_option": OutputOptions.sync_only, - "data_files": None, }, "search_initial_tolerance_mm": 18, "search_iteration_limit": 3, diff --git a/api/tests/opentrons/hardware_control/backends/test_ot3_controller.py b/api/tests/opentrons/hardware_control/backends/test_ot3_controller.py index ac25d19a3e2..5ffee581de4 100644 --- a/api/tests/opentrons/hardware_control/backends/test_ot3_controller.py +++ b/api/tests/opentrons/hardware_control/backends/test_ot3_controller.py @@ -39,7 +39,6 @@ OT3Config, GantryLoad, LiquidProbeSettings, - OutputOptions, ) from opentrons.config.robot_configs import build_config_ot3 from opentrons_hardware.firmware_bindings.arbitration_id import ArbitrationId @@ -61,7 +60,6 @@ UpdateState, EstopState, CurrentConfig, - InstrumentProbeType, ) from opentrons.hardware_control.errors import ( InvalidPipetteName, @@ -180,13 +178,11 @@ def fake_liquid_settings() -> LiquidProbeSettings: plunger_speed=10, plunger_impulse_time=0.2, sensor_threshold_pascals=15, - output_option=OutputOptions.can_bus_only, aspirate_while_sensing=False, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files={InstrumentProbeType.PRIMARY: "fake_file_name"}, ) @@ -707,6 +703,17 @@ async def test_ready_for_movement( assert controller.check_motor_status(axes) == ready +def probe_move_group_run_side_effect( + head: NodeId, tool: NodeId +) -> Iterator[Dict[NodeId, MotorPositionStatus]]: + """Return homed position for axis that is present and was commanded to home.""" + positions = { + head: MotorPositionStatus(0.0, 0.0, True, True, MoveCompleteAck(1)), + tool: MotorPositionStatus(0.0, 0.0, True, True, MoveCompleteAck(1)), + } + yield positions + + @pytest.mark.parametrize("mount", [OT3Mount.LEFT, OT3Mount.RIGHT]) async def test_liquid_probe( mount: OT3Mount, @@ -716,6 +723,11 @@ async def test_liquid_probe( mock_send_stop_threshold: mock.AsyncMock, ) -> None: fake_max_p_dist = 70 + head_node = axis_to_node(Axis.by_mount(mount)) + tool_node = sensor_node_for_mount(mount) + mock_move_group_run.side_effect = probe_move_group_run_side_effect( + head_node, tool_node + ) try: await controller.liquid_probe( mount=mount, @@ -725,18 +737,17 @@ async def test_liquid_probe( threshold_pascals=fake_liquid_settings.sensor_threshold_pascals, plunger_impulse_time=fake_liquid_settings.plunger_impulse_time, num_baseline_reads=fake_liquid_settings.samples_for_baselining, - output_option=fake_liquid_settings.output_option, ) except PipetteLiquidNotFoundError: # the move raises a liquid not found now since we don't call the move group and it doesn't # get any positions back pass move_groups = mock_move_group_run.call_args_list[0][0][0]._move_groups - head_node = axis_to_node(Axis.by_mount(mount)) - tool_node = sensor_node_for_mount(mount) # in tool_sensors, pipette moves down, then sensor move goes assert move_groups[0][0][tool_node].stop_condition == MoveStopCondition.none - assert move_groups[1][0][tool_node].stop_condition == MoveStopCondition.sync_line + assert ( + move_groups[1][0][tool_node].stop_condition == MoveStopCondition.sensor_report + ) assert len(move_groups) == 2 assert move_groups[0][0][tool_node] assert move_groups[1][0][head_node], move_groups[2][0][tool_node] diff --git a/api/tests/opentrons/hardware_control/test_ot3_api.py b/api/tests/opentrons/hardware_control/test_ot3_api.py index 08ecb3afa43..783f438d914 100644 --- a/api/tests/opentrons/hardware_control/test_ot3_api.py +++ b/api/tests/opentrons/hardware_control/test_ot3_api.py @@ -1,5 +1,6 @@ """ Tests for behaviors specific to the OT3 hardware controller. """ +import asyncio from typing import ( AsyncIterator, Iterator, @@ -26,7 +27,6 @@ GantryLoad, CapacitivePassSettings, LiquidProbeSettings, - OutputOptions, ) from opentrons.hardware_control.dev_types import ( AttachedGripper, @@ -98,6 +98,8 @@ from opentrons.hardware_control.module_control import AttachedModulesControl from opentrons.hardware_control.backends.types import HWStopCondition +from opentrons_hardware.firmware_bindings.constants import SensorId +from opentrons_hardware.sensors.types import SensorDataType # TODO (spp, 2023-08-22): write tests for ot3api.stop & ot3api.halt @@ -109,7 +111,6 @@ def fake_settings() -> CapacitivePassSettings: max_overrun_distance_mm=2, speed_mm_per_s=4, sensor_threshold_pf=1.0, - output_option=OutputOptions.sync_only, ) @@ -120,13 +121,11 @@ def fake_liquid_settings() -> LiquidProbeSettings: plunger_speed=15, plunger_impulse_time=0.2, sensor_threshold_pascals=15, - output_option=OutputOptions.can_bus_only, aspirate_while_sensing=False, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files={InstrumentProbeType.PRIMARY: "fake_file_name"}, ) @@ -488,8 +487,6 @@ def _update_position( speed_mm_per_s: float, threshold_pf: float, probe: InstrumentProbeType, - output_option: OutputOptions = OutputOptions.sync_only, - data_file: Optional[str] = None, ) -> None: hardware_backend._position[moving] += distance_mm / 2 @@ -827,13 +824,11 @@ async def test_liquid_probe( plunger_speed=15, plunger_impulse_time=0.2, sensor_threshold_pascals=15, - output_option=OutputOptions.can_bus_only, aspirate_while_sensing=True, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files={InstrumentProbeType.PRIMARY: "fake_file_name"}, ) fake_max_z_dist = 10.0 non_responsive_z_mm = ot3_hardware.liquid_probe_non_responsive_z_distance( @@ -860,10 +855,9 @@ async def test_liquid_probe( fake_settings_aspirate.sensor_threshold_pascals, fake_settings_aspirate.plunger_impulse_time, fake_settings_aspirate.samples_for_baselining, - fake_settings_aspirate.output_option, - fake_settings_aspirate.data_files, probe=InstrumentProbeType.PRIMARY, force_both_sensors=False, + response_queue=None, ) await ot3_hardware.liquid_probe( @@ -1098,13 +1092,11 @@ async def test_multi_liquid_probe( plunger_speed=71.5, plunger_impulse_time=0.2, sensor_threshold_pascals=15, - output_option=OutputOptions.can_bus_only, aspirate_while_sensing=True, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files={InstrumentProbeType.PRIMARY: "fake_file_name"}, ) fake_max_z_dist = 10.0 await ot3_hardware.liquid_probe( @@ -1119,10 +1111,9 @@ async def test_multi_liquid_probe( fake_settings_aspirate.sensor_threshold_pascals, fake_settings_aspirate.plunger_impulse_time, fake_settings_aspirate.samples_for_baselining, - fake_settings_aspirate.output_option, - fake_settings_aspirate.data_files, probe=InstrumentProbeType.PRIMARY, force_both_sensors=False, + response_queue=None, ) assert mock_liquid_probe.call_count == 3 @@ -1155,10 +1146,11 @@ async def _fake_pos_update_and_raise( threshold_pascals: float, plunger_impulse_time: float, num_baseline_reads: int, - output_format: OutputOptions = OutputOptions.can_bus_only, - data_files: Optional[Dict[InstrumentProbeType, str]] = None, probe: InstrumentProbeType = InstrumentProbeType.PRIMARY, force_both_sensors: bool = False, + response_queue: Optional[ + asyncio.Queue[Dict[SensorId, List[SensorDataType]]] + ] = None, ) -> float: pos = self._position pos[Axis.by_mount(mount)] += mount_speed * ( @@ -1176,13 +1168,11 @@ async def _fake_pos_update_and_raise( plunger_speed=71.5, plunger_impulse_time=0.2, sensor_threshold_pascals=15, - output_option=OutputOptions.can_bus_only, aspirate_while_sensing=True, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files={InstrumentProbeType.PRIMARY: "fake_file_name"}, ) # with a mount speed of 5, pass overlap of 0.5 and a 0.2s delay on z # the actual distance traveled is 3.5mm per pass @@ -1233,8 +1223,6 @@ async def test_capacitive_probe( 4, 1.0, InstrumentProbeType.PRIMARY, - fake_settings.output_option, - fake_settings.data_files, ) original = moving.set_in_point(here, 0) diff --git a/hardware-testing/hardware_testing/examples/capacitive_probe_ot3.py b/hardware-testing/hardware_testing/examples/capacitive_probe_ot3.py index 241309de41a..9f775246cb6 100644 --- a/hardware-testing/hardware_testing/examples/capacitive_probe_ot3.py +++ b/hardware-testing/hardware_testing/examples/capacitive_probe_ot3.py @@ -2,7 +2,7 @@ import argparse import asyncio -from opentrons.config.types import CapacitivePassSettings, OutputOptions +from opentrons.config.types import CapacitivePassSettings from opentrons.hardware_control.ot3api import OT3API from hardware_testing.opentrons_api import types @@ -44,14 +44,12 @@ max_overrun_distance_mm=3, speed_mm_per_s=1, sensor_threshold_pf=STABLE_CAP_PF, - output_option=OutputOptions.sync_only, ) PROBE_SETTINGS_XY_AXIS = CapacitivePassSettings( prep_distance_mm=CUTOUT_SIZE / 2, max_overrun_distance_mm=3, speed_mm_per_s=1, sensor_threshold_pf=STABLE_CAP_PF, - output_option=OutputOptions.sync_only, ) diff --git a/hardware-testing/hardware_testing/examples/capacitive_probe_ot3_tunable.py b/hardware-testing/hardware_testing/examples/capacitive_probe_ot3_tunable.py index 2b4496e0eb2..672b0909cb2 100644 --- a/hardware-testing/hardware_testing/examples/capacitive_probe_ot3_tunable.py +++ b/hardware-testing/hardware_testing/examples/capacitive_probe_ot3_tunable.py @@ -2,9 +2,8 @@ import argparse import asyncio -from opentrons.config.types import CapacitivePassSettings, OutputOptions +from opentrons.config.types import CapacitivePassSettings from opentrons.hardware_control.ot3api import OT3API -from opentrons.hardware_control.types import InstrumentProbeType from hardware_testing.opentrons_api import types from hardware_testing.opentrons_api import helpers_ot3 @@ -46,15 +45,12 @@ max_overrun_distance_mm=3, speed_mm_per_s=1, sensor_threshold_pf=CAP_REL_THRESHOLD_PF, - output_option=OutputOptions.sync_only, ) PROBE_SETTINGS_Z_AXIS_OUTPUT = CapacitivePassSettings( prep_distance_mm=10, max_overrun_distance_mm=3, speed_mm_per_s=1, sensor_threshold_pf=CAP_REL_THRESHOLD_PF, - output_option=OutputOptions.sync_buffer_to_csv, - data_files={InstrumentProbeType.PRIMARY: "/data/capacitive_sensor_data.csv"}, ) diff --git a/hardware-testing/hardware_testing/gravimetric/config.py b/hardware-testing/hardware_testing/gravimetric/config.py index b783908d5e6..304087748d1 100644 --- a/hardware-testing/hardware_testing/gravimetric/config.py +++ b/hardware-testing/hardware_testing/gravimetric/config.py @@ -3,9 +3,8 @@ from typing import List, Dict, Tuple from typing_extensions import Final from enum import Enum -from opentrons.config.types import LiquidProbeSettings, OutputOptions +from opentrons.config.types import LiquidProbeSettings from opentrons.protocol_api.labware import Well -from opentrons.hardware_control.types import InstrumentProbeType class ConfigType(Enum): @@ -170,13 +169,11 @@ def _get_liquid_probe_settings( plunger_speed=lqid_cfg["plunger_speed"], plunger_impulse_time=0.2, sensor_threshold_pascals=lqid_cfg["sensor_threshold_pascals"], - output_option=OutputOptions.sync_only, aspirate_while_sensing=False, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files={InstrumentProbeType.PRIMARY: "/data/testing_data/pressure.csv"}, ) diff --git a/hardware-testing/hardware_testing/liquid_sense/execute.py b/hardware-testing/hardware_testing/liquid_sense/execute.py index 01cb0d27375..001abdaa82f 100644 --- a/hardware-testing/hardware_testing/liquid_sense/execute.py +++ b/hardware-testing/hardware_testing/liquid_sense/execute.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Dict, Any, List, Tuple, Optional from .report import store_tip_results, store_trial, store_baseline_trial -from opentrons.config.types import LiquidProbeSettings, OutputOptions +from opentrons.config.types import LiquidProbeSettings from .__main__ import RunArgs from hardware_testing.gravimetric.workarounds import get_sync_hw_api from hardware_testing.gravimetric.helpers import ( @@ -445,13 +445,11 @@ def _run_trial( plunger_speed=plunger_speed, plunger_impulse_time=0.2, sensor_threshold_pascals=lqid_cfg["sensor_threshold_pascals"], - output_option=OutputOptions.sync_buffer_to_csv, aspirate_while_sensing=run_args.aspirate, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files=data_files, ) hw_mount = OT3Mount.LEFT if run_args.pipette.mount == "left" else OT3Mount.RIGHT diff --git a/hardware-testing/hardware_testing/production_qc/pipette_assembly_qc_ot3/__main__.py b/hardware-testing/hardware_testing/production_qc/pipette_assembly_qc_ot3/__main__.py index 28c86520d15..a2c1254f098 100644 --- a/hardware-testing/hardware_testing/production_qc/pipette_assembly_qc_ot3/__main__.py +++ b/hardware-testing/hardware_testing/production_qc/pipette_assembly_qc_ot3/__main__.py @@ -18,7 +18,7 @@ from opentrons_hardware.firmware_bindings.messages.messages import MessageDefinition from opentrons_hardware.firmware_bindings.constants import SensorType, SensorId -from opentrons.config.types import LiquidProbeSettings, OutputOptions +from opentrons.config.types import LiquidProbeSettings from opentrons.hardware_control.types import ( TipStateType, FailedTipStateCheck, @@ -1378,13 +1378,11 @@ async def _test_liquid_probe( plunger_speed=probe_cfg.plunger_speed, plunger_impulse_time=0.2, sensor_threshold_pascals=probe_cfg.sensor_threshold_pascals, - output_option=OutputOptions.can_bus_only, # FIXME: remove aspirate_while_sensing=False, z_overlap_between_passes_mm=0.1, plunger_reset_offset=2.0, samples_for_baselining=20, sample_time_sec=0.004, - data_files=None, ) end_z = await api.liquid_probe( mount, max_z_distance_machine_coords, probe_settings, probe=probe diff --git a/hardware-testing/hardware_testing/production_qc/robot_assembly_qc_ot3/test_instruments.py b/hardware-testing/hardware_testing/production_qc/robot_assembly_qc_ot3/test_instruments.py index 3301c1d7ab0..65d4618ff74 100644 --- a/hardware-testing/hardware_testing/production_qc/robot_assembly_qc_ot3/test_instruments.py +++ b/hardware-testing/hardware_testing/production_qc/robot_assembly_qc_ot3/test_instruments.py @@ -1,7 +1,7 @@ """Test Instruments.""" from typing import List, Tuple, Optional, Union -from opentrons.config.types import CapacitivePassSettings, OutputOptions +from opentrons.config.types import CapacitivePassSettings from opentrons.hardware_control.ot3api import OT3API from hardware_testing.data.csv_report import ( @@ -30,7 +30,6 @@ max_overrun_distance_mm=0, speed_mm_per_s=Z_PROBE_DISTANCE_MM / Z_PROBE_TIME_SECONDS, sensor_threshold_pf=1.0, - output_option=OutputOptions.can_bus_only, ) RELATIVE_MOVE_FROM_HOME_DELTA = Point(x=-500, y=-300) diff --git a/hardware-testing/hardware_testing/scripts/gripper_ot3.py b/hardware-testing/hardware_testing/scripts/gripper_ot3.py index 6c64d84105d..de2578c9ff8 100644 --- a/hardware-testing/hardware_testing/scripts/gripper_ot3.py +++ b/hardware-testing/hardware_testing/scripts/gripper_ot3.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Optional, List, Any, Dict -from opentrons.config.defaults_ot3 import CapacitivePassSettings, OutputOptions +from opentrons.config.defaults_ot3 import CapacitivePassSettings from opentrons.hardware_control.ot3api import OT3API from hardware_testing.opentrons_api import types @@ -73,7 +73,6 @@ max_overrun_distance_mm=1, speed_mm_per_s=1, sensor_threshold_pf=0.5, - output_option=OutputOptions.sync_only, ) LABWARE_PROBE_CORNER_TOP_LEFT_XY = { "plate": Point(x=5, y=-5), diff --git a/hardware/opentrons_hardware/firmware_bindings/messages/messages.py b/hardware/opentrons_hardware/firmware_bindings/messages/messages.py index 0249ddec69e..35683bc1afb 100644 --- a/hardware/opentrons_hardware/firmware_bindings/messages/messages.py +++ b/hardware/opentrons_hardware/firmware_bindings/messages/messages.py @@ -74,6 +74,7 @@ defs.BaselineSensorResponse, defs.SetSensorThresholdRequest, defs.ReadFromSensorResponse, + defs.BatchReadFromSensorResponse, defs.SensorThresholdResponse, defs.SensorDiagnosticRequest, defs.SensorDiagnosticResponse, diff --git a/hardware/opentrons_hardware/hardware_control/tool_sensors.py b/hardware/opentrons_hardware/hardware_control/tool_sensors.py index 173a8c2738b..95076f01c1c 100644 --- a/hardware/opentrons_hardware/hardware_control/tool_sensors.py +++ b/hardware/opentrons_hardware/hardware_control/tool_sensors.py @@ -1,5 +1,6 @@ """Functions for commanding motion limited by tool sensors.""" import asyncio +from contextlib import AsyncExitStack from functools import partial from typing import ( Union, @@ -11,6 +12,7 @@ AsyncContextManager, Optional, AsyncIterator, + Mapping, ) from logging import getLogger from numpy import float64 @@ -41,6 +43,7 @@ from opentrons_hardware.sensors.sensor_driver import SensorDriver, LogListener from opentrons_hardware.sensors.types import ( sensor_fixed_point_conversion, + SensorDataType, ) from opentrons_hardware.sensors.sensor_types import ( SensorInformation, @@ -61,28 +64,13 @@ ) LOG = getLogger(__name__) + PipetteProbeTarget = Literal[NodeId.pipette_left, NodeId.pipette_right] InstrumentProbeTarget = Union[PipetteProbeTarget, Literal[NodeId.gripper]] ProbeSensorDict = Union[ Dict[SensorId, PressureSensor], Dict[SensorId, CapacitiveSensor] ] -pressure_output_file_heading = [ - "time(s)", - "Pressure(pascals)", - "z_velocity(mm/s)", - "plunger_velocity(mm/s)", - "threshold(pascals)", -] - -capacitive_output_file_heading = [ - "time(s)", - "Capacitance(farads)", - "z_velocity(mm/s)", - "plunger_velocity(mm/s)", - "threshold(farads)", -] - def _fix_pass_step_for_buffer( move_group: MoveGroupStep, @@ -167,124 +155,6 @@ def _build_pass_step( return move_group -async def run_sync_buffer_to_csv( - messenger: CanMessenger, - mount_speed: float, - plunger_speed: float, - threshold: float, - head_node: NodeId, - move_group: MoveGroupRunner, - log_files: Dict[SensorId, str], - tool: InstrumentProbeTarget, - sensor_type: SensorType, - output_file_heading: list[str], - raise_z: Optional[MoveGroupRunner] = None, -) -> Dict[NodeId, MotorPositionStatus]: - """Runs the sensor pass move group and creates a csv file with the results.""" - sensor_metadata = [0, 0, mount_speed, plunger_speed, threshold] - positions = await move_group.run(can_messenger=messenger) - # wait a little to see the dropoff curve - await asyncio.sleep(0.15) - for sensor_id in log_files.keys(): - await messenger.ensure_send( - node_id=tool, - message=BindSensorOutputRequest( - payload=BindSensorOutputRequestPayload( - sensor=SensorTypeField(sensor_type), - sensor_id=SensorIdField(sensor_id), - binding=SensorOutputBindingField(SensorOutputBinding.none), - ) - ), - expected_nodes=[tool], - ) - if raise_z is not None: - # if probing is finished, move the head node back up before requesting the data buffer - if positions[head_node].move_ack == MoveCompleteAck.stopped_by_condition: - await raise_z.run(can_messenger=messenger) - for sensor_id in log_files.keys(): - sensor_capturer = LogListener( - mount=head_node, - data_file=log_files[sensor_id], - file_heading=output_file_heading, - sensor_metadata=sensor_metadata, - ) - async with sensor_capturer: - messenger.add_listener(sensor_capturer, None) - request = SendAccumulatedSensorDataRequest( - payload=SendAccumulatedSensorDataPayload( - sensor_id=SensorIdField(sensor_id), - sensor_type=SensorTypeField(sensor_type), - ) - ) - await messenger.send( - node_id=tool, - message=request, - ) - await sensor_capturer.wait_for_complete( - message_index=request.payload.message_index.value - ) - messenger.remove_listener(sensor_capturer) - return positions - - -async def run_stream_output_to_csv( - messenger: CanMessenger, - sensors: ProbeSensorDict, - mount_speed: float, - plunger_speed: float, - threshold: float, - head_node: NodeId, - move_group: MoveGroupRunner, - log_files: Dict[SensorId, str], - output_file_heading: list[str], -) -> Dict[NodeId, MotorPositionStatus]: - """Runs the sensor pass move group and creates a csv file with the results.""" - sensor_metadata = [0, 0, mount_speed, plunger_speed, threshold] - sensor_capturer = LogListener( - mount=head_node, - data_file=log_files[ - next(iter(log_files)) - ], # hardcode to the first file, need to think more on this - file_heading=output_file_heading, - sensor_metadata=sensor_metadata, - ) - binding = [SensorOutputBinding.sync, SensorOutputBinding.report] - binding_field = SensorOutputBindingField.from_flags(binding) - for sensor_id in sensors.keys(): - sensor_info = sensors[sensor_id].sensor - await messenger.ensure_send( - node_id=sensor_info.node_id, - message=BindSensorOutputRequest( - payload=BindSensorOutputRequestPayload( - sensor=SensorTypeField(sensor_info.sensor_type), - sensor_id=SensorIdField(sensor_info.sensor_id), - binding=binding_field, - ) - ), - expected_nodes=[sensor_info.node_id], - ) - - messenger.add_listener(sensor_capturer, None) - async with sensor_capturer: - positions = await move_group.run(can_messenger=messenger) - messenger.remove_listener(sensor_capturer) - - for sensor_id in sensors.keys(): - sensor_info = sensors[sensor_id].sensor - await messenger.ensure_send( - node_id=sensor_info.node_id, - message=BindSensorOutputRequest( - payload=BindSensorOutputRequestPayload( - sensor=SensorTypeField(sensor_info.sensor_type), - sensor_id=SensorIdField(sensor_info.sensor_id), - binding=SensorOutputBindingField(SensorOutputBinding.none), - ) - ), - expected_nodes=[sensor_info.node_id], - ) - return positions - - async def _setup_pressure_sensors( messenger: CanMessenger, sensor_id: SensorId, @@ -351,42 +221,42 @@ async def _setup_capacitive_sensors( return result -async def _run_with_binding( +async def finalize_logs( messenger: CanMessenger, - sensors: ProbeSensorDict, - sensor_runner: MoveGroupRunner, - binding: List[SensorOutputBinding], -) -> Dict[NodeId, MotorPositionStatus]: - binding_field = SensorOutputBindingField.from_flags(binding) - for sensor_id in sensors.keys(): - sensor_info = sensors[sensor_id].sensor + tool: NodeId, + listeners: Dict[SensorId, LogListener], + sensors: Mapping[SensorId, Union[CapacitiveSensor, PressureSensor]], +) -> None: + """Signal the sensors to finish sending their data and wait for it to flush out.""" + for s_id in sensors.keys(): + # Tell the sensor to stop recording await messenger.ensure_send( - node_id=sensor_info.node_id, - message=BindSensorOutputRequest( - payload=BindSensorOutputRequestPayload( - sensor=SensorTypeField(sensor_info.sensor_type), - sensor_id=SensorIdField(sensor_info.sensor_id), - binding=binding_field, - ) - ), - expected_nodes=[sensor_info.node_id], - ) - - result = await sensor_runner.run(can_messenger=messenger) - for sensor_id in sensors.keys(): - sensor_info = sensors[sensor_id].sensor - await messenger.ensure_send( - node_id=sensor_info.node_id, + node_id=tool, message=BindSensorOutputRequest( payload=BindSensorOutputRequestPayload( - sensor=SensorTypeField(sensor_info.sensor_type), - sensor_id=SensorIdField(sensor_info.sensor_id), + sensor=SensorTypeField(sensors[s_id].sensor.sensor_type), + sensor_id=SensorIdField(s_id), binding=SensorOutputBindingField(SensorOutputBinding.none), ) ), - expected_nodes=[sensor_info.node_id], + expected_nodes=[tool], ) - return result + request = SendAccumulatedSensorDataRequest( + payload=SendAccumulatedSensorDataPayload( + sensor_id=SensorIdField(s_id), + sensor_type=SensorTypeField(sensors[s_id].sensor.sensor_type), + ) + ) + # set the message index of the Ack that signals this sensor is finished sending data + listeners[s_id].set_stop_ack(request.payload.message_index.value) + # tell the sensor to clear it's queue + await messenger.send( + node_id=tool, + message=request, + ) + # wait for the data to finish sending + for listener in listeners.values(): + await listener.wait_for_complete() async def liquid_probe( @@ -399,15 +269,13 @@ async def liquid_probe( threshold_pascals: float, plunger_impulse_time: float, num_baseline_reads: int, - csv_output: bool = False, - sync_buffer_output: bool = False, - can_bus_only_output: bool = False, - data_files: Optional[Dict[SensorId, str]] = None, sensor_id: SensorId = SensorId.S0, force_both_sensors: bool = False, + response_queue: Optional[ + asyncio.Queue[Dict[SensorId, List[SensorDataType]]] + ] = None, ) -> Dict[NodeId, MotorPositionStatus]: """Move the mount and pipette simultaneously while reading from the pressure sensor.""" - log_files: Dict[SensorId, str] = {} if not data_files else data_files sensor_driver = SensorDriver() threshold_fixed_point = threshold_pascals * sensor_fixed_point_conversion sensor_binding = None @@ -420,7 +288,7 @@ async def liquid_probe( + SensorOutputBinding.report + SensorOutputBinding.multi_sensor_sync ) - pressure_sensors = await _setup_pressure_sensors( + pressure_sensors: Dict[SensorId, PressureSensor] = await _setup_pressure_sensors( messenger, sensor_id, tool, @@ -440,6 +308,7 @@ async def liquid_probe( duration=float64(plunger_impulse_time), present_nodes=[tool], ) + sensor_group = _build_pass_step( movers=[head_node, tool], distance={head_node: max_z_distance, tool: p_pass_distance}, @@ -449,64 +318,56 @@ async def liquid_probe( stop_condition=MoveStopCondition.sync_line, binding_flags=sensor_binding, ) - if sync_buffer_output: - sensor_group = _fix_pass_step_for_buffer( - sensor_group, - movers=[head_node, tool], - distance={head_node: max_z_distance, tool: p_pass_distance}, - speed={head_node: mount_speed, tool: plunger_speed}, - sensor_type=SensorType.pressure, - sensor_id=sensor_id, - stop_condition=MoveStopCondition.sync_line, - binding_flags=sensor_binding, - ) + sensor_group = _fix_pass_step_for_buffer( + sensor_group, + movers=[head_node, tool], + distance={head_node: max_z_distance, tool: p_pass_distance}, + speed={head_node: mount_speed, tool: plunger_speed}, + sensor_type=SensorType.pressure, + sensor_id=sensor_id, + stop_condition=MoveStopCondition.sync_line, + binding_flags=sensor_binding, + ) sensor_runner = MoveGroupRunner(move_groups=[[lower_plunger], [sensor_group]]) - if csv_output: - return await run_stream_output_to_csv( - messenger, - pressure_sensors, - mount_speed, - plunger_speed, - threshold_pascals, - head_node, - sensor_runner, - log_files, - pressure_output_file_heading, - ) - elif sync_buffer_output: - raise_z = create_step( - distance={head_node: float64(max_z_distance)}, - velocity={head_node: float64(-1 * mount_speed)}, - acceleration={}, - duration=float64(max_z_distance / mount_speed), - present_nodes=[head_node], - ) - raise_z_runner = MoveGroupRunner(move_groups=[[raise_z]]) - - return await run_sync_buffer_to_csv( - messenger=messenger, - mount_speed=mount_speed, - plunger_speed=plunger_speed, - threshold=threshold_pascals, - head_node=head_node, - move_group=sensor_runner, - raise_z=raise_z_runner, - log_files=log_files, - tool=tool, - sensor_type=SensorType.pressure, - output_file_heading=pressure_output_file_heading, - ) - elif can_bus_only_output: - binding = [SensorOutputBinding.sync, SensorOutputBinding.report] - return await _run_with_binding( - messenger, pressure_sensors, sensor_runner, binding - ) - else: # none - binding = [SensorOutputBinding.sync] - return await _run_with_binding( - messenger, pressure_sensors, sensor_runner, binding - ) + + raise_z = create_step( + distance={head_node: float64(max_z_distance)}, + velocity={head_node: float64(-1 * mount_speed)}, + acceleration={}, + duration=float64(max_z_distance / mount_speed), + present_nodes=[head_node], + ) + + raise_z_runner = MoveGroupRunner(move_groups=[[raise_z]]) + listeners = { + s_id: LogListener(messenger, pressure_sensors[s_id]) + for s_id in pressure_sensors.keys() + } + + LOG.info( + f"Starting LLD pass: {head_node} {sensor_id} max p distance {max_p_distance} max z distance {max_z_distance}" + ) + async with AsyncExitStack() as binding_stack: + for listener in listeners.values(): + await binding_stack.enter_async_context(listener) + positions = await sensor_runner.run(can_messenger=messenger) + if positions[head_node].move_ack == MoveCompleteAck.stopped_by_condition: + LOG.info( + f"Liquid found {head_node} motor_postion {positions[head_node].motor_position} encoder position {positions[head_node].encoder_position}" + ) + await raise_z_runner.run(can_messenger=messenger) + await finalize_logs(messenger, tool, listeners, pressure_sensors) + + # give response data to any consumer that wants it + if response_queue: + for s_id in listeners.keys(): + data = listeners[s_id].get_data() + if data: + for d in data: + response_queue.put_nowait({s_id: data}) + + return positions async def check_overpressure( @@ -536,10 +397,9 @@ async def capacitive_probe( mount_speed: float, sensor_id: SensorId = SensorId.S0, relative_threshold_pf: float = 1.0, - csv_output: bool = False, - sync_buffer_output: bool = False, - can_bus_only_output: bool = False, - data_files: Optional[Dict[SensorId, str]] = None, + response_queue: Optional[ + asyncio.Queue[dict[SensorId, list[SensorDataType]]] + ] = None, ) -> MotorPositionStatus: """Move the specified tool down until its capacitive sensor triggers. @@ -549,7 +409,6 @@ async def capacitive_probe( The direction is sgn(distance)*sgn(speed), so you can set the direction either by negating speed or negating distance. """ - log_files: Dict[SensorId, str] = {} if not data_files else data_files sensor_driver = SensorDriver() pipette_present = tool in [NodeId.pipette_left, NodeId.pipette_right] @@ -577,53 +436,36 @@ async def capacitive_probe( sensor_id=sensor_id, stop_condition=MoveStopCondition.sync_line, ) - if sync_buffer_output: - sensor_group = _fix_pass_step_for_buffer( - sensor_group, - movers=movers, - distance=probe_distance, - speed=probe_speed, - sensor_type=SensorType.capacitive, - sensor_id=sensor_id, - stop_condition=MoveStopCondition.sync_line, - ) + + sensor_group = _fix_pass_step_for_buffer( + sensor_group, + movers=movers, + distance=probe_distance, + speed=probe_speed, + sensor_type=SensorType.capacitive, + sensor_id=sensor_id, + stop_condition=MoveStopCondition.sync_line, + ) runner = MoveGroupRunner(move_groups=[[sensor_group]]) - if csv_output: - positions = await run_stream_output_to_csv( - messenger, - capacitive_sensors, - mount_speed, - 0.0, - relative_threshold_pf, - mover, - runner, - log_files, - capacitive_output_file_heading, - ) - elif sync_buffer_output: - positions = await run_sync_buffer_to_csv( - messenger, - mount_speed, - 0.0, - relative_threshold_pf, - mover, - runner, - log_files, - tool=tool, - sensor_type=SensorType.capacitive, - output_file_heading=capacitive_output_file_heading, - ) - elif can_bus_only_output: - binding = [SensorOutputBinding.sync, SensorOutputBinding.report] - positions = await _run_with_binding( - messenger, capacitive_sensors, runner, binding - ) - else: - binding = [SensorOutputBinding.sync] - positions = await _run_with_binding( - messenger, capacitive_sensors, runner, binding - ) + + listeners = { + s_id: LogListener(messenger, capacitive_sensors[s_id]) + for s_id in capacitive_sensors.keys() + } + async with AsyncExitStack() as binding_stack: + for listener in listeners.values(): + await binding_stack.enter_async_context(listener) + positions = await runner.run(can_messenger=messenger) + await finalize_logs(messenger, tool, listeners, capacitive_sensors) + + # give response data to any consumer that wants it + if response_queue: + for s_id in listeners.keys(): + data = listeners[s_id].get_data() + if data: + for d in data: + response_queue.put_nowait({s_id: data}) return positions[mover] diff --git a/hardware/opentrons_hardware/sensors/__init__.py b/hardware/opentrons_hardware/sensors/__init__.py index adc4f0c52af..3ae059861a1 100644 --- a/hardware/opentrons_hardware/sensors/__init__.py +++ b/hardware/opentrons_hardware/sensors/__init__.py @@ -1 +1,3 @@ """Sub-module for sensor drivers.""" + +SENSOR_LOG_NAME = "pipettes-sensor-log" diff --git a/hardware/opentrons_hardware/sensors/sensor_driver.py b/hardware/opentrons_hardware/sensors/sensor_driver.py index 611bc091970..0f1904f8a26 100644 --- a/hardware/opentrons_hardware/sensors/sensor_driver.py +++ b/hardware/opentrons_hardware/sensors/sensor_driver.py @@ -1,9 +1,8 @@ """Capacitve Sensor Driver Class.""" import time import asyncio -import csv -from typing import Optional, AsyncIterator, Any, Sequence +from typing import Optional, AsyncIterator, Any, Sequence, List, Union from contextlib import asynccontextmanager, suppress from logging import getLogger @@ -19,7 +18,6 @@ from opentrons_hardware.firmware_bindings.constants import ( SensorOutputBinding, SensorThresholdMode, - NodeId, ) from opentrons_hardware.sensors.types import ( SensorDataType, @@ -32,7 +30,12 @@ SensorThresholdInformation, ) -from opentrons_hardware.sensors.sensor_types import BaseSensorType, ThresholdSensorType +from opentrons_hardware.sensors.sensor_types import ( + BaseSensorType, + ThresholdSensorType, + PressureSensor, + CapacitiveSensor, +) from opentrons_hardware.firmware_bindings.messages.payloads import ( BindSensorOutputRequestPayload, ) @@ -46,8 +49,10 @@ ) from .sensor_abc import AbstractSensorDriver from .scheduler import SensorScheduler +from . import SENSOR_LOG_NAME LOG = getLogger(__name__) +SENSOR_LOG = getLogger(SENSOR_LOG_NAME) class SensorDriver(AbstractSensorDriver): @@ -226,43 +231,50 @@ class LogListener: def __init__( self, - mount: NodeId, - data_file: Any, - file_heading: Sequence[str], - sensor_metadata: Sequence[Any], + messenger: CanMessenger, + sensor: Union[PressureSensor, CapacitiveSensor], ) -> None: """Build the capturer.""" - self.csv_writer = Any - self.data_file = data_file - self.file_heading = file_heading - self.sensor_metadata = sensor_metadata - self.response_queue: asyncio.Queue[float] = asyncio.Queue() - self.mount = mount + self.response_queue: asyncio.Queue[SensorDataType] = asyncio.Queue() + self.tool = sensor.sensor.node_id self.start_time = 0.0 self.event: Any = None + self.messenger = messenger + self.sensor = sensor + self.type = sensor.sensor.sensor_type + self.id = sensor.sensor.sensor_id - async def __aenter__(self) -> None: - """Create a csv heading for logging pressure readings.""" - self.data_file = open(self.data_file, "w") - self.csv_writer = csv.writer(self.data_file) - self.csv_writer.writerows([self.file_heading, self.sensor_metadata]) + def get_data(self) -> Optional[List[SensorDataType]]: + """Return the sensor data captured by this listener.""" + if self.response_queue.empty(): + return None + data: List[SensorDataType] = [] + while not self.response_queue.empty(): + data.append(self.response_queue.get_nowait()) + return data + async def __aenter__(self) -> None: + """Start logging sensor readings.""" + self.messenger.add_listener(self, None) self.start_time = time.time() + SENSOR_LOG.info(f"Data capture for {self.tool.name} started {self.start_time}") async def __aexit__(self, *args: Any) -> None: - """Close csv file.""" - self.data_file.close() + """Finish the capture.""" + self.messenger.remove_listener(self) + SENSOR_LOG.info(f"Data capture for {self.tool.name} ended {time.time()}") - async def wait_for_complete( - self, wait_time: float = 10, message_index: int = 0 - ) -> None: - """Wait for the data to stop, only use this with a send_accumulated_data_request.""" + def set_stop_ack(self, message_index: int = 0) -> None: + """Tell the Listener which message index to wait for.""" self.event = asyncio.Event() self.expected_ack = message_index + + async def wait_for_complete(self, wait_time: float = 10) -> None: + """Wait for the data to stop.""" with suppress(asyncio.TimeoutError): await asyncio.wait_for(self.event.wait(), wait_time) if not self.event.is_set(): - LOG.error("Did not receive the full data set from the sensor") + SENSOR_LOG.error("Did not receive the full data set from the sensor") self.event = None def __call__( @@ -271,30 +283,44 @@ def __call__( arbitration_id: ArbitrationId, ) -> None: """Callback entry point for capturing messages.""" + if arbitration_id.parts.originating_node_id != self.tool: + # check that this is from the node we care about + return if isinstance(message, message_definitions.ReadFromSensorResponse): + if ( + message.payload.sensor_id.value is not self.id + or message.payload.sensor is not self.type + ): + # ignore sensor responses from other sensors + return data = sensor_types.SensorDataType.build( message.payload.sensor_data, message.payload.sensor - ).to_float() + ) self.response_queue.put_nowait(data) - current_time = round((time.time() - self.start_time), 3) - self.csv_writer.writerow([current_time, data]) # type: ignore + SENSOR_LOG.info( + f"Revieved from {arbitration_id}: {message.payload.sensor_id}:{message.payload.sensor}: {data}" + ) if isinstance(message, message_definitions.BatchReadFromSensorResponse): data_length = message.payload.data_length.value data_bytes = message.payload.sensor_data.value data_ints = [ - int.from_bytes(data_bytes[i * 4 : i * 4 + 4]) + int.from_bytes(data_bytes[i * 4 : i * 4 + 4], byteorder="little") for i in range(data_length) ] - for d in data_ints: - data = sensor_types.SensorDataType.build( - d, message.payload.sensor - ).to_float() - self.response_queue.put_nowait(data) - current_time = round((time.time() - self.start_time), 3) - self.csv_writer.writerow([current_time, data]) + data_floats = [ + sensor_types.SensorDataType.build(d, message.payload.sensor) + for d in data_ints + ] + + for d in data_floats: + self.response_queue.put_nowait(d) + SENSOR_LOG.info( + f"Revieved from {arbitration_id}: {message.payload.sensor_id}:{message.payload.sensor}: {data_floats}" + ) if isinstance(message, message_definitions.Acknowledgement): if ( self.event is not None and message.payload.message_index.value == self.expected_ack ): + SENSOR_LOG.info("Finished receiving sensor data") self.event.set() diff --git a/hardware/tests/opentrons_hardware/hardware_control/test_tool_sensors.py b/hardware/tests/opentrons_hardware/hardware_control/test_tool_sensors.py index 2dc7614da63..0c53b81057a 100644 --- a/hardware/tests/opentrons_hardware/hardware_control/test_tool_sensors.py +++ b/hardware/tests/opentrons_hardware/hardware_control/test_tool_sensors.py @@ -1,12 +1,10 @@ """Test the tool-sensor coordination code.""" import logging from mock import patch, AsyncMock, call -import os import pytest from contextlib import asynccontextmanager from typing import Iterator, List, Tuple, AsyncIterator, Any, Dict, Callable from opentrons_hardware.firmware_bindings.messages.message_definitions import ( - AddLinearMoveRequest, ExecuteMoveGroupRequest, MoveCompleted, ReadFromSensorResponse, @@ -50,7 +48,6 @@ SensorType, SensorThresholdMode, SensorOutputBinding, - MoveStopCondition, ) from opentrons_hardware.sensors.scheduler import SensorScheduler from opentrons_hardware.sensors.sensor_driver import SensorDriver @@ -187,78 +184,7 @@ def check_second_move( ), ] - def get_responder() -> Iterator[ - Callable[ - [NodeId, MessageDefinition], List[Tuple[NodeId, MessageDefinition, NodeId]] - ] - ]: - yield check_first_move - yield check_second_move - - responder_getter = get_responder() - - def move_responder( - node_id: NodeId, message: MessageDefinition - ) -> List[Tuple[NodeId, MessageDefinition, NodeId]]: - message.payload.serialize() - if isinstance(message, ExecuteMoveGroupRequest): - responder = next(responder_getter) - return responder(node_id, message) - else: - return [] - - message_send_loopback.add_responder(move_responder) - - position = await liquid_probe( - messenger=mock_messenger, - tool=target_node, - head_node=motor_node, - max_p_distance=70, - mount_speed=10, - plunger_speed=8, - threshold_pascals=threshold_pascals, - plunger_impulse_time=0.2, - num_baseline_reads=20, - csv_output=False, - sync_buffer_output=False, - can_bus_only_output=False, - sensor_id=SensorId.S0, - ) - assert position[motor_node].positions_only()[0] == 14 - assert mock_sensor_threshold.call_args_list[0][0][0] == SensorThresholdInformation( - sensor=sensor_info, - data=SensorDataType.build(threshold_pascals * 65536, sensor_info.sensor_type), - mode=SensorThresholdMode.absolute, - ) - - -@pytest.mark.parametrize( - "csv_output, sync_buffer_output, can_bus_only_output, move_stop_condition", - [ - (True, False, False, MoveStopCondition.sync_line), - (True, True, False, MoveStopCondition.sensor_report), - (False, False, True, MoveStopCondition.sync_line), - ], -) -async def test_liquid_probe_output_options( - mock_messenger: AsyncMock, - mock_bind_output: AsyncMock, - message_send_loopback: CanLoopback, - mock_sensor_threshold: AsyncMock, - csv_output: bool, - sync_buffer_output: bool, - can_bus_only_output: bool, - move_stop_condition: MoveStopCondition, -) -> None: - """Test that liquid_probe targets the right nodes.""" - sensor_info = SensorInformation( - sensor_type=SensorType.pressure, - sensor_id=SensorId.S0, - node_id=NodeId.pipette_left, - ) - test_csv_file: str = os.path.join(os.getcwd(), "test.csv") - - def check_first_move( + def check_third_move( node_id: NodeId, message: MessageDefinition ) -> List[Tuple[NodeId, MessageDefinition, NodeId]]: return [ @@ -274,44 +200,10 @@ def check_first_move( ack_id=UInt8Field(1), ) ), - NodeId.pipette_left, + motor_node, ) ] - def check_second_move( - node_id: NodeId, message: MessageDefinition - ) -> List[Tuple[NodeId, MessageDefinition, NodeId]]: - return [ - ( - NodeId.host, - MoveCompleted( - payload=MoveCompletedPayload( - group_id=UInt8Field(1), - seq_id=UInt8Field(0), - current_position_um=UInt32Field(14000), - encoder_position_um=Int32Field(14000), - position_flags=MotorPositionFlagsField(0), - ack_id=UInt8Field(2), - ) - ), - NodeId.head_l, - ), - ( - NodeId.host, - MoveCompleted( - payload=MoveCompletedPayload( - group_id=UInt8Field(1), - seq_id=UInt8Field(0), - current_position_um=UInt32Field(14000), - encoder_position_um=Int32Field(14000), - position_flags=MotorPositionFlagsField(0), - ack_id=UInt8Field(2), - ) - ), - NodeId.pipette_left, - ), - ] - def get_responder() -> Iterator[ Callable[ [NodeId, MessageDefinition], List[Tuple[NodeId, MessageDefinition, NodeId]] @@ -319,6 +211,7 @@ def get_responder() -> Iterator[ ]: yield check_first_move yield check_second_move + yield check_third_move responder_getter = get_responder() @@ -330,42 +223,26 @@ def move_responder( responder = next(responder_getter) return responder(node_id, message) else: - if ( - isinstance(message, AddLinearMoveRequest) - and node_id == NodeId.pipette_left - and message.payload.group_id == 2 - ): - assert ( - message.payload.request_stop_condition.value == move_stop_condition - ) return [] message_send_loopback.add_responder(move_responder) - try: - position = await liquid_probe( - messenger=mock_messenger, - tool=NodeId.pipette_left, - head_node=NodeId.head_l, - max_p_distance=70, - mount_speed=10, - plunger_speed=8, - threshold_pascals=14, - plunger_impulse_time=0.2, - num_baseline_reads=20, - csv_output=csv_output, - sync_buffer_output=sync_buffer_output, - can_bus_only_output=can_bus_only_output, - data_files={SensorId.S0: test_csv_file}, - sensor_id=SensorId.S0, - ) - finally: - if os.path.isfile(test_csv_file): - # clean up the test file this creates if it exists - os.remove(test_csv_file) - assert position[NodeId.head_l].positions_only()[0] == 14 + + position = await liquid_probe( + messenger=mock_messenger, + tool=target_node, + head_node=motor_node, + max_p_distance=70, + mount_speed=10, + plunger_speed=8, + threshold_pascals=threshold_pascals, + plunger_impulse_time=0.2, + num_baseline_reads=20, + sensor_id=SensorId.S0, + ) + assert position[motor_node].positions_only()[0] == 14 assert mock_sensor_threshold.call_args_list[0][0][0] == SensorThresholdInformation( sensor=sensor_info, - data=SensorDataType.build(14 * 65536, sensor_info.sensor_type), + data=SensorDataType.build(threshold_pascals * 65536, sensor_info.sensor_type), mode=SensorThresholdMode.absolute, )