From f113949c2e9fa3a18e862980113b000fa97d780b Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 6 Oct 2022 14:45:10 -0400 Subject: [PATCH] feat: offer full unstructured data --- src/nd2/_sdk/latest.pyi | 7 +++- src/nd2/nd2file.py | 77 ++++++++++++++++++++++++++--------------- 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/src/nd2/_sdk/latest.pyi b/src/nd2/_sdk/latest.pyi index 5b332ac..a63ed4a 100644 --- a/src/nd2/_sdk/latest.pyi +++ b/src/nd2/_sdk/latest.pyi @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union import numpy as np @@ -7,6 +7,9 @@ from .. import structures class ND2Reader: path: str + _meta_map: Dict[str, int] # map of metadata keys to their byte offset + _frame_map: Dict[int, int] # map of sequence index to byte offset + def __init__( self, path: Union[str, Path], @@ -42,3 +45,5 @@ class ND2Reader: def _custom_data(self) -> Dict[str, Any]: ... def _read_image(self, index: int) -> np.ndarray: ... def channel_names(self) -> List[str]: ... + def _get_meta_chunk(self, key: str) -> bytes: + """Return the metadata chunk for the given key in `_meta_map`.""" diff --git a/src/nd2/nd2file.py b/src/nd2/nd2file.py index a1610e4..1b51316 100644 --- a/src/nd2/nd2file.py +++ b/src/nd2/nd2file.py @@ -37,6 +37,7 @@ import xarray as xr from typing_extensions import Literal + from ._sdk.latest import ND2Reader as LatestSDKReader from .structures import Position @@ -169,34 +170,34 @@ def experiment(self) -> List[ExpLoop]: tuple(p.stagePositionUm): p.name for p in item.parameters.points } if not any(names.values()): - _exp = self.unstructured_metadata(unnest=True) + _exp = self.unstructured_metadata( + include={"ImageMetadataLV"}, unnest=True + )["ImageMetadataLV"] if n >= len(_exp): continue with contextlib.suppress(Exception): _fix_names(_exp[n], item.parameters.points) return exp - @overload def unstructured_metadata( - self, unnest: Literal[True], strip_prefix: bool = True - ) -> List[Dict[str, Any]]: - ... - - @overload - def unstructured_metadata( - self, unnest: Literal[False] = False, strip_prefix: bool = True + self, + *, + unnest: bool = False, + strip_prefix: bool = True, + include: Optional[Set[str]] = None, + exclude: Optional[Set[str]] = None, ) -> Dict[str, Any]: - ... + """Exposes, and attempts to decode, each metadata chunk in the file. - def unstructured_metadata( - self, unnest: bool = False, strip_prefix: bool = True - ) -> Union[list, dict]: - """Exposes all metadata in the `ImageMetadataLV` portion of the nd2 header. + This is provided as a *experimental* fallback in the event that + `ND2File.experiment` does not contain all of the information you need. No + attempt is made to parse or validate the metadata, and the format of various + sections, *may* change in future versions of nd2. Consumption of this metadata + should use appropriate exception handling! - This is provided as a fallback in the event that ND2File.experiment does not - contain all of the information you need. No attempt is made to parse the - metadata. Consumption of this metadata should use appropriate exception - handling. + The 'ImageMetadataLV' chunk is the most likely to contain useful information, + but if you're generally looking for "hidden" metadata, it may be helpful to + look at the full output. Parameters ---------- @@ -208,12 +209,18 @@ def unstructured_metadata( Whether to strip the type information from the front of the keys in the dict. For example, if `True`: `uiModeFQ` becomes `ModeFQ` and `bUsePFS` becomes `UsePFS`, etc... by default `True` + include : Optional[Set[str]], optional + If provided, only include the specified keys in the output. by default, + all metadata sections found in the file are included. + exclude : Optional[Set[str]], optional + If provided, exclude the specified keys from the output. by default `None` Returns ------- - Union[list[list | dict], dict] - If unnest is `True`, returns a `list` of dicts or lists, else a `dict` is - returned where nested experiment loop levels are available at `NextLevelEx`. + Dict[str, Any] + A dict of the unstructured metadata, with keys that are the type of the + metadata chunk (things like 'CustomData|RoiMetadata_v1' or + 'ImageMetadataLV'), and values that are associated metadata chunk. """ if self.is_legacy: raise NotImplementedError( @@ -222,14 +229,28 @@ def unstructured_metadata( from ._nd2decode import decode_metadata, unnest_experiments - try: - meta = self._rdr._get_meta_chunk("ImageMetadataLV") # type: ignore - except KeyError: - return [] if unnest else {} + output: Dict[str, Any] = {} - data = decode_metadata(meta, strip_prefix=strip_prefix) - data = data["SLxExperiment"] - return unnest_experiments(data) if unnest else data + rdr = cast("LatestSDKReader", self._rdr) + keys = set(include) if include else set(rdr._meta_map) + if exclude: + keys = {k for k in keys if k not in exclude} + + for key in sorted(keys): + try: + meta: bytes = rdr._get_meta_chunk(key) + if meta.startswith(b"<"): + # probably xml + decoded: Any = meta.decode("utf-8") + else: + decoded = decode_metadata(meta, strip_prefix=strip_prefix) + if key == "ImageMetadataLV" and unnest: + decoded = unnest_experiments(decoded) + except Exception: + decoded = meta + + output[key] = decoded + return output @cached_property def metadata(self) -> Union[Metadata, dict]: