From 7397235d331af542df8c9a03df8153e265b2285e Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 8 Sep 2022 15:03:21 -0400 Subject: [PATCH] fix: fix occasional wrong image width --- src/nd2/_sdk/latest.pyx | 11 +++++++++++ src/nd2/nd2file.py | 6 +++--- tests/test_reader.py | 33 +++++++++++++++++++++++++-------- tests/test_sdk.py | 13 ++++++++++++- 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/src/nd2/_sdk/latest.pyx b/src/nd2/_sdk/latest.pyx index e1ce059..b004dc5 100644 --- a/src/nd2/_sdk/latest.pyx +++ b/src/nd2/_sdk/latest.pyx @@ -113,6 +113,17 @@ cdef class ND2Reader: cont = self._metadata().get('contents') attrs = self._attributes() nC = cont.get('channelCount') if cont else attrs.get("componentCount", 1) + # widthPx doesn't always equal widthBytes / bytesPerPixel ... but when it doesn't + # the image is slanted anyway. For now, we just force it here. + w = attrs.get('widthBytes') // (attrs.get("componentCount", 1) * attrs.get('bitsPerComponentInMemory') // 8) + if w != attrs['widthPx']: + wb = attrs.get('widthBytes') + bpp = (attrs.get('bitsPerComponentInMemory') // 8) + warnings.warn( + f"widthPx ({attrs['widthPx']}) != widthBytes ({wb}) / bytesPerPixel ({bpp}). " + f"Forcing widthPx to {w} (widthBytes / bytesPerPixel)." + ) + attrs['widthPx'] = w self.__attributes = structures.Attributes(**attrs, channelCount=nC) return self.__attributes diff --git a/src/nd2/nd2file.py b/src/nd2/nd2file.py index e3c602c..e8a6ec0 100644 --- a/src/nd2/nd2file.py +++ b/src/nd2/nd2file.py @@ -399,7 +399,7 @@ def _dask_block(self, copy: bool, block_id: Tuple[int]) -> np.ndarray: f"Cannot get chunk {block_id} for single frame image." ) idx = 0 - data = self._get_frame(cast(int, idx)) + data = self._get_frame(int(idx)) # type: ignore data = data.copy() if copy else data return data[(np.newaxis,) * ncoords] finally: @@ -498,8 +498,8 @@ def _coord_shape(self) -> Tuple[int, ...]: def _frame_count(self) -> int: return int(np.prod(self._coord_shape)) - def _get_frame(self, index: int) -> np.ndarray: - frame = self._rdr._read_image(index) + def _get_frame(self, index: SupportsInt) -> np.ndarray: + frame = self._rdr._read_image(int(index)) frame.shape = self._raw_frame_shape return frame.transpose((2, 0, 1, 3)).squeeze() diff --git a/tests/test_reader.py b/tests/test_reader.py index 8d66417..7eda609 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -2,6 +2,7 @@ import os import pickle import sys +from contextlib import nullcontext from pathlib import Path import dask.array as da @@ -16,15 +17,31 @@ DATA = Path(__file__).parent / "data" -def test_metadata_extraction(new_nd2): +def _warning_ctx(fname: Path): + if fname.name in { + "jonas_control002.nd2", + "jonas_JJ1473_control_24h_JJ1473_control_24h_03.nd2", + }: + return pytest.warns(UserWarning, match="widthPx") + return nullcontext() + + +def test_metadata_extraction(new_nd2: Path): assert ND2File.is_supported_file(new_nd2) - with ND2File(new_nd2) as nd: + with _warning_ctx(new_nd2), ND2File(new_nd2) as nd: assert nd.path == str(new_nd2) assert not nd.closed # assert isinstance(nd._rdr._seq_count(), int) assert isinstance(nd.attributes, structures.Attributes) + # this is one of the "skewed" files where widthPx seems + # to be set incorrectly in the actual metadata + if new_nd2.name == "jonas_control002.nd2": + assert nd.attributes.widthPx == 248 + assert nd.shape == (65, 9, 152, 248) + assert nd.sizes["X"] == 248 + # TODO: deal with typing when metadata is completely missing assert isinstance(nd.metadata, structures.Metadata) assert isinstance(nd.frame_metadata(0), structures.FrameMetadata) @@ -41,7 +58,7 @@ def test_metadata_extraction(new_nd2): def test_read_safety(new_nd2: Path): - with ND2File(new_nd2) as nd: + with _warning_ctx(new_nd2), ND2File(new_nd2) as nd: for i in range(nd._frame_count): nd._rdr._read_image(i) @@ -50,7 +67,7 @@ def test_position(new_nd2): """use position to extract a single stage position with asarray.""" if new_nd2.stat().st_size > 250_000_000: pytest.skip("skipping read on big files") - with ND2File(new_nd2) as nd: + with _warning_ctx(new_nd2), ND2File(new_nd2) as nd: dx = nd.to_xarray(delayed=True, position=0, squeeze=False) nx = nd.to_xarray(delayed=False, position=0, squeeze=False) assert dx.sizes[AXIS.POSITION] == 1 @@ -62,7 +79,7 @@ def test_position(new_nd2): def test_dask(new_nd2): - with ND2File(new_nd2) as nd: + with _warning_ctx(new_nd2), ND2File(new_nd2) as nd: dsk = nd.to_dask() assert isinstance(dsk, da.Array) assert dsk.shape == nd.shape @@ -79,7 +96,7 @@ def test_dask_closed(single_nd2): @pytest.mark.skipif(bool(os.getenv("CIBUILDWHEEL")), reason="slow") def test_full_read(new_nd2): - with ND2File(new_nd2) as nd: + with _warning_ctx(new_nd2), ND2File(new_nd2) as nd: if new_nd2.stat().st_size > 500_000_000: pytest.skip("skipping full read on big files") delayed_xarray: np.ndarray = np.asarray(nd.to_xarray(delayed=True)) @@ -109,7 +126,7 @@ def test_full_read_legacy(old_nd2): def test_xarray(new_nd2): - with ND2File(new_nd2) as nd: + with _warning_ctx(new_nd2), ND2File(new_nd2) as nd: xarr = nd.to_xarray() assert isinstance(xarr, xr.DataArray) assert isinstance(xarr.data, da.Array) @@ -176,7 +193,7 @@ def test_bioformats_parity(new_nd2: Path, bfshapes: dict): bf_info = {k: v for k, v in bfshapes[new_nd2.name]["shape"].items() if v > 1} except KeyError: pytest.skip(f"{new_nd2.name} not in stats") - with ND2File(new_nd2) as nd: + with _warning_ctx(new_nd2), ND2File(new_nd2) as nd: # doing these weird checks/asserts for better error messages if len(bf_info) != len(nd.sizes): assert bf_info == nd.sizes diff --git a/tests/test_sdk.py b/tests/test_sdk.py index c2d3582..342038e 100644 --- a/tests/test_sdk.py +++ b/tests/test_sdk.py @@ -1,3 +1,4 @@ +from contextlib import nullcontext from pathlib import Path import numpy as np @@ -6,8 +7,18 @@ from nd2._sdk import latest +# duplicated in test_reader +def _warning_ctx(fname: Path): + if fname.name in { + "jonas_control002.nd2", + "jonas_JJ1473_control_24h_JJ1473_control_24h_03.nd2", + }: + return pytest.warns(UserWarning, match="widthPx") + return nullcontext() + + def test_new_sdk(new_nd2: Path): - with latest.ND2Reader(new_nd2, read_using_sdk=True) as nd: + with _warning_ctx(new_nd2), latest.ND2Reader(new_nd2, read_using_sdk=True) as nd: a = nd._attributes() assert isinstance(a, dict) assert isinstance(nd._metadata(), dict)