diff --git a/cubed/storage/virtual.py b/cubed/storage/virtual.py index b510f8e1e..2b3bf4892 100644 --- a/cubed/storage/virtual.py +++ b/cubed/storage/virtual.py @@ -8,6 +8,7 @@ from cubed.backend_array_api import namespace as nxp from cubed.backend_array_api import numpy_array_to_backend_array from cubed.types import T_DType, T_RegularChunks, T_Shape +from cubed.utils import memory_repr class VirtualEmptyArray: @@ -104,7 +105,12 @@ def __init__( self, array: np.ndarray, # TODO: generalise chunks: T_RegularChunks, + max_nbytes: int = 10**6, ): + if array.nbytes > max_nbytes: + raise ValueError( + f"Size of in memory array is {memory_repr(array.nbytes)} which exceeds maximum of {memory_repr(max_nbytes)}. Consider loading the array from storage using `from_array`." + ) self.array = array # use an in-memory Zarr array as a template since it normalizes its properties # and is needed for oindex diff --git a/cubed/tests/test_core.py b/cubed/tests/test_core.py index 8518c8e77..cc98566fb 100644 --- a/cubed/tests/test_core.py +++ b/cubed/tests/test_core.py @@ -40,6 +40,15 @@ def modal_executor(request): return request.param +def test_as_array_fails(spec): + a = np.ones((1000, 1000)) + with pytest.raises( + ValueError, + match="Size of in memory array is 8.0 MB which exceeds maximum of 1.0 MB.", + ): + xp.asarray(a, chunks=(100, 100), spec=spec) + + def test_regular_chunks(spec): xp.ones((5, 5), chunks=((2, 2, 1), (5,)), spec=spec) with pytest.raises(ValueError):