Skip to content

Commit

Permalink
Fail if array created with asarray is greater than 1MB
Browse files Browse the repository at this point in the history
  • Loading branch information
tomwhite committed Dec 17, 2023
1 parent 1d55905 commit 91d572f
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 0 deletions.
6 changes: 6 additions & 0 deletions cubed/storage/virtual.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from cubed.backend_array_api import namespace as nxp
from cubed.backend_array_api import numpy_array_to_backend_array
from cubed.types import T_DType, T_RegularChunks, T_Shape
from cubed.utils import memory_repr


class VirtualEmptyArray:
Expand Down Expand Up @@ -104,7 +105,12 @@ def __init__(
self,
array: np.ndarray, # TODO: generalise
chunks: T_RegularChunks,
max_nbytes: int = 10**6,
):
if array.nbytes > max_nbytes:
raise ValueError(
f"Size of in memory array is {memory_repr(array.nbytes)} which exceeds maximum of {memory_repr(max_nbytes)}. Consider loading the array from storage using `from_array`."
)
self.array = array
# use an in-memory Zarr array as a template since it normalizes its properties
# and is needed for oindex
Expand Down
9 changes: 9 additions & 0 deletions cubed/tests/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,15 @@ def modal_executor(request):
return request.param


def test_as_array_fails(spec):
a = np.ones((1000, 1000))
with pytest.raises(
ValueError,
match="Size of in memory array is 8.0 MB which exceeds maximum of 1.0 MB.",
):
xp.asarray(a, chunks=(100, 100), spec=spec)


def test_regular_chunks(spec):
xp.ones((5, 5), chunks=((2, 2, 1), (5,)), spec=spec)
with pytest.raises(ValueError):
Expand Down

0 comments on commit 91d572f

Please sign in to comment.