From 3c4f62f9d7dee511ae96e211845214fb1a6fc5c6 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 31 Jan 2023 08:39:23 -0800 Subject: [PATCH 1/4] deprecate open_zarr --- doc/api.rst | 2 +- doc/user-guide/io.rst | 6 ++--- doc/whats-new.rst | 4 ++- xarray/backends/zarr.py | 12 +++++++++ xarray/tests/test_backends.py | 46 ++++++++++++++++++++++++----------- 5 files changed, 51 insertions(+), 19 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 0d56fc73997..8c2885558fb 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -596,7 +596,6 @@ Dataset methods open_dataset open_mfdataset open_rasterio - open_zarr save_mfdataset Dataset.as_numpy Dataset.from_dataframe @@ -1139,6 +1138,7 @@ Deprecated / Pending Deprecation .. autosummary:: :toctree: generated/ + open_zarr Dataset.drop DataArray.drop Dataset.apply diff --git a/doc/user-guide/io.rst b/doc/user-guide/io.rst index 9e5d7ad71a3..47e1b4cf44a 100644 --- a/doc/user-guide/io.rst +++ b/doc/user-guide/io.rst @@ -626,11 +626,11 @@ To store variable length strings, convert them to object arrays first with ``dtype=object``. To read back a zarr dataset that has been created this way, we use the -:py:func:`open_zarr` method: +:py:func:`open_dataset` function: .. ipython:: python - ds_zarr = xr.open_zarr("path/to/directory.zarr") + ds_zarr = xr.open_dataset("path/to/directory.zarr", engine="zarr") ds_zarr Cloud Storage Buckets @@ -671,7 +671,7 @@ instance and pass this, as follows: # write to the bucket ds.to_zarr(store=gcsmap) # read it back - ds_gcs = xr.open_zarr(gcsmap) + ds_gcs = xr.open_dataset(gcsmap, engine="zarr") (or use the utility function ``fsspec.get_mapper()``). diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 26bd72b0727..2912983729b 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -45,7 +45,9 @@ Breaking changes Deprecations ~~~~~~~~~~~~ - +- The `open_zarr` function has been deprecated in favor of `open_dataset(..., engine='zarr')`. + (:issue:`7495`, :pull:`7496`). + By `Joe Hamman `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index e30e7e9f4d8..654ec66ade0 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -787,17 +787,29 @@ def open_zarr( dataset : Dataset The newly created dataset. + .. deprecated:: 1.6.0 + `ndobj_old` will be removed in NumPy 2.0.0, it is replaced by + `ndobj_new` because the latter works also with array subclasses. + See Also -------- open_dataset open_mfdataset + + References ---------- http://zarr.readthedocs.io/ """ from xarray.backends.api import open_dataset + warnings.warn( + "open_zarr is Deprecated in favor of open_dataset(store, ..., engine='zarr')" + "See https://github.com/pydata/xarray/issues/7495 for more information", + DeprecationWarning, + ) + if chunks == "auto": try: import dask.array # noqa diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index b49500bd00c..0fbce439246 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1812,12 +1812,12 @@ def test_read_non_consolidated_warning(self) -> None: RuntimeWarning, match="Failed to open Zarr store with consolidated", ): - with xr.open_zarr(store, **self.version_kwargs) as ds: + with xr.open_dataset(store, engine="zarr", **self.version_kwargs) as ds: assert_identical(ds, expected) def test_non_existent_store(self) -> None: with pytest.raises(FileNotFoundError, match=r"No such file or directory:"): - xr.open_zarr(f"{uuid.uuid4()}") + xr.open_dataset(f"{uuid.uuid4()}", engine="zarr") def test_with_chunkstore(self) -> None: expected = create_test_data() @@ -2365,8 +2365,11 @@ def test_write_region(self, consolidated, compute, use_dask) -> None: **self.version_kwargs, ) if compute: - with xr.open_zarr( - store, consolidated=consolidated, **self.version_kwargs + with xr.open_dataset( + store, + consolidated=consolidated, + engine="zarr", + **self.version_kwargs, ) as actual: assert_identical(actual, zeros) for i in range(0, 10, 2): @@ -2377,8 +2380,8 @@ def test_write_region(self, consolidated, compute, use_dask) -> None: consolidated=consolidated, **self.version_kwargs, ) - with xr.open_zarr( - store, consolidated=consolidated, **self.version_kwargs + with xr.open_dataset( + store, consolidated=consolidated, engine="zarr", **self.version_kwargs ) as actual: assert_identical(actual, nonzeros) @@ -2392,7 +2395,7 @@ def test_write_region_mode(self, mode) -> None: nonzeros.isel(region).to_zarr( store, region=region, mode=mode, **self.version_kwargs ) - with xr.open_zarr(store, **self.version_kwargs) as actual: + with xr.open_dataset(store, engine="zarr", **self.version_kwargs) as actual: assert_identical(actual, nonzeros) @requires_dask @@ -2556,9 +2559,11 @@ def test_open_zarr_use_cftime(self) -> None: ds = create_test_data() with self.create_zarr_target() as store_target: ds.to_zarr(store_target, **self.version_kwargs) - ds_a = xr.open_zarr(store_target, **self.version_kwargs) + ds_a = xr.open_dataset(store_target, engine="zarr", **self.version_kwargs) assert_identical(ds, ds_a) - ds_b = xr.open_zarr(store_target, use_cftime=True, **self.version_kwargs) + ds_b = xr.open_dataset( + store_target, use_cftime=True, engine="zarr", **self.version_kwargs + ) assert xr.coding.times.contains_cftime_datetimes(ds_b.time) def test_write_read_select_write(self) -> None: @@ -2568,7 +2573,7 @@ def test_write_read_select_write(self) -> None: # NOTE: using self.roundtrip, which uses open_dataset, will not trigger the bug. with self.create_zarr_target() as initial_store: ds.to_zarr(initial_store, mode="w", **self.version_kwargs) - ds1 = xr.open_zarr(initial_store, **self.version_kwargs) + ds1 = xr.open_dataset(initial_store, engine="zarr", **self.version_kwargs) # Combination of where+squeeze triggers error on write. ds_sel = ds1.where(ds1.coords["dim3"] == "a", drop=True).squeeze("dim3") @@ -2583,7 +2588,9 @@ def test_attributes(self, obj) -> None: ds = obj if isinstance(obj, Dataset) else obj.to_dataset() with self.create_zarr_target() as store_target: ds.to_zarr(store_target, **self.version_kwargs) - assert_identical(ds, xr.open_zarr(store_target, **self.version_kwargs)) + assert_identical( + ds, xr.open_dataset(store_target, engine="zarr", **self.version_kwargs) + ) obj.attrs["bad"] = DataArray() ds = obj if isinstance(obj, Dataset) else obj.to_dataset() @@ -2666,7 +2673,9 @@ def test_zarr_storage_options() -> None: ds = create_test_data() store_target = "memory://test.zarr" ds.to_zarr(store_target, storage_options={"test": "zarr_write"}) - ds_a = xr.open_zarr(store_target, storage_options={"test": "zarr_read"}) + ds_a = xr.open_dataset( + store_target, engine="zarr", storage_options={"test": "zarr_read"} + ) assert_identical(ds, ds_a) @@ -5696,7 +5705,7 @@ def _create_nczarr(filename): def test_open_nczarr(self) -> None: with create_tmp_file(suffix=".zarr") as tmp: expected = self._create_nczarr(tmp) - actual = xr.open_zarr(tmp, consolidated=False) + actual = xr.open_dataset(tmp, engine="zarr", consolidated=False) assert_identical(expected, actual) def test_overwriting_nczarr(self) -> None: @@ -5704,7 +5713,7 @@ def test_overwriting_nczarr(self) -> None: ds = self._create_nczarr(tmp) expected = ds[["var1"]] expected.to_zarr(tmp, mode="w") - actual = xr.open_zarr(tmp, consolidated=False) + actual = xr.open_dataset(tmp, engine="zarr", consolidated=False) assert_identical(expected, actual) @pytest.mark.parametrize("mode", ["a", "r+"]) @@ -5723,3 +5732,12 @@ def test_raise_writing_to_nczarr(self, mode) -> None: def test_pickle_open_mfdataset_dataset(): ds = open_example_mfdataset(["bears.nc"]) assert_identical(ds, pickle.loads(pickle.dumps(ds))) + + +@requires_zarr +def test_open_zarr_deprecation_warns(): + ds = create_test_data() + with create_tmp_file(suffix=".zarr") as tmp: + ds.to_zarr(tmp, mode="w") + with pytest.warns(DeprecationWarning): + xr.open_zarr(tmp) From f0c456f11a09eeeda74a2edde5ddce1adc7ca1b5 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 31 Jan 2023 09:06:14 -0800 Subject: [PATCH 2/4] Update xarray/backends/zarr.py --- xarray/backends/zarr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 654ec66ade0..9145986095d 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -787,9 +787,9 @@ def open_zarr( dataset : Dataset The newly created dataset. - .. deprecated:: 1.6.0 - `ndobj_old` will be removed in NumPy 2.0.0, it is replaced by - `ndobj_new` because the latter works also with array subclasses. + .. deprecated:: v2023.02.0 + `open_zarr` will be removed in Xarray v2023.06.0, please use + `open_dataset(..., engine='zarr')` from now on. See Also -------- From 3d9c43cb824e2c6baf882607c8d8ce24281429c9 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 31 Jan 2023 14:00:10 -0800 Subject: [PATCH 3/4] Update xarray/backends/zarr.py Co-authored-by: Illviljan <14371165+Illviljan@users.noreply.github.com> --- xarray/backends/zarr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 9145986095d..03de77f9e5e 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -805,7 +805,7 @@ def open_zarr( from xarray.backends.api import open_dataset warnings.warn( - "open_zarr is Deprecated in favor of open_dataset(store, ..., engine='zarr')" + "open_zarr is Deprecated in favor of open_dataset(..., engine='zarr', chunks={})" "See https://github.com/pydata/xarray/issues/7495 for more information", DeprecationWarning, ) From 0a377da3c54e306a264b35aa514e0bf9327c79ea Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 31 Jan 2023 14:00:17 -0800 Subject: [PATCH 4/4] Update xarray/backends/zarr.py Co-authored-by: Illviljan <14371165+Illviljan@users.noreply.github.com> --- xarray/backends/zarr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 03de77f9e5e..83eaa98d139 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -789,7 +789,7 @@ def open_zarr( .. deprecated:: v2023.02.0 `open_zarr` will be removed in Xarray v2023.06.0, please use - `open_dataset(..., engine='zarr')` from now on. + `open_dataset(..., engine='zarr', chunks={})` from now on. See Also --------