From b8f646810bd5418895f0d2f95db774a9af52d4ba Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 30 Apr 2020 13:10:29 +0100 Subject: [PATCH 1/4] Test attrs handling in open_mfdataset --- xarray/tests/test_backends.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 177435fa864..bc5e4e32275 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2662,6 +2662,36 @@ def test_open_mfdataset_does_same_as_concat(self, combine, opt, join): ds_expect = xr.concat([ds1, ds2], data_vars=opt, dim="t", join=join) assert_identical(ds, ds_expect) + def test_open_mfdataset_dataset_attr_by_coords(self): + """ + Case when an attribute of type list differs across the multiple files + """ + with self.setup_files_and_datasets() as (files, [ds1, ds2]): + # Give the files an inconsistent attribute + for i, f in enumerate(files): + ds = open_dataset(f).load() + ds.attrs["test_dataset_attr"] = 10 + i + ds.close() + ds.to_netcdf(f) + + with xr.open_mfdataset(files, combine="by_coords", concat_dim="t") as ds: + assert ds.test_dataset_attr == 10 + + def test_open_mfdataset_dataarray_attr_by_coords(self): + """ + Case when an attribute of type list differs across the multiple files + """ + with self.setup_files_and_datasets() as (files, [ds1, ds2]): + # Give the files an inconsistent attribute + for i, f in enumerate(files): + ds = open_dataset(f).load() + ds["v1"].attrs["test_dataarray_attr"] = i + ds.close() + ds.to_netcdf(f) + + with xr.open_mfdataset(files, combine="by_coords", concat_dim="t") as ds: + assert ds["v1"].test_dataarray_attr == 0 + @pytest.mark.parametrize("combine", ["nested", "by_coords"]) @pytest.mark.parametrize("opt", ["all", "minimal", "different"]) def test_open_mfdataset_exact_join_raises_error(self, combine, opt): From d46e305218ec0edd2353eec863f8b3553559d2f4 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 7 May 2020 23:24:08 +0100 Subject: [PATCH 2/4] Fix attrs handling in open_mfdataset() Need to pass combine_attrs="drop", to allow attrs_file to set the attrs. --- xarray/backends/api.py | 8 +++++++- xarray/tests/test_backends.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 0919d2a582b..4077d7a02c8 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -967,12 +967,18 @@ def open_mfdataset( coords=coords, ids=ids, join=join, + combine_attrs="drop", ) elif combine == "by_coords": # Redo ordering from coordinates, ignoring how they were ordered # previously combined = combine_by_coords( - datasets, compat=compat, data_vars=data_vars, coords=coords, join=join + datasets, + compat=compat, + data_vars=data_vars, + coords=coords, + join=join, + combine_attrs="drop", ) else: raise ValueError( diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index bc5e4e32275..1e33eccb83e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2664,7 +2664,7 @@ def test_open_mfdataset_does_same_as_concat(self, combine, opt, join): def test_open_mfdataset_dataset_attr_by_coords(self): """ - Case when an attribute of type list differs across the multiple files + Case when an attribute differs across the multiple files """ with self.setup_files_and_datasets() as (files, [ds1, ds2]): # Give the files an inconsistent attribute @@ -2679,7 +2679,7 @@ def test_open_mfdataset_dataset_attr_by_coords(self): def test_open_mfdataset_dataarray_attr_by_coords(self): """ - Case when an attribute of type list differs across the multiple files + Case when an attribute of a member DataArray differs across the multiple files """ with self.setup_files_and_datasets() as (files, [ds1, ds2]): # Give the files an inconsistent attribute From c2b0c062c02013d090263dd44c98f386bcc8f8a3 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 24 Jun 2020 10:59:50 +0100 Subject: [PATCH 3/4] Update whats-new.rst --- doc/whats-new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4b5bb1e491f..b92a48a6868 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -105,6 +105,8 @@ New Features Bug fixes ~~~~~~~~~ +- Errors combining attrs in open_mfdataset (:issue:`4009`, :pull:`4173`) + By `John Omotani `_ - If groupby receives a ``DataArray`` with name=None, assign a default name (:issue:`158`) By `Phil Butcher `_. - Support dark mode in VS code (:issue:`4024`) From 4faa2f9677f89ce80e73ec98f2e3f39d08947997 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 24 Jun 2020 15:41:45 +0000 Subject: [PATCH 4/4] Update doc/whats-new.rst --- doc/whats-new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b92a48a6868..6a6934e25c5 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -105,7 +105,7 @@ New Features Bug fixes ~~~~~~~~~ -- Errors combining attrs in open_mfdataset (:issue:`4009`, :pull:`4173`) +- Fix errors combining attrs in :py:func:`open_mfdataset` (:issue:`4009`, :pull:`4173`) By `John Omotani `_ - If groupby receives a ``DataArray`` with name=None, assign a default name (:issue:`158`) By `Phil Butcher `_.