From 15f1198481fdcfed52ef95ebc195b8a50586b6ee Mon Sep 17 00:00:00 2001 From: zxdawn Date: Mon, 13 Apr 2020 14:46:13 +0800 Subject: [PATCH 1/6] Check ancillary_variables in exclude_attrs --- satpy/writers/cf_writer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/satpy/writers/cf_writer.py b/satpy/writers/cf_writer.py index 10ae9d8c69..3379728b75 100644 --- a/satpy/writers/cf_writer.py +++ b/satpy/writers/cf_writer.py @@ -484,8 +484,9 @@ def _collect_datasets(self, datasets, epoch=EPOCH, flatten_attrs=False, exclude_ pretty=False, compression=None): """Collect and prepare datasets to be written.""" ds_collection = {} - for ds in datasets: - ds_collection.update(get_extra_ds(ds)) + if 'ancillary_variables' not in exclude_attrs: + for ds in datasets: + ds_collection.update(get_extra_ds(ds)) datas = {} start_times = [] From cd9bcda0a26862f6376eea8af7ac92e6edc1b39e Mon Sep 17 00:00:00 2001 From: zxdawn Date: Mon, 13 Apr 2020 15:22:50 +0800 Subject: [PATCH 2/6] Bugfix: add the option of extra --- satpy/writers/cf_writer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/satpy/writers/cf_writer.py b/satpy/writers/cf_writer.py index 3379728b75..5d5598c828 100644 --- a/satpy/writers/cf_writer.py +++ b/satpy/writers/cf_writer.py @@ -153,11 +153,12 @@ def create_grid_mapping(area): return area.area_id, grid_mapping -def get_extra_ds(dataset): +def get_extra_ds(dataset, extra): """Get the extra datasets associated to *dataset*.""" ds_collection = {} - for ds in dataset.attrs.get('ancillary_variables', []): - ds_collection.update(get_extra_ds(ds)) + if extra: + for ds in dataset.attrs.get('ancillary_variables', []): + ds_collection.update(get_extra_ds(ds)) ds_collection[dataset.attrs['name']] = dataset return ds_collection @@ -484,9 +485,12 @@ def _collect_datasets(self, datasets, epoch=EPOCH, flatten_attrs=False, exclude_ pretty=False, compression=None): """Collect and prepare datasets to be written.""" ds_collection = {} - if 'ancillary_variables' not in exclude_attrs: - for ds in datasets: - ds_collection.update(get_extra_ds(ds)) + if 'ancillary_variables' in exclude_attrs: + extra = False + else: + extra = True + for i,ds in enumerate(datasets): + ds_collection.update(get_extra_ds(ds, extra)) datas = {} start_times = [] From 4ec6a87276471603f97db460b117a916bdd51020 Mon Sep 17 00:00:00 2001 From: zxdawn Date: Mon, 13 Apr 2020 15:29:03 +0800 Subject: [PATCH 3/6] Delete own test --- satpy/writers/cf_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/writers/cf_writer.py b/satpy/writers/cf_writer.py index 5d5598c828..5d36077618 100644 --- a/satpy/writers/cf_writer.py +++ b/satpy/writers/cf_writer.py @@ -489,7 +489,7 @@ def _collect_datasets(self, datasets, epoch=EPOCH, flatten_attrs=False, exclude_ extra = False else: extra = True - for i,ds in enumerate(datasets): + for ds in datasets: ds_collection.update(get_extra_ds(ds, extra)) datas = {} From f9fffd5609d962f72ed7a0ad9a21a7cf5454495d Mon Sep 17 00:00:00 2001 From: zxdawn Date: Tue, 14 Apr 2020 10:20:57 +0800 Subject: [PATCH 4/6] Check whether ancillary_variables has been processed --- satpy/writers/cf_writer.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/satpy/writers/cf_writer.py b/satpy/writers/cf_writer.py index 5d36077618..8ca2cce76b 100644 --- a/satpy/writers/cf_writer.py +++ b/satpy/writers/cf_writer.py @@ -153,12 +153,13 @@ def create_grid_mapping(area): return area.area_id, grid_mapping -def get_extra_ds(dataset, extra): +def get_extra_ds(dataset, keys=None): """Get the extra datasets associated to *dataset*.""" ds_collection = {} - if extra: - for ds in dataset.attrs.get('ancillary_variables', []): - ds_collection.update(get_extra_ds(ds)) + for ds in dataset.attrs.get('ancillary_variables', []): + if keys and ds.name not in keys: + keys.append(ds.name) + ds_collection.update(get_extra_ds(ds, keys)) ds_collection[dataset.attrs['name']] = dataset return ds_collection @@ -485,12 +486,8 @@ def _collect_datasets(self, datasets, epoch=EPOCH, flatten_attrs=False, exclude_ pretty=False, compression=None): """Collect and prepare datasets to be written.""" ds_collection = {} - if 'ancillary_variables' in exclude_attrs: - extra = False - else: - extra = True for ds in datasets: - ds_collection.update(get_extra_ds(ds, extra)) + ds_collection.update(get_extra_ds(ds)) datas = {} start_times = [] From 8c2cab8befd7477f9acbfb86eccc23de9b1251af Mon Sep 17 00:00:00 2001 From: zxdawn Date: Thu, 4 Jun 2020 15:04:19 +0800 Subject: [PATCH 5/6] Add test_ancillary_variables --- satpy/tests/writer_tests/test_cf.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index 001d18714f..e8addbf7fd 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -139,6 +139,29 @@ def test_save_array_coords(self): self.assertEqual(f['test-array'].attrs['prerequisites'], expected_prereq) + def test_ancillary_variables(self): + '''Test ancillary_variables cited each other''' + import xarray as xr + from satpy import Scene + scn = Scene() + start_time = datetime(2018, 5, 30, 10, 0) + end_time = datetime(2018, 5, 30, 10, 15) + da = xr.DataArray([1, 2, 3], + attrs=dict(start_time=start_time, + end_time=end_time, + prerequisites=[DatasetID('hej')])) + scn['test-array-1'] = da + scn['test-array-2'] = da.copy() + scn['test-array-1'].attrs['ancillary_variables'] = [scn['test-array-2']] + scn['test-array-2'].attrs['ancillary_variables'] = [scn['test-array-1']] + with TempFile() as filename: + scn.save_datasets(filename=filename, writer='cf') + with xr.open_dataset(filename) as f: + self.assertEqual(f['test-array-1'].attrs['ancillary_variables'], + 'test-array-2') + self.assertEqual(f['test-array-2'].attrs['ancillary_variables'], + 'test-array-1') + def test_groups(self): """Test creating a file with groups.""" import xarray as xr From 195fb1165e508bec4d4ecae28846503795130b91 Mon Sep 17 00:00:00 2001 From: zxdawn Date: Thu, 4 Jun 2020 20:39:04 +0800 Subject: [PATCH 6/6] Better docstring --- satpy/tests/writer_tests/test_cf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index e8addbf7fd..162a3e0bcb 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -140,7 +140,7 @@ def test_save_array_coords(self): expected_prereq) def test_ancillary_variables(self): - '''Test ancillary_variables cited each other''' + """Test ancillary_variables cited each other.""" import xarray as xr from satpy import Scene scn = Scene()