From eb81350589b997e16fad949b25e3022dcf97dd71 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 2 Oct 2024 13:51:19 -0500 Subject: [PATCH] Fix real reflectance products not showing in product list --- integration_tests/features/polar2grid.feature | 1 + polar2grid/readers/_base.py | 4 +- polar2grid/readers/avhrr_l1b_aapp.py | 6 +- polar2grid/tests/_avhrr_fixtures.py | 138 +++++++++++++++++- polar2grid/tests/test_glue.py | 16 ++ polar2grid/utils/legacy_compat.py | 22 ++- 6 files changed, 175 insertions(+), 12 deletions(-) diff --git a/integration_tests/features/polar2grid.feature b/integration_tests/features/polar2grid.feature index d6825451..5f48df71 100644 --- a/integration_tests/features/polar2grid.feature +++ b/integration_tests/features/polar2grid.feature @@ -59,3 +59,4 @@ Feature: Test polar2grid output images | command | source | output | | polar2grid.sh -r viirs_sdr -w geotiff -vv -f | viirs_sdr_day/input/test1 | true_color,i01,i01_rad,ifog | | polar2grid.sh -r viirs_l1b -w geotiff -vv -f | viirs_l1b_night/input/test1 | adaptive_dnb,m12,i04,ifog | + | polar2grid.sh -r avhrr_l1b -w geotiff -vv -f | avhrr/input/test1 | band1_vis,band2_vis,band3a_vis,band4_bt,band5_bt | diff --git a/polar2grid/readers/_base.py b/polar2grid/readers/_base.py index 6359ecb2..3f19b559 100644 --- a/polar2grid/readers/_base.py +++ b/polar2grid/readers/_base.py @@ -97,7 +97,7 @@ def get_available_products( self, p2g_product_names: Optional[list[str]] = None, possible_satpy_ids: Optional[list[DataID]] = None, - ) -> tuple[list[str], list[str]]: + ) -> tuple[list[str], list[str], list[str]]: """Get custom/satpy products and polar2grid products that are available for loading.""" if possible_satpy_ids is None: possible_satpy_ids = self.scn.available_dataset_ids(composites=True) @@ -110,7 +110,7 @@ def get_available_products( "products will be listed with internal Satpy names.", self._binary_name, ) - return sorted(set([x["name"] for x in possible_satpy_ids])), [] + return sorted(set([x["name"] for x in possible_satpy_ids])), [], [] return self._alias_handler.available_product_names( p2g_product_names, available_custom_products, possible_satpy_ids ) diff --git a/polar2grid/readers/avhrr_l1b_aapp.py b/polar2grid/readers/avhrr_l1b_aapp.py index bf98d2e0..47edf6d5 100644 --- a/polar2grid/readers/avhrr_l1b_aapp.py +++ b/polar2grid/readers/avhrr_l1b_aapp.py @@ -113,12 +113,12 @@ def __init__(self, scn: Scene, user_products: list[str]): for chan_name in ["1", "2", "3a"]: if modifiers: logger.debug(f"Using visible channel modifiers: {modifiers}") - self._modified_aliases[chan_name] = DataQuery( - name=chan_name, calibration="reflectance", modifiers=modifiers - ) self._modified_aliases[f"band{chan_name}_vis"] = DataQuery( name=chan_name, calibration="reflectance", modifiers=modifiers ) + # self._modified_aliases[chan_name] = DataQuery( + # name=chan_name, calibration="reflectance", modifiers=modifiers + # ) super().__init__(scn, user_products) def get_default_products(self) -> list[str]: diff --git a/polar2grid/tests/_avhrr_fixtures.py b/polar2grid/tests/_avhrr_fixtures.py index 2691de7a..70c4cd07 100644 --- a/polar2grid/tests/_avhrr_fixtures.py +++ b/polar2grid/tests/_avhrr_fixtures.py @@ -38,7 +38,42 @@ AVHRR_CHUNKS = AVHRR_SHAPE AVHRR_IDS = [ - make_dataid(name="1", wavelength=(0.58, 0.63, 0.68), resolution=1050, calibration="reflectance"), + make_dataid( + name="1", + wavelength=(0.58, 0.63, 0.68), + resolution=1050, + calibration="reflectance", + ), + make_dataid( + name="2", + wavelength=(0.725, 0.8625, 1.0), + resolution=1050, + calibration="reflectance", + ), + make_dataid( + name="3a", + wavelength=(1.58, 1.61, 1.64), + resolution=1050, + calibration="reflectance", + ), + make_dataid( + name="3b", + wavelength=(3.55, 3.74, 3.93), + resolution=1050, + calibration="brightness_temperature", + ), + make_dataid( + name="4", + wavelength=(10.3, 10.8, 11.3), + resolution=1050, + calibration="brightness_temperature", + ), + make_dataid( + name="5", + wavelength=(11.5, 12.0, 12.5), + resolution=1050, + calibration="brightness_temperature", + ), ] @@ -94,14 +129,111 @@ def avhrr_l1b_1_data_array(avhrr_l1b_swath_def) -> xr.DataArray: @pytest.fixture -def avhrr_l1b_1_scene(avhrr_l1b_1_data_array) -> Scene: +def avhrr_l1b_2_data_array(avhrr_l1b_swath_def) -> xr.DataArray: + return xr.DataArray( + da.zeros(AVHRR_SHAPE, dtype=np.float32), + dims=("y", "x"), + attrs={ + "area": avhrr_l1b_swath_def, + "platform_name": "noaa19", + "sensor": "avhrr-3", + "name": "2", + "start_time": START_TIME, + "end_time": START_TIME, + "resolution": 1050, + "reader": "avhrr_l1b_aapp", + "calibration": "reflectance", + "standard_name": "toa_bidirectional_reflectance", + "units": "%", + }, + ) + + +@pytest.fixture +def avhrr_l1b_3a_data_array(avhrr_l1b_swath_def) -> xr.DataArray: + return xr.DataArray( + da.zeros(AVHRR_SHAPE, dtype=np.float32), + dims=("y", "x"), + attrs={ + "area": avhrr_l1b_swath_def, + "platform_name": "noaa19", + "sensor": "avhrr-3", + "name": "3a", + "start_time": START_TIME, + "end_time": START_TIME, + "resolution": 1050, + "reader": "avhrr_l1b_aapp", + "calibration": "reflectance", + "standard_name": "toa_bidirectional_reflectance", + "units": "%", + }, + ) + + +# No 3b. Assume day time scene + + +@pytest.fixture +def avhrr_l1b_4_data_array(avhrr_l1b_swath_def) -> xr.DataArray: + return xr.DataArray( + da.zeros(AVHRR_SHAPE, dtype=np.float32), + dims=("y", "x"), + attrs={ + "area": avhrr_l1b_swath_def, + "platform_name": "noaa19", + "sensor": "avhrr-3", + "name": "4", + "start_time": START_TIME, + "end_time": START_TIME, + "resolution": 1050, + "reader": "avhrr_l1b_aapp", + "calibration": "brightness_temperature", + "standard_name": "toa_brightness_temperature", + "units": "K", + }, + ) + + +@pytest.fixture +def avhrr_l1b_5_data_array(avhrr_l1b_swath_def) -> xr.DataArray: + return xr.DataArray( + da.zeros(AVHRR_SHAPE, dtype=np.float32), + dims=("y", "x"), + attrs={ + "area": avhrr_l1b_swath_def, + "platform_name": "noaa19", + "sensor": "avhrr-3", + "name": "5", + "start_time": START_TIME, + "end_time": START_TIME, + "resolution": 1050, + "reader": "avhrr_l1b_aapp", + "calibration": "brightness_temperature", + "standard_name": "toa_brightness_temperature", + "units": "K", + }, + ) + + +@pytest.fixture +def avhrr_l1b_1_scene( + avhrr_l1b_1_data_array, + avhrr_l1b_2_data_array, + avhrr_l1b_3a_data_array, + avhrr_l1b_4_data_array, + avhrr_l1b_5_data_array, +) -> Scene: scn = _TestingScene( reader="avhrr_l1b_aapp", filenames=["/fake/filename"], data_array_dict={ AVHRR_IDS[0]: avhrr_l1b_1_data_array.copy(), + AVHRR_IDS[1]: avhrr_l1b_2_data_array.copy(), + AVHRR_IDS[2]: avhrr_l1b_3a_data_array.copy(), + AVHRR_IDS[4]: avhrr_l1b_4_data_array.copy(), + AVHRR_IDS[5]: avhrr_l1b_5_data_array.copy(), }, all_dataset_ids=AVHRR_IDS, - available_dataset_ids=AVHRR_IDS[:1], + available_dataset_ids=AVHRR_IDS[:3] + AVHRR_IDS[4:], ) return scn diff --git a/polar2grid/tests/test_glue.py b/polar2grid/tests/test_glue.py index f09ebb94..80d666d5 100644 --- a/polar2grid/tests/test_glue.py +++ b/polar2grid/tests/test_glue.py @@ -346,3 +346,19 @@ def test_extra_config_path( extra_cpath = gettempdir() path_idx = captured.err.index(f"Adding enhancement configuration from file: {str(extra_cpath)}") assert builtin_path_idx < path_idx + + def test_avhrr_list_products(self, avhrr_l1b_1_scene, chtmpdir, capsys): + """Test list products includes expected products.""" + from polar2grid.glue import main + + with prepare_glue_exec(avhrr_l1b_1_scene, max_computes=0): + args = ["-r", "avhrr_l1b_aapp", "-w", "geotiff", "--list-products", "-f", str(chtmpdir)] + ret = main(args) + output_files = glob(str(chtmpdir / "*.tif")) + assert len(output_files) == 0 + assert ret == 0 + captured = capsys.readouterr() + stdout = captured.out + print(stdout) + for exp_product in ("band1_vis", "band2_vis", "band3a_vis", "band4_bt", "band5_bt"): + assert exp_product in stdout diff --git a/polar2grid/utils/legacy_compat.py b/polar2grid/utils/legacy_compat.py index 2cd920bb..3c1f1e6c 100644 --- a/polar2grid/utils/legacy_compat.py +++ b/polar2grid/utils/legacy_compat.py @@ -27,7 +27,7 @@ import logging from typing import Generator, Iterable, Optional, Union -from satpy import DataID, DataQuery, Scene +from satpy import DataID, DataQuery, Scene, DatasetDict logger = logging.getLogger(__name__) @@ -173,9 +173,9 @@ def convert_satpy_to_p2g_name( satpy_id_to_p2g_name = {} for p2g_name in possible_p2g_names: satpy_data_query = self._all_aliases.get(p2g_name, p2g_name) - try: - matching_satpy_id = satpy_id_dict[satpy_data_query] - except KeyError: + matching_satpy_id = _get_matching_satpy_id(satpy_id_dict, satpy_data_query) + if matching_satpy_id is None: + # no match continue if matching_satpy_id in satpy_id_to_p2g_name: @@ -245,6 +245,20 @@ def available_product_names( return available_p2g_names, available_custom_names, available_satpy_names +def _get_matching_satpy_id(satpy_id_dict: DatasetDict, satpy_data_query: DataQuery | str) -> DataID: + matching_satpy_id = None + while matching_satpy_id is None: + # iterate until no modifiers exist in the query and we still can't find it + try: + matching_satpy_id = satpy_id_dict[satpy_data_query] + return matching_satpy_id + except KeyError: + if isinstance(satpy_data_query, str) or not satpy_data_query.is_modified(): + break + satpy_data_query = satpy_data_query.create_less_modified_query() + return matching_satpy_id + + _SENSOR_ALIASES = { "avhrr-3": "avhrr", }