Fix real reflectance products not showing in product list

integration_tests/features/polar2grid.feature

@@ -59,3 +59,4 @@ Feature: Test polar2grid output images
       | command                                                                                      | source                      | output                       |
       | polar2grid.sh -r viirs_sdr -w geotiff -vv -f                                                 | viirs_sdr_day/input/test1   | true_color,i01,i01_rad,ifog  |
       | polar2grid.sh -r viirs_l1b -w geotiff -vv -f                                                 | viirs_l1b_night/input/test1 | adaptive_dnb,m12,i04,ifog      |
+      | polar2grid.sh -r avhrr_l1b -w geotiff -vv -f                                                 | avhrr/input/test1           | band1_vis,band2_vis,band3a_vis,band4_bt,band5_bt |

polar2grid/readers/_base.py

@@ -97,7 +97,7 @@ def get_available_products(
         self,
         p2g_product_names: Optional[list[str]] = None,
         possible_satpy_ids: Optional[list[DataID]] = None,
-    ) -> tuple[list[str], list[str]]:
+    ) -> tuple[list[str], list[str], list[str]]:
         """Get custom/satpy products and polar2grid products that are available for loading."""
         if possible_satpy_ids is None:
             possible_satpy_ids = self.scn.available_dataset_ids(composites=True)
@@ -110,7 +110,7 @@ def get_available_products(
                     "products will be listed with internal Satpy names.",
                     self._binary_name,
                 )
-                return sorted(set([x["name"] for x in possible_satpy_ids])), []
+                return sorted(set([x["name"] for x in possible_satpy_ids])), [], []
         return self._alias_handler.available_product_names(
             p2g_product_names, available_custom_products, possible_satpy_ids
         )

polar2grid/readers/avhrr_l1b_aapp.py

@@ -113,12 +113,12 @@ def __init__(self, scn: Scene, user_products: list[str]):
         for chan_name in ["1", "2", "3a"]:
             if modifiers:
                 logger.debug(f"Using visible channel modifiers: {modifiers}")
-            self._modified_aliases[chan_name] = DataQuery(
-                name=chan_name, calibration="reflectance", modifiers=modifiers
-            )
             self._modified_aliases[f"band{chan_name}_vis"] = DataQuery(
                 name=chan_name, calibration="reflectance", modifiers=modifiers
             )
+            # self._modified_aliases[chan_name] = DataQuery(
+            #     name=chan_name, calibration="reflectance", modifiers=modifiers
+            # )
         super().__init__(scn, user_products)
 
     def get_default_products(self) -> list[str]:

polar2grid/tests/_avhrr_fixtures.py

@@ -38,7 +38,42 @@
 AVHRR_CHUNKS = AVHRR_SHAPE
 
 AVHRR_IDS = [
-    make_dataid(name="1", wavelength=(0.58, 0.63, 0.68), resolution=1050, calibration="reflectance"),
+    make_dataid(
+        name="1",
+        wavelength=(0.58, 0.63, 0.68),
+        resolution=1050,
+        calibration="reflectance",
+    ),
+    make_dataid(
+        name="2",
+        wavelength=(0.725, 0.8625, 1.0),
+        resolution=1050,
+        calibration="reflectance",
+    ),
+    make_dataid(
+        name="3a",
+        wavelength=(1.58, 1.61, 1.64),
+        resolution=1050,
+        calibration="reflectance",
+    ),
+    make_dataid(
+        name="3b",
+        wavelength=(3.55, 3.74, 3.93),
+        resolution=1050,
+        calibration="brightness_temperature",
+    ),
+    make_dataid(
+        name="4",
+        wavelength=(10.3, 10.8, 11.3),
+        resolution=1050,
+        calibration="brightness_temperature",
+    ),
+    make_dataid(
+        name="5",
+        wavelength=(11.5, 12.0, 12.5),
+        resolution=1050,
+        calibration="brightness_temperature",
+    ),
 ]
 
 
@@ -94,14 +129,111 @@
 
 
 @pytest.fixture
-def avhrr_l1b_1_scene(avhrr_l1b_1_data_array) -> Scene:
+def avhrr_l1b_2_data_array(avhrr_l1b_swath_def) -> xr.DataArray:
+    return xr.DataArray(
+        da.zeros(AVHRR_SHAPE, dtype=np.float32),
+        dims=("y", "x"),
+        attrs={
+            "area": avhrr_l1b_swath_def,
+            "platform_name": "noaa19",
+            "sensor": "avhrr-3",
+            "name": "2",
+            "start_time": START_TIME,
+            "end_time": START_TIME,
+            "resolution": 1050,
+            "reader": "avhrr_l1b_aapp",
+            "calibration": "reflectance",
+            "standard_name": "toa_bidirectional_reflectance",
+            "units": "%",
+        },
+    )
+
+
+@pytest.fixture
+def avhrr_l1b_3a_data_array(avhrr_l1b_swath_def) -> xr.DataArray:
+    return xr.DataArray(
+        da.zeros(AVHRR_SHAPE, dtype=np.float32),
+        dims=("y", "x"),
+        attrs={
+            "area": avhrr_l1b_swath_def,
+            "platform_name": "noaa19",
+            "sensor": "avhrr-3",
+            "name": "3a",
+            "start_time": START_TIME,
+            "end_time": START_TIME,
+            "resolution": 1050,
+            "reader": "avhrr_l1b_aapp",
+            "calibration": "reflectance",
+            "standard_name": "toa_bidirectional_reflectance",
+            "units": "%",
+        },
+    )
+
+
+# No 3b. Assume day time scene
+
+
+@pytest.fixture
+def avhrr_l1b_4_data_array(avhrr_l1b_swath_def) -> xr.DataArray:
+    return xr.DataArray(
+        da.zeros(AVHRR_SHAPE, dtype=np.float32),
+        dims=("y", "x"),
+        attrs={
+            "area": avhrr_l1b_swath_def,
+            "platform_name": "noaa19",
+            "sensor": "avhrr-3",
+            "name": "4",
+            "start_time": START_TIME,
+            "end_time": START_TIME,
+            "resolution": 1050,
+            "reader": "avhrr_l1b_aapp",
+            "calibration": "brightness_temperature",
+            "standard_name": "toa_brightness_temperature",
+            "units": "K",
+        },
+    )
+
+
+@pytest.fixture
+def avhrr_l1b_5_data_array(avhrr_l1b_swath_def) -> xr.DataArray:
+    return xr.DataArray(
+        da.zeros(AVHRR_SHAPE, dtype=np.float32),
+        dims=("y", "x"),
+        attrs={
+            "area": avhrr_l1b_swath_def,
+            "platform_name": "noaa19",
+            "sensor": "avhrr-3",
+            "name": "5",
+            "start_time": START_TIME,
+            "end_time": START_TIME,
+            "resolution": 1050,
+            "reader": "avhrr_l1b_aapp",
+            "calibration": "brightness_temperature",
+            "standard_name": "toa_brightness_temperature",
+            "units": "K",
+        },
+    )
+
+
+@pytest.fixture
+def avhrr_l1b_1_scene(
+    avhrr_l1b_1_data_array,
+    avhrr_l1b_2_data_array,
+    avhrr_l1b_3a_data_array,
+    avhrr_l1b_4_data_array,
+    avhrr_l1b_5_data_array,
+) -> Scene:
     scn = _TestingScene(
         reader="avhrr_l1b_aapp",
         filenames=["/fake/filename"],
         data_array_dict={
             AVHRR_IDS[0]: avhrr_l1b_1_data_array.copy(),
+            AVHRR_IDS[1]: avhrr_l1b_2_data_array.copy(),
+            AVHRR_IDS[2]: avhrr_l1b_3a_data_array.copy(),
+            AVHRR_IDS[4]: avhrr_l1b_4_data_array.copy(),
+            AVHRR_IDS[5]: avhrr_l1b_5_data_array.copy(),
         },
         all_dataset_ids=AVHRR_IDS,
-        available_dataset_ids=AVHRR_IDS[:1],
+        available_dataset_ids=AVHRR_IDS[:3] + AVHRR_IDS[4:],
     )
     return scn

polar2grid/tests/test_glue.py

@@ -346,3 +346,19 @@ def test_extra_config_path(
                 extra_cpath = gettempdir()
             path_idx = captured.err.index(f"Adding enhancement configuration from file: {str(extra_cpath)}")
             assert builtin_path_idx < path_idx
+
+    def test_avhrr_list_products(self, avhrr_l1b_1_scene, chtmpdir, capsys):
+        """Test list products includes expected products."""
+        from polar2grid.glue import main
+
+        with prepare_glue_exec(avhrr_l1b_1_scene, max_computes=0):
+            args = ["-r", "avhrr_l1b_aapp", "-w", "geotiff", "--list-products", "-f", str(chtmpdir)]
+            ret = main(args)
+        output_files = glob(str(chtmpdir / "*.tif"))
+        assert len(output_files) == 0
+        assert ret == 0
+        captured = capsys.readouterr()
+        stdout = captured.out
+        print(stdout)
+        for exp_product in ("band1_vis", "band2_vis", "band3a_vis", "band4_bt", "band5_bt"):
+            assert exp_product in stdout

polar2grid/utils/legacy_compat.py

@@ -27,7 +27,7 @@
 import logging
 from typing import Generator, Iterable, Optional, Union
 
-from satpy import DataID, DataQuery, Scene
+from satpy import DataID, DataQuery, Scene, DatasetDict
 
 logger = logging.getLogger(__name__)
 
@@ -173,9 +173,9 @@ def convert_satpy_to_p2g_name(
         satpy_id_to_p2g_name = {}
         for p2g_name in possible_p2g_names:
             satpy_data_query = self._all_aliases.get(p2g_name, p2g_name)
-            try:
-                matching_satpy_id = satpy_id_dict[satpy_data_query]
-            except KeyError:
+            matching_satpy_id = _get_matching_satpy_id(satpy_id_dict, satpy_data_query)
+            if matching_satpy_id is None:
+                # no match
                 continue
 
             if matching_satpy_id in satpy_id_to_p2g_name:
@@ -245,6 +245,20 @@ def available_product_names(
         return available_p2g_names, available_custom_names, available_satpy_names
 
 
+def _get_matching_satpy_id(satpy_id_dict: DatasetDict, satpy_data_query: DataQuery | str) -> DataID:
+    matching_satpy_id = None
+    while matching_satpy_id is None:
+        # iterate until no modifiers exist in the query and we still can't find it
+        try:
+            matching_satpy_id = satpy_id_dict[satpy_data_query]
+            return matching_satpy_id
+        except KeyError:
+            if isinstance(satpy_data_query, str) or not satpy_data_query.is_modified():
+                break
+            satpy_data_query = satpy_data_query.create_less_modified_query()
+    return matching_satpy_id
+
+
 _SENSOR_ALIASES = {
     "avhrr-3": "avhrr",
 }