Support loading ICON grid from ICON rootpath (#2337)

Co-authored-by: Valeriu Predoi <valeriu.predoi@gmail.com>

doc/quickstart/find_data.rst

@@ -388,7 +388,9 @@ This grid file can either be specified as absolute or relative (to
 with the facet ``horizontal_grid`` in the recipe or the extra facets (see
 below), or retrieved automatically from the `grid_file_uri` attribute of the
 input files.
-In the latter case, the file is downloaded once and then cached.
+In the latter case, ESMValCore first searches the input directories specified
+for ICON for a grid file with that name, and if that was not successful, tries
+to download the file and cache it.
 The cached file is valid for 7 days.
 
 ESMValCore can automatically make native ICON data `UGRID
@@ -467,7 +469,7 @@ Key                 Description                      Default value if not specif
 =================== ================================ ===================================
 ``horizontal_grid`` Absolute or relative (to         If not given, use file attribute
                     ``auxiliary_data_dir`` defined   ``grid_file_uri`` to retrieve ICON
-                    in the                           grid file
+                    in the                           grid file (see details above)
                     :ref:`user configuration file`)
                     path to the ICON grid file
 ``latitude``        Standard name of the latitude    ``latitude``

esmvalcore/cmor/_fixes/icon/_base_fixes.py

@@ -1,4 +1,5 @@
 """Fix base classes for ICON on-the-fly CMORizer."""
+from __future__ import annotations
 
 import logging
 import os
@@ -14,9 +15,11 @@
 import numpy as np
 import requests
 from iris import NameConstraint
+from iris.cube import Cube, CubeList
 from iris.experimental.ugrid import Connectivity, Mesh
 
-from ..native_datasets import NativeDatasetFix
+from esmvalcore.cmor._fixes.native_datasets import NativeDatasetFix
+from esmvalcore.local import _get_rootpath, _replace_tags, _select_drs
 
 logger = logging.getLogger(__name__)
 
@@ -240,7 +243,7 @@ def add_additional_cubes(self, cubes):
             'zghalf_file',
         ]
         for facet in facets_to_consider:
-            if facet not in self.extra_facets:
+            if self.extra_facets.get(facet) is None:
                 continue
             path_to_add = self._get_path_from_facet(facet)
             logger.debug("Adding cubes from %s", path_to_add)
@@ -271,18 +274,58 @@ def _tmp_local_file(local_file: Path) -> Path:
         with NamedTemporaryFile(prefix=f"{local_file}.") as file:
             return Path(file.name)
 
-    def _get_grid_from_cube_attr(self, cube: iris.cube.Cube) -> iris.cube.Cube:
+    def _get_grid_from_cube_attr(self, cube: Cube) -> Cube:
         """Get horizontal grid from `grid_file_uri` attribute of cube."""
         (grid_url, grid_name) = self._get_grid_url(cube)
 
         # If already loaded, return the horizontal grid
         if grid_name in self._horizontal_grids:
             return self._horizontal_grids[grid_name]
 
-        # Check if grid file has recently been downloaded and load it if
-        # possible
+        # First, check if the grid file is available in the ICON rootpath
+        grid = self._get_grid_from_rootpath(grid_name)
+
+        # Second, if that didn't work, try to download grid (or use cached
+        # version of it if possible)
+        if grid is None:
+            grid = self._get_downloaded_grid(grid_url, grid_name)
+
+        # Cache grid for later use
+        self._horizontal_grids[grid_name] = grid
+
+        return grid
+
+    def _get_grid_from_rootpath(self, grid_name: str) -> CubeList | None:
+        """Try to get grid from the ICON rootpath."""
+        rootpaths = _get_rootpath('ICON')
+        dirname_template = _select_drs('input_dir', 'ICON')
+        dirname_globs = _replace_tags(dirname_template, self.extra_facets)
+        possible_grid_paths = [
+            r / d / grid_name for r in rootpaths for d in dirname_globs
+        ]
+        for grid_path in possible_grid_paths:
+            if grid_path.is_file():
+                logger.debug("Using ICON grid file '%s'", grid_path)
+                cubes = self._load_cubes(grid_path)
+                return cubes
+        return None
+
+    def _get_downloaded_grid(self, grid_url: str, grid_name: str) -> CubeList:
+        """Get downloaded horizontal grid.
+
+        Check if grid file has recently been downloaded. If not, download grid
+        file here.
+
+        Note
+        ----
+        In order to make this function thread-safe, the downloaded grid file is
+        first saved to a temporary location, then copied to the actual location
+        later.
+
+        """
         grid_path = self.CACHE_DIR / grid_name
 
+        # Check cache
         valid_cache = False
         if grid_path.exists():
             mtime = grid_path.stat().st_mtime
@@ -295,6 +338,7 @@ def _get_grid_from_cube_attr(self, cube: iris.cube.Cube) -> iris.cube.Cube:
                 logger.debug("Existing cached ICON grid file '%s' is outdated",
                              grid_path)
 
+        # File is not present in cache or too old -> download it
         if not valid_cache:
             self.CACHE_DIR.mkdir(parents=True, exist_ok=True)
             tmp_path = self._tmp_local_file(grid_path)
@@ -320,9 +364,8 @@ def _get_grid_from_cube_attr(self, cube: iris.cube.Cube) -> iris.cube.Cube:
                 grid_path,
             )
 
-        self._horizontal_grids[grid_name] = self._load_cubes(grid_path)
-
-        return self._horizontal_grids[grid_name]
+        cubes = self._load_cubes(grid_path)
+        return cubes
 
     def get_horizontal_grid(self, cube):
         """Get copy of ICON horizontal grid.
@@ -361,7 +404,7 @@ def get_horizontal_grid(self, cube):
             file.
 
         """
-        if 'horizontal_grid' in self.extra_facets:
+        if self.extra_facets.get('horizontal_grid') is not None:
             grid = self._get_grid_from_facet()
         else:
             grid = self._get_grid_from_cube_attr(cube)
@@ -402,15 +445,15 @@ def get_mesh(self, cube):
         """
         # If specified by the user, use `horizontal_grid` facet to determine
         # grid name; otherwise, use the `grid_file_uri` attribute of the cube
-        if 'horizontal_grid' in self.extra_facets:
+        if self.extra_facets.get('horizontal_grid') is not None:
             grid_path = self._get_path_from_facet(
                 'horizontal_grid', 'Horizontal grid file'
             )
             grid_name = grid_path.name
         else:
             (_, grid_name) = self._get_grid_url(cube)
 
-        # Re-use mesh if possible
+        # Reuse mesh if possible
         if grid_name in self._meshes:
             logger.debug("Reusing ICON mesh for grid %s", grid_name)
         else:
@@ -436,7 +479,7 @@ def _get_start_index(horizontal_grid):
         return np.int32(np.min(vertex_index.data))
 
     @staticmethod
-    def _load_cubes(path):
+    def _load_cubes(path: Path | str) -> CubeList:
         """Load cubes and ignore certain warnings."""
         with warnings.catch_warnings():
             warnings.filterwarnings(
@@ -453,7 +496,7 @@ def _load_cubes(path):
                 category=UserWarning,
                 module='iris',
             )
-            cubes = iris.load(str(path))
+            cubes = iris.load(path)
         return cubes
 
     @staticmethod

esmvalcore/cmor/_fixes/icon/icon.py

@@ -1,6 +1,7 @@
 """On-the-fly CMORizer for ICON."""
 
 import logging
+import warnings
 from datetime import datetime, timedelta
 
 import dask.array as da
@@ -493,7 +494,15 @@ def _fix_invalid_time_units(time_coord):
         # this results in times that are off by 1s (e.g., 13:59:59 instead of
         # 14:00:00).
         rounded_datetimes = (year_month_day + day_float).round('s')
-        new_datetimes = np.array(rounded_datetimes.dt.to_pydatetime())
+        with warnings.catch_warnings():
+            # We already fixed the deprecated code as recommended in the
+            # warning, but it still shows up -> ignore it
+            warnings.filterwarnings(
+                'ignore',
+                message="The behavior of DatetimeProperties.to_pydatetime .*",
+                category=FutureWarning,
+            )
+            new_datetimes = np.array(rounded_datetimes.dt.to_pydatetime())
         new_dt_points = date2num(np.array(new_datetimes), new_t_units)
 
         # Modify time coordinate in place

tests/integration/cmor/_fixes/icon/test_icon.py

@@ -1,4 +1,5 @@
 """Tests for the ICON on-the-fly CMORizer."""
+from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
 from unittest import mock
@@ -121,6 +122,7 @@ def _get_fix(mip, short_name, fix_name, session=None):
     )
     extra_facets = get_extra_facets(dataset, ())
     extra_facets['frequency'] = 'mon'
+    extra_facets['exp'] = 'amip'
     vardef = get_var_info(project='ICON', mip=mip, short_name=short_name)
     cls = getattr(esmvalcore.cmor._fixes.icon.icon, fix_name)
     fix = cls(vardef, extra_facets=extra_facets, session=session)
@@ -1211,6 +1213,33 @@ def test_get_horizontal_grid_from_attr_cached_in_dict(
     mock_get_grid_from_facet.assert_not_called()
 
 
+@mock.patch.object(IconFix, '_get_grid_from_facet', autospec=True)
+def test_get_horizontal_grid_from_attr_rootpath(
+    mock_get_grid_from_facet, monkeypatch, tmp_path
+):
+    """Test fix."""
+    rootpath = deepcopy(CFG['rootpath'])
+    rootpath['ICON'] = str(tmp_path)
+    monkeypatch.setitem(CFG, 'rootpath', rootpath)
+    cube = Cube(0, attributes={'grid_file_uri': 'grid.nc'})
+    grid_cube = Cube(0, var_name='test_grid_cube')
+    (tmp_path / 'amip').mkdir(parents=True, exist_ok=True)
+    iris.save(grid_cube, tmp_path / 'amip' / 'grid.nc')
+
+    fix = get_allvars_fix('Amon', 'tas')
+    fix._horizontal_grids['grid_from_facet.nc'] = mock.sentinel.wrong_grid
+
+    grid = fix.get_horizontal_grid(cube)
+    assert len(fix._horizontal_grids) == 2
+    assert 'grid.nc' in fix._horizontal_grids
+    assert 'grid_from_facet.nc' in fix._horizontal_grids  # has not been used
+    assert fix._horizontal_grids['grid.nc'] == grid
+    assert len(grid) == 1
+    assert grid[0].var_name == 'test_grid_cube'
+    assert grid[0].shape == ()
+    mock_get_grid_from_facet.assert_not_called()
+
+
 @mock.patch.object(IconFix, '_get_grid_from_facet', autospec=True)
 @mock.patch('esmvalcore.cmor._fixes.icon._base_fixes.requests', autospec=True)
 def test_get_horizontal_grid_from_attr_cached_in_file(
@@ -1232,6 +1261,7 @@ def test_get_horizontal_grid_from_attr_cached_in_file(
     assert isinstance(grid, CubeList)
     assert len(grid) == 1
     assert grid[0].var_name == 'grid'
+    assert grid[0].shape == ()
     assert len(fix._horizontal_grids) == 1
     assert 'grid_file.nc' in fix._horizontal_grids
     assert fix._horizontal_grids['grid_file.nc'] == grid