diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 31ee262ef5..131bf8f99e 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -388,7 +388,9 @@ This grid file can either be specified as absolute or relative (to with the facet ``horizontal_grid`` in the recipe or the extra facets (see below), or retrieved automatically from the `grid_file_uri` attribute of the input files. -In the latter case, the file is downloaded once and then cached. +In the latter case, ESMValCore first searches the input directories specified +for ICON for a grid file with that name, and if that was not successful, tries +to download the file and cache it. The cached file is valid for 7 days. ESMValCore can automatically make native ICON data `UGRID @@ -467,7 +469,7 @@ Key Description Default value if not specif =================== ================================ =================================== ``horizontal_grid`` Absolute or relative (to If not given, use file attribute ``auxiliary_data_dir`` defined ``grid_file_uri`` to retrieve ICON - in the grid file + in the grid file (see details above) :ref:`user configuration file`) path to the ICON grid file ``latitude`` Standard name of the latitude ``latitude`` diff --git a/esmvalcore/cmor/_fixes/icon/_base_fixes.py b/esmvalcore/cmor/_fixes/icon/_base_fixes.py index 2fcdbeaf3d..3255a075b6 100644 --- a/esmvalcore/cmor/_fixes/icon/_base_fixes.py +++ b/esmvalcore/cmor/_fixes/icon/_base_fixes.py @@ -1,4 +1,5 @@ """Fix base classes for ICON on-the-fly CMORizer.""" +from __future__ import annotations import logging import os @@ -14,9 +15,11 @@ import numpy as np import requests from iris import NameConstraint +from iris.cube import Cube, CubeList from iris.experimental.ugrid import Connectivity, Mesh -from ..native_datasets import NativeDatasetFix +from esmvalcore.cmor._fixes.native_datasets import NativeDatasetFix +from esmvalcore.local import _get_rootpath, _replace_tags, _select_drs logger = logging.getLogger(__name__) @@ -240,7 +243,7 @@ def add_additional_cubes(self, cubes): 'zghalf_file', ] for facet in facets_to_consider: - if facet not in self.extra_facets: + if self.extra_facets.get(facet) is None: continue path_to_add = self._get_path_from_facet(facet) logger.debug("Adding cubes from %s", path_to_add) @@ -271,7 +274,7 @@ def _tmp_local_file(local_file: Path) -> Path: with NamedTemporaryFile(prefix=f"{local_file}.") as file: return Path(file.name) - def _get_grid_from_cube_attr(self, cube: iris.cube.Cube) -> iris.cube.Cube: + def _get_grid_from_cube_attr(self, cube: Cube) -> Cube: """Get horizontal grid from `grid_file_uri` attribute of cube.""" (grid_url, grid_name) = self._get_grid_url(cube) @@ -279,10 +282,50 @@ def _get_grid_from_cube_attr(self, cube: iris.cube.Cube) -> iris.cube.Cube: if grid_name in self._horizontal_grids: return self._horizontal_grids[grid_name] - # Check if grid file has recently been downloaded and load it if - # possible + # First, check if the grid file is available in the ICON rootpath + grid = self._get_grid_from_rootpath(grid_name) + + # Second, if that didn't work, try to download grid (or use cached + # version of it if possible) + if grid is None: + grid = self._get_downloaded_grid(grid_url, grid_name) + + # Cache grid for later use + self._horizontal_grids[grid_name] = grid + + return grid + + def _get_grid_from_rootpath(self, grid_name: str) -> CubeList | None: + """Try to get grid from the ICON rootpath.""" + rootpaths = _get_rootpath('ICON') + dirname_template = _select_drs('input_dir', 'ICON') + dirname_globs = _replace_tags(dirname_template, self.extra_facets) + possible_grid_paths = [ + r / d / grid_name for r in rootpaths for d in dirname_globs + ] + for grid_path in possible_grid_paths: + if grid_path.is_file(): + logger.debug("Using ICON grid file '%s'", grid_path) + cubes = self._load_cubes(grid_path) + return cubes + return None + + def _get_downloaded_grid(self, grid_url: str, grid_name: str) -> CubeList: + """Get downloaded horizontal grid. + + Check if grid file has recently been downloaded. If not, download grid + file here. + + Note + ---- + In order to make this function thread-safe, the downloaded grid file is + first saved to a temporary location, then copied to the actual location + later. + + """ grid_path = self.CACHE_DIR / grid_name + # Check cache valid_cache = False if grid_path.exists(): mtime = grid_path.stat().st_mtime @@ -295,6 +338,7 @@ def _get_grid_from_cube_attr(self, cube: iris.cube.Cube) -> iris.cube.Cube: logger.debug("Existing cached ICON grid file '%s' is outdated", grid_path) + # File is not present in cache or too old -> download it if not valid_cache: self.CACHE_DIR.mkdir(parents=True, exist_ok=True) tmp_path = self._tmp_local_file(grid_path) @@ -320,9 +364,8 @@ def _get_grid_from_cube_attr(self, cube: iris.cube.Cube) -> iris.cube.Cube: grid_path, ) - self._horizontal_grids[grid_name] = self._load_cubes(grid_path) - - return self._horizontal_grids[grid_name] + cubes = self._load_cubes(grid_path) + return cubes def get_horizontal_grid(self, cube): """Get copy of ICON horizontal grid. @@ -361,7 +404,7 @@ def get_horizontal_grid(self, cube): file. """ - if 'horizontal_grid' in self.extra_facets: + if self.extra_facets.get('horizontal_grid') is not None: grid = self._get_grid_from_facet() else: grid = self._get_grid_from_cube_attr(cube) @@ -402,7 +445,7 @@ def get_mesh(self, cube): """ # If specified by the user, use `horizontal_grid` facet to determine # grid name; otherwise, use the `grid_file_uri` attribute of the cube - if 'horizontal_grid' in self.extra_facets: + if self.extra_facets.get('horizontal_grid') is not None: grid_path = self._get_path_from_facet( 'horizontal_grid', 'Horizontal grid file' ) @@ -410,7 +453,7 @@ def get_mesh(self, cube): else: (_, grid_name) = self._get_grid_url(cube) - # Re-use mesh if possible + # Reuse mesh if possible if grid_name in self._meshes: logger.debug("Reusing ICON mesh for grid %s", grid_name) else: @@ -436,7 +479,7 @@ def _get_start_index(horizontal_grid): return np.int32(np.min(vertex_index.data)) @staticmethod - def _load_cubes(path): + def _load_cubes(path: Path | str) -> CubeList: """Load cubes and ignore certain warnings.""" with warnings.catch_warnings(): warnings.filterwarnings( @@ -453,7 +496,7 @@ def _load_cubes(path): category=UserWarning, module='iris', ) - cubes = iris.load(str(path)) + cubes = iris.load(path) return cubes @staticmethod diff --git a/esmvalcore/cmor/_fixes/icon/icon.py b/esmvalcore/cmor/_fixes/icon/icon.py index 39562b5db1..5135faade2 100644 --- a/esmvalcore/cmor/_fixes/icon/icon.py +++ b/esmvalcore/cmor/_fixes/icon/icon.py @@ -1,6 +1,7 @@ """On-the-fly CMORizer for ICON.""" import logging +import warnings from datetime import datetime, timedelta import dask.array as da @@ -493,7 +494,15 @@ def _fix_invalid_time_units(time_coord): # this results in times that are off by 1s (e.g., 13:59:59 instead of # 14:00:00). rounded_datetimes = (year_month_day + day_float).round('s') - new_datetimes = np.array(rounded_datetimes.dt.to_pydatetime()) + with warnings.catch_warnings(): + # We already fixed the deprecated code as recommended in the + # warning, but it still shows up -> ignore it + warnings.filterwarnings( + 'ignore', + message="The behavior of DatetimeProperties.to_pydatetime .*", + category=FutureWarning, + ) + new_datetimes = np.array(rounded_datetimes.dt.to_pydatetime()) new_dt_points = date2num(np.array(new_datetimes), new_t_units) # Modify time coordinate in place diff --git a/tests/integration/cmor/_fixes/icon/test_icon.py b/tests/integration/cmor/_fixes/icon/test_icon.py index 4f9f4c4c02..6cbb3d7989 100644 --- a/tests/integration/cmor/_fixes/icon/test_icon.py +++ b/tests/integration/cmor/_fixes/icon/test_icon.py @@ -1,4 +1,5 @@ """Tests for the ICON on-the-fly CMORizer.""" +from copy import deepcopy from datetime import datetime from pathlib import Path from unittest import mock @@ -121,6 +122,7 @@ def _get_fix(mip, short_name, fix_name, session=None): ) extra_facets = get_extra_facets(dataset, ()) extra_facets['frequency'] = 'mon' + extra_facets['exp'] = 'amip' vardef = get_var_info(project='ICON', mip=mip, short_name=short_name) cls = getattr(esmvalcore.cmor._fixes.icon.icon, fix_name) fix = cls(vardef, extra_facets=extra_facets, session=session) @@ -1211,6 +1213,33 @@ def test_get_horizontal_grid_from_attr_cached_in_dict( mock_get_grid_from_facet.assert_not_called() +@mock.patch.object(IconFix, '_get_grid_from_facet', autospec=True) +def test_get_horizontal_grid_from_attr_rootpath( + mock_get_grid_from_facet, monkeypatch, tmp_path +): + """Test fix.""" + rootpath = deepcopy(CFG['rootpath']) + rootpath['ICON'] = str(tmp_path) + monkeypatch.setitem(CFG, 'rootpath', rootpath) + cube = Cube(0, attributes={'grid_file_uri': 'grid.nc'}) + grid_cube = Cube(0, var_name='test_grid_cube') + (tmp_path / 'amip').mkdir(parents=True, exist_ok=True) + iris.save(grid_cube, tmp_path / 'amip' / 'grid.nc') + + fix = get_allvars_fix('Amon', 'tas') + fix._horizontal_grids['grid_from_facet.nc'] = mock.sentinel.wrong_grid + + grid = fix.get_horizontal_grid(cube) + assert len(fix._horizontal_grids) == 2 + assert 'grid.nc' in fix._horizontal_grids + assert 'grid_from_facet.nc' in fix._horizontal_grids # has not been used + assert fix._horizontal_grids['grid.nc'] == grid + assert len(grid) == 1 + assert grid[0].var_name == 'test_grid_cube' + assert grid[0].shape == () + mock_get_grid_from_facet.assert_not_called() + + @mock.patch.object(IconFix, '_get_grid_from_facet', autospec=True) @mock.patch('esmvalcore.cmor._fixes.icon._base_fixes.requests', autospec=True) def test_get_horizontal_grid_from_attr_cached_in_file( @@ -1232,6 +1261,7 @@ def test_get_horizontal_grid_from_attr_cached_in_file( assert isinstance(grid, CubeList) assert len(grid) == 1 assert grid[0].var_name == 'grid' + assert grid[0].shape == () assert len(fix._horizontal_grids) == 1 assert 'grid_file.nc' in fix._horizontal_grids assert fix._horizontal_grids['grid_file.nc'] == grid