More comprehensive unit harmonization

tests/test_postprocess.py

@@ -145,10 +145,9 @@ def test_assign_coord_to_dim_of_different_name_keyerror(sample_dataset):
 
 
 @pytest.mark.parametrize('sample_dataset', ['lambert_conformal'], indirect=True)
-@pytest.mark.parametrize('variable,bracket', (('QNRAIN', '('), ('NOAHRES', '{')))
-def test_clean_brackets_from_units(sample_dataset, variable, bracket):
-    ds = sample_dataset.pipe(xwrf.postprocess._clean_brackets_from_units)
-    assert bracket not in ds[variable].attrs['units']
+def test_bracket_units_transl(sample_dataset):
+    assert {'QNRAIN', 'NOAHRES'}.issubset(set(sample_dataset.variables))
+    xwrf.postprocess._make_units_pint_friendly(sample_dataset)
 
 
 @pytest.mark.parametrize('sample_dataset', ['lambert_conformal'], indirect=True)

xwrf/config.py

@@ -8,5 +8,9 @@
 with open(fn) as f:
     defaults = yaml.safe_load(f)
 
+unit_harm_fn = os.path.join(os.path.dirname(__file__), 'unit_harmonization_map.yaml')
+with open(unit_harm_fn) as f:
+    defaults['unit_harmonization_map'] = yaml.safe_load(f)
+
 config = Config('xwrf', defaults=[defaults])
 config.ensure_file(fn, comment=False)

xwrf/config.yaml

@@ -38,33 +38,6 @@ brackets_to_clean_from_units:
   - '('
   - ')'
 
-unit_harmonization_map:
-  kelvin:
-    - Kelvin
-  '1':
-    - fraction
-    - Dimensionless
-  invalid:
-    - '-'
-    - flag
-    - '0/1 Flag'
-    - whoknows
-    - category
-    - none
-    - dBZ
-    - dbZ
-  meters:
-    - 'meters MSL'
-  g/m2/s:
-    - 'g/m2/s CO2'
-  umol/m2/s:
-    - 'umol co2/m2/s'
-  J/m**3/kelvin:
-    - 'J/m**3/Kelvin'
-  W/m/kelvin:
-    - 'W/m-K'
-    - 'W/m/Kelvin'
-
 cf_attribute_map:
   south_north:
     units: m

xwrf/postprocess.py

@@ -1,6 +1,5 @@
 from __future__ import annotations  # noqa: F401
 
-import re
 import warnings
 
 import numpy as np
@@ -37,23 +36,10 @@ def _decode_times(ds: xr.Dataset) -> xr.Dataset:
     return ds
 
 
-def _clean_brackets_from_units(ds: xr.Dataset) -> xr.Dataset:
-    """
-    Cleans brackets from units attributes
-    """
-    sep = '\\'
-    regex = re.compile(f'[{sep.join(config.get("brackets_to_clean_from_units"))}]')
-    for var in ds.variables:
-        if 'units' in ds[var].attrs:
-            ds[var].attrs['units'] = regex.sub('', ds[var].attrs['units'])
-    return ds
-
-
 def _make_units_pint_friendly(ds: xr.Dataset) -> xr.Dataset:
     """
     Harmonizes awkward WRF units into pint-friendly ones
     """
-    ds = _clean_brackets_from_units(ds)
     # We have to invert the mapping from "new_unit -> wrf_units" to "wrf_unit -> new_unit"
     wrf_units_map = {
         v: k for (k, val_list) in config.get('unit_harmonization_map').items() for v in val_list

xwrf/unit_harmonization_map.yaml

@@ -0,0 +1,72 @@
+1e-6:
+  - ppmv
+  - ppm
+1e-6 m2 s-1 K kg-1:
+  - pvu
+J m-2:
+  - Joules m^-2
+J m-3 K-1:
+  - J/m**3/Kelvin
+W m-1 K-1:
+  - W/m-K
+  - W/m.K
+  - W/m/Kelvin
+W m-2:
+  - W m{-2}
+cm-3:
+  - '#/cm-3'
+'1':
+  - Dimensionless
+  - fraction
+  - numerical value
+  - dimensionless
+  - area/area
+  - '#'
+g kg-1:
+  - g_smoke/kg_air
+g m-2 s-1:
+  - g/m2/s CO2
+  - g/m2/s C
+h-1:
+  - flash origin density
+  - flash column-1
+  - Source column-1
+invalid:
+  - '?'
+  - none
+  - '-'
+  - flag
+  - category
+  - ~?
+  - dBz
+  - dbZ
+  - dBZ
+  - '0/1 Flag'
+kelvin:
+  - Kelvin
+kJ m-2:
+  - KJ/m2
+kg kg-1:
+  - '# kg kg-1'
+  - Kg Kg-1
+kg kg-1 s-1:
+  - kg kg_wet-1 s-1
+kg-1:
+  - '#/kg-dryair'
+  - '# kg-1'
+  - '# kg(-1)'
+  - '  kg(-1)'
+m:
+  - m asl
+  - m agl
+  - 'meters MSL'
+m s-1:
+  - 'm/s * area'
+min:
+  - minutes since simulation start
+min-1:
+  - min{-1}
+ug/kg:
+  - ug/kg-dryair
+umol m-2 s-1:
+  - umol co2/m2/s