align units throughout code (#364)

* proper passing of units * support for WATHTBRKD * avoid duplications * updated whatsnew
Deltares · Oct 31, 2024 · 3dbac31 · 3dbac31
1 parent 69302d5
commit 3dbac31
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 31 deletions.
diff --git a/docs/whats-new.md b/docs/whats-new.md
@@ -6,6 +6,9 @@
 - `datetime.timezone` support in `hatyan.write_components()` to support components from ddlpy timeseries in [#359](https://github.com/Deltares/hatyan/pull/359)
 - adding metdata to timeseries from ddlpy in `hatyan.ddlpy_to_hatyan()` in [#360](https://github.com/Deltares/hatyan/pull/360)
 
+### Fix
+- aligned (conversion of) units in [#364](https://github.com/Deltares/hatyan/pull/364)
+
 
 ## 2.9.0 (2024-09-11)
 

diff --git a/hatyan/components.py b/hatyan/components.py
@@ -157,7 +157,11 @@ def write_components(comp, filename):
     station = metadata.pop('station')
     grootheid = metadata.pop('grootheid')
     vertref = metadata.pop('vertref')
-    unit = metadata.pop('eenheid')
+    eenheid = metadata.pop('eenheid')
+    # change unit because of *100 below
+    assert eenheid == 'm'
+    factor = 100
+    eenheid = 'cm'
 
     tstart = metadata.pop('tstart')
     tstop = metadata.pop('tstop')
@@ -170,7 +174,7 @@ def write_components(comp, filename):
     tstop_str = tstop.strftime("%Y%m%d  %H%M")
 
     if 'A0' in comp.index.tolist():
-        midd = comp.loc['A0','A']*100
+        midd = comp.loc['A0','A']*factor
         comp = comp.drop('A0',axis=0)
     else:
         midd = 0
@@ -219,7 +223,7 @@ def write_components(comp, filename):
             for key in metadata.keys():
                 f.write(f'* {key} : {metadata[key]}\n')
 
-        f.write(f'STAT  {station}    {grootheid}    {vertref}    {unit}    {waarnemingssoort}\n')
+        f.write(f'STAT  {station}    {grootheid}    {vertref}    {eenheid}    {waarnemingssoort}\n')
         f.write(f'PERD  {tstart_str}  {tstop_str}     {tzone_min}\n')
         f.write( 'CODE      3\n')
         f.write(f'MIDD {midd:9.3f}\n')
@@ -228,7 +232,7 @@ def write_components(comp, filename):
             comp_one = comp.loc[compname]
             f.write("COMP %4i %12.6f %9.3f %7.2f  %-12s\n" % (comp_one['const_no'],
                                                               comp_one['const_speed'], 
-                                                              comp_one['A']*100, 
+                                                              comp_one['A']*factor, 
                                                               comp_one['phi_deg']%360, 
                                                               compname))
 
@@ -421,7 +425,7 @@ def read_components(filename):
 
     Aphi_datapd_A0line = pd.DataFrame({'A': [A0_cm], 'phi': [0], 'name': ['A0']})
     Aphi_datapd_raw = pd.concat([Aphi_datapd_A0line,Aphi_datapd_raw_noA0],ignore_index=True)
-    comp_pd = pd.DataFrame({'A': Aphi_datapd_raw['A'].values/100, 'phi_deg': Aphi_datapd_raw['phi'].values}, index=Aphi_datapd_raw['name'].values)
+    comp_pd = pd.DataFrame({'A': Aphi_datapd_raw['A'].values, 'phi_deg': Aphi_datapd_raw['phi'].values}, index=Aphi_datapd_raw['name'].values)
 
     # add metadata
     if not (stat_available & perd_available):
@@ -430,6 +434,10 @@ def read_components(filename):
                        "Add metadata to the header of the component file like this:\n"
                        "STAT  DENHDR    WATHTE     NAP     cm    1\n"
                        "PERD  20090101  0000  20121231  2300     60")
+    # convert cm to m
+    assert eenheid == 'cm'
+    comp_pd['A'] /= 100
+    eenheid = 'm'
     metadata = {'station':station,
                 'grootheid':grootheid, 'eenheid':eenheid,
                 'vertref':vertref,

diff --git a/hatyan/ddlpy_helpers.py b/hatyan/ddlpy_helpers.py
@@ -32,10 +32,12 @@ def ddlpy_to_hatyan(ddlpy_meas, ddlpy_meas_exttyp=None):
 
     """
 
-
     ts_pd = ddlpy_to_hatyan_plain(ddlpy_meas, isnumeric=True)
-    ts_pd['values'] /= 100 #convert from cm to m
     metadata = metadata_from_ddlpy(ddlpy_meas)
+    # conver units from cm to meters
+    assert metadata['eenheid'] == 'cm'
+    ts_pd['values'] /= 100 #convert from cm to m
+    metadata['eenheid'] = 'm'
     ts_pd.attrs = metadata
     if ddlpy_meas_exttyp is None:
         return ts_pd

diff --git a/hatyan/metadata.py b/hatyan/metadata.py
@@ -75,14 +75,18 @@ def wns_from_metadata(metadata):
     """
 
     meta_sel = {key:metadata[key] for key in ['grootheid','eenheid','vertref']}
+    grootheid = metadata['grootheid']
+    eenheid = metadata['eenheid']
+    vertref = metadata['vertref']
+    assert eenheid in ['m']
 
-    if meta_sel == {'grootheid':'WATHTE', 'eenheid':'cm', 'vertref':'NAP'}:
+    if (grootheid == 'WATHTE') & (vertref == 'NAP'):
         wns = 1
-    elif meta_sel == {'grootheid':'WATHTE', 'eenheid':'cm', 'vertref':'MSL'}:
+    elif (grootheid == 'WATHTE') & (vertref == 'MSL'):
         wns = 54
-    elif meta_sel == {'grootheid':'WATHTBRKD', 'eenheid':'cm', 'vertref':'NAP'}:
+    elif (grootheid == 'WATHTBRKD') & (vertref == 'NAP'):
         wns = 18
-    elif meta_sel == {'grootheid':'WATHTBRKD', 'eenheid':'cm', 'vertref':'MSL'}:
+    elif (grootheid == 'WATHTBRKD') & (vertref == 'MSL'):
         wns = 55
     else:
         raise ValueError(f'combination of quantity/unit/vertref not found available in wns_from_metadata():\n{meta_sel}')

diff --git a/hatyan/timeseries.py b/hatyan/timeseries.py
@@ -1395,8 +1395,8 @@ def read_dia_nonequidistant(filename, diablocks_pd, block_id):
     data_pd_HWLW['qualitycode'] = data_pd_HWLWtemp.iloc[:,1].astype('int')
     data_pd_HWLW = data_pd_HWLW.drop('HWLWcode/qualitycode',axis='columns')
 
-    #convert value from cm to m
-    data_pd_HWLW['values'] = data_pd_HWLW['valuecm:'].str.strip(':').astype('int')/100
+    #construct df
+    data_pd_HWLW['values'] = data_pd_HWLW['valuecm:'].str.strip(':').astype('int')
     data_pd_HWLW = data_pd_HWLW.drop('valuecm:',axis='columns')
 
     bool_hiaat = data_pd_HWLW['qualitycode'] == 99
@@ -1442,7 +1442,7 @@ def read_dia_equidistant(filename, diablocks_pd, block_id):
 
     # convert HWLW+quality code to separate columns
     data_pd_temp = data_pd.loc[:,'valuecm/qualitycode'].str.split('/', expand=True)
-    data_pd['values'] = data_pd_temp.iloc[:,0].astype('int')/100
+    data_pd['values'] = data_pd_temp.iloc[:,0].astype('int')
     data_pd['qualitycode'] = data_pd_temp.iloc[:,1].astype('int')
     data_pd = data_pd.drop('valuecm/qualitycode',axis='columns')
 
@@ -1524,7 +1524,7 @@ def read_dia(filename, station=None, block_ids=None, allow_duplicates=False):
         if not pd.Series(block_ids_one).isin(diablocks_pd.index).all():
             raise ValueError(f"Invalid values in block_ids list ({block_ids_one}), "
                              f"possible are {diablocks_pd.index.tolist()} (all integers)")
-            
+
         if station is not None:
             if not isinstance(station,str):
                 raise TypeError('Station argument should be of type string')
@@ -1556,6 +1556,12 @@ def read_dia(filename, station=None, block_ids=None, allow_duplicates=False):
     data_pd_all = pd.concat(data_pd_list)
     metadata_compare(metadata_list)
     metadata = metadata_list[0].copy()
+
+    # convert cm to m
+    assert metadata['eenheid'] == 'cm'
+    data_pd_all['values'] /= 100
+    metadata['eenheid'] = 'm'
+
     data_pd_all = metadata_add_to_obj(data_pd_all,metadata)
 
     if allow_duplicates:

diff --git a/tests/test_ddlpy_helpers.py b/tests/test_ddlpy_helpers.py
@@ -21,14 +21,15 @@ def locations():
 
 
 @pytest.mark.unittest
-def test_ddlpy_to_hatyan(locations):
+@pytest.mark.parametrize("grootheid", [pytest.param(gr, id=gr) for gr in ['WATHTE','WATHTBRKD']])
+def test_ddlpy_to_hatyan(locations, grootheid):
     # input parameters
     tstart_dt = dt.datetime(2023,12,24)
     tstop_dt = dt.datetime(2024,1,5)
 
     bool_station = locations.index.isin(['HOEKVHLD'])
     bool_hoedanigheid = locations['Hoedanigheid.Code'].isin(['NAP'])
-    bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE'])
+    bool_grootheid = locations['Grootheid.Code'].isin([grootheid])
     bool_groepering = locations['Groepering.Code'].isin(['NVT'])
     locs_wathte = locations.loc[bool_station & bool_grootheid &
                                 bool_groepering & bool_hoedanigheid]
@@ -48,9 +49,9 @@ def test_ddlpy_to_hatyan(locations):
     # check if metadata is complete and correct
     meta_fromts = ts_measwl.attrs
     meta_expected = {
-        'grootheid': 'WATHTE',
+        'grootheid': grootheid,
         'groepering': 'NVT',
-        'eenheid': 'cm',
+        'eenheid': 'm',
         'vertref': 'NAP',
         'station': 'HOEKVHLD',
         'origin': 'ddlpy',
@@ -91,7 +92,7 @@ def test_convert_hwlwstr2num(locations):
     meta_expected = {
         'grootheid': 'WATHTE',
         'groepering': 'GETETM2',
-        'eenheid': 'cm',
+        'eenheid': 'm',
         'vertref': 'NAP',
         'station': 'HOEKVHLD',
         'origin': 'ddlpy',
@@ -125,7 +126,7 @@ def test_ddlpy_to_components(tmp_path, locations):
     meta_expected = {
         'grootheid': 'WATHTE',
         'groepering': 'NVT',
-        'eenheid': 'cm',
+        'eenheid': 'm',
         'vertref': 'NAP',
         'station': 'VLISSGN',
         'origin': 'ddlpy'

diff --git a/tests/test_metadata.py b/tests/test_metadata.py
@@ -25,7 +25,7 @@ def test_readts_dia_metadata_multifile():
 
     meta_expected = {'station': 'VLISSGN',
      'grootheid': 'WATHTE',
-     'eenheid': 'cm',
+     'eenheid': 'm',
      'vertref': 'NAP',
      'TYP': 'TE',
      'groepering': 'NVT',
@@ -67,7 +67,7 @@ def test_anapred_metadata():
 
     meta_expected_xfac0 = {'station': 'VLISSGN',
      'grootheid': 'WATHTBRKD',
-     'eenheid': 'cm',
+     'eenheid': 'm',
      'vertref': 'NAP',
      'TYP': 'TE',
      'groepering': 'NVT',
@@ -106,15 +106,15 @@ def test_hwlw_metadata():
 
     meas_ext_meta_expected = {'station': 'VLISSGN',
      'grootheid': 'WATHTE',
-     'eenheid': 'cm',
+     'eenheid': 'm',
      'vertref': 'NAP',
      'TYP': 'TE',
      'groepering': 'NVT',
      'origin': 'from timeseries dia file'}
 
     pred_ext_meta_expected = {'station': 'VLISSGN',
      'grootheid': 'WATHTBRKD',
-     'eenheid': 'cm',
+     'eenheid': 'm',
      'vertref': 'NAP',
      'TYP': 'TE',
      'groepering': 'NVT',
@@ -142,7 +142,7 @@ def test_readts_dia_metadata_multiblock():
 
     meta_expected = {'station': 'HOEKVHLD',
      'grootheid': 'WATHTE',
-     'eenheid': 'cm',
+     'eenheid': 'm',
      'vertref': 'NAP',
      'TYP': 'TN',
      'groepering': 'GETETM2',
@@ -160,18 +160,18 @@ def test_metadata_compare():
         'TYP': 'TE',
         'groepering': 'NVT',
         'grootheid': 'WATHTE',
-        'eenheid': 'cm',
+        'eenheid': 'm',
         }
 
     metadata_compare([metadata,metadata,metadata])
 
 
 @pytest.mark.unittest
 def test_wns_from_metadata():
-    metadata_1 = {'grootheid':'WATHTE', 'eenheid':'cm', 'vertref':'NAP'}
-    metadata_54 = {'grootheid':'WATHTE', 'eenheid':'cm', 'vertref':'MSL'}
-    metadata_18 = {'grootheid':'WATHTBRKD', 'eenheid':'cm', 'vertref':'NAP'}
-    metadata_55 = {'grootheid':'WATHTBRKD', 'eenheid':'cm', 'vertref':'MSL'}
+    metadata_1 = {'grootheid':'WATHTE', 'eenheid':'m', 'vertref':'NAP'}
+    metadata_54 = {'grootheid':'WATHTE', 'eenheid':'m', 'vertref':'MSL'}
+    metadata_18 = {'grootheid':'WATHTBRKD', 'eenheid':'m', 'vertref':'NAP'}
+    metadata_55 = {'grootheid':'WATHTBRKD', 'eenheid':'m', 'vertref':'MSL'}
 
     wns_1 = wns_from_metadata(metadata_1)
     wns_54 = wns_from_metadata(metadata_54)