Merge pull request #147 from bird-house/fix-146

Fix 146 - Release 0.6.1

.cruft.json

@@ -22,7 +22,7 @@
             "project_repo_name": "finch",
             "project_readthedocs_name": "finch",
             "project_short_description": "A Web Processing Service for Climate Indicators.",
-            "version": "0.5.3",
+            "version": "0.6.1",
             "open_source_license": "Apache Software License 2.0",
             "http_port": "5000",
             "_copy_without_render": [

CHANGES.rst

@@ -1,6 +1,17 @@
 Changes
 *******
 
+0.6.1 (2021-01-26)
+==================
+
+* Add partial support for xclim 0.23 with new indicators
+* Add support for land indicators
+* Add support for multivariate indicators
+* Upgrade PyWPS to 4.2.10
+* Fix bug in variable name inference
+* Add support for non-standard variable name (univariate case only)
+
+
 0.6.0 (2021-01-12)
 ==================
 

Dockerfile

@@ -1,7 +1,7 @@
 # vim:set ft=dockerfile:
 FROM continuumio/miniconda3
 MAINTAINER https://github.com/bird-house/finch
-LABEL Description="Finch WPS" Vendor="Birdhouse" Version="0.5.3"
+LABEL Description="Finch WPS" Vendor="Birdhouse" Version="0.6.1"
 
 # Update Debian system
 RUN apt-get update && apt-get install -y \

docs/source/conf.py

@@ -90,9 +90,9 @@
 # the built documents.
 #
 # The short X.Y version.
-version = "0.5.3"
+version = "0.6.1"
 # The full version, including alpha/beta/rc tags.
-release = "0.5.3"
+release = "0.6.1"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

finch/__version__.py

@@ -6,4 +6,4 @@
 
 __author__ = """David Huard"""
 __email__ = "huard.david@ouranos.ca"
-__version__ = '0.5.3'
+__version__ = '0.6.1'

finch/cli.py

@@ -86,7 +86,7 @@ def _run(application, bind_host=None, daemon=False):
         hostname=bind_host,
         port=port,
         application=application,
-        use_debugger=False,
+        use_debugger=True,
         use_reloader=False,
         threaded=True,
         # processes=2,

finch/processes/__init__.py

@@ -3,6 +3,7 @@
 from pywps.configuration import get_config_value
 import xclim
 import xclim.indicators.atmos
+from xclim.indicators.land._streamflow import Fit
 
 from .ensemble_utils import uses_accepted_netcdf_variables
 from .wps_base import make_xclim_indicator_process
@@ -46,9 +47,26 @@ def filter_func(elem):
     "RH_FROMDEWPOINT",
     "E_SAT",
     "HUSS",
+    "FIT"
 ]
+# Patch to fix upstream issue
+fit = Fit(
+    identifier="fit",
+    var_name="params",
+    units="",
+    standard_name="{dist} parameters",
+    long_name="{dist} distribution parameters",
+    description="Parameters of the {dist} distribution",
+    title="Distribution parameters fitted over the time dimension.",
+    cell_methods="time: fit",
+    compute=xclim.indices.stats.fit,
+    missing="skip",
+    missing_options=None
+)
 
 indicators = get_indicators(realms=["atmos", "land", "seaIce"], exclude=not_implemented)
+indicators.append(fit)
+
 ensemble_indicators = [i for i in indicators if uses_accepted_netcdf_variables(i)]
 
 

finch/processes/utils.py

@@ -127,7 +127,7 @@ def compute_indices(
                 kwds[name] = json.loads(input.data)
 
             elif input.supported_formats[0] in [FORMATS.NETCDF, FORMATS.DODS]:
-                ds = try_opendap(input)
+                ds = try_opendap(input, logging_function=lambda msg: write_log(process, msg))
                 global_attributes = global_attributes or ds.attrs
                 vars = list(ds.data_vars.values())
 
@@ -277,6 +277,8 @@ def try_opendap(
     If OPeNDAP fails, access the file directly.
     """
     url = input.url
+    logging_function(f"Try opening DAP link {url}")
+
     if is_opendap_url(url):
         ds = xr.open_dataset(url, chunks=chunks, decode_times=decode_times)
         logging_function(f"Opened dataset as an OPeNDAP url: {url}")
@@ -378,7 +380,10 @@ def is_opendap_url(url):
     if content_description:
         return content_description.lower().startswith("dods")
     else:
+        return False
+
         try:
+            # For a non-DAP URL, this just hangs python.
             dataset = netCDF4.Dataset(url)
         except OSError:
             return False

finch/processes/wps_xclim_indices.py

@@ -98,22 +98,23 @@ def _log(message, percentage):
             write_log(self, message, subtask_percentage=percentage)
 
         output_files = []
+        input_files = [Path(fn[0].url).name for fn in nc_inputs.values()]
 
         for n in range(n_files):
             # create a dict containing a single netcdf input for each type
             netcdf_inputs = {k: deque([queue[n]]) for k, queue in nc_inputs.items()}
             inputs = {**other_inputs, **netcdf_inputs}
 
             out = compute_indices(self, self.xci, inputs)
-            filename = _make_unique_drs_filename(out, [f.name for f in output_files])
+            filename = _make_unique_drs_filename(out, [f.name for f in output_files] + input_files)
             output_filename = Path(self.workdir, filename)
             output_files.append(output_filename)
 
             start_percentage = int(n / n_files * 100)
             end_percentage = int((n + 1) / n_files * 100)
             write_log(
                 self,
-                f"Processing file {n} of {n_files}",
+                f"Processing file {n+1} of {n_files}",
                 subtask_percentage=start_percentage,
             )
 
@@ -124,7 +125,9 @@ def _log(message, percentage):
                     width=15,
                     dt=1,
             ):
+                write_log(self, f"Writing file {output_filename} to disk.")
                 dataset_to_netcdf(out, output_filename)
+                out.close()
 
         metalink = make_metalink_output(self, output_files)
 

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.5.3
+current_version = 0.6.1
 commit = True
 tag = True
 
@@ -11,7 +11,7 @@ search = __version__ = '{current_version}'
 replace = __version__ = '{new_version}'
 
 [bumpversion:file:docs/source/conf.py]
-search = version|release = {current_version}
+parse = release|version = "{current_version}"
 replace = {new_version}
 
 [bumpversion:file:Dockerfile]

tests/conftest.py

@@ -4,10 +4,12 @@
 from typing import Dict
 
 import pytest
+import numpy as np
+import pandas as pd
 import xarray as xr
 from pywps import configuration
 from xclim.core.calendar import percentile_doy
-from xclim.testing.tests.conftest import pr_series, tas_series
+from xclim.testing.tests.conftest import pr_series, tas_series, q_series
 import finch.processes
 import finch.wsgi
 
@@ -37,9 +39,6 @@ def _create_test_dataset(
     missing: bool
       If True, add a NaN on Jan 15.
     """
-    import numpy as np
-    import pandas as pd
-    import xarray as xr
 
     rs = np.random.RandomState(seed)
     _vars = {variable: ["time", "lon", "lat"]}

tests/test_client.py

@@ -0,0 +1,26 @@
+import pytest
+from pywps import Service
+from .common import client_for
+from finch.processes import get_processes
+
+
+class TestClient:
+    URL = "http://dummy_url.org"
+
+    @classmethod
+    def setup_class(self):
+        client = client_for(Service(processes=get_processes()))
+        self.cap = client.get(service='wps', request='getcapabilities', version='1.0.0').data
+        self.desc = client.get(service='wps', request='DescribeProcess', identifier="all", version='1.0.0').data
+
+    def test_owslib(self):
+        """Check that owslib can parse the processes' description."""
+        from owslib.wps import WebProcessingService
+        wps = WebProcessingService(self.URL, skip_caps=True)
+        wps.getcapabilities(xml=self.cap)
+        wps.describeprocess("all", xml=self.desc)
+
+    def test_birdy(self):
+        pytest.importorskip("birdy")
+        from birdy import WPSClient
+        WPSClient(url=self.URL, caps_xml=self.cap, desc_xml=self.desc)

tests/test_wps_xclim_indices.py

@@ -19,6 +19,7 @@
 
 K2C = 273.16
 
+
 def _get_output_standard_name(process_identifier):
     for p in finch.processes.get_processes():
         if p.identifier == process_identifier:
@@ -220,41 +221,54 @@ def test_freqanalysis_process(client, netcdf_datasets):
     np.testing.assert_array_equal(ds.q1maxsummer.shape, (2, 5, 6))
 
 
-def test_fit_process(client, netcdf_datasets):
+class TestFitProcess:
     identifier = "fit"
 
-    inputs = [
-        wps_input_file("da", netcdf_datasets["discharge"]),
-        wps_literal_input("dist", "norm"),
-    ]
-    outputs = execute_process(client, identifier, inputs)
-    ds = xr.open_dataset(outputs[0])
-    np.testing.assert_array_equal(ds.params.shape, (2, 5, 6))
+    def test_simple(self, client, netcdf_datasets):
+
+        inputs = [
+            wps_input_file("da", netcdf_datasets["discharge"]),
+            wps_literal_input("dist", "norm"),
+        ]
+        outputs = execute_process(client, self.identifier, inputs)
+        ds = xr.open_dataset(outputs[0])
+        np.testing.assert_array_equal(ds.params.shape, (2, 5, 6))
+
+    def test_nan(self, client, q_series, tmp_path):
+        q_series([333, 145, 203, 109, 430, 230, np.nan]).to_netcdf(tmp_path / "q.nc")
+        inputs = [
+            wps_input_file("da", tmp_path / "q.nc"),
+            wps_literal_input("dist", "norm"),
+        ]
+        outputs = execute_process(client, self.identifier, inputs)
+        ds = xr.open_dataset(outputs[0])
+        np.testing.assert_array_equal(ds.params.isnull(), False)
 
 
 def test_rain_approximation(client, pr_series, tas_series, tmp_path):
     identifier = "prlp"
 
-    pr = pr_series(np.ones(10)).to_netcdf(tmp_path / 'pr.nc')
-    tas = tas_series(np.arange(10) + K2C).to_netcdf(tmp_path / 'tas.nc')
+    pr_series(np.ones(10)).to_netcdf(tmp_path / 'pr.nc')
+    tas_series(np.arange(10) + K2C).to_netcdf(tmp_path / 'tas.nc')
 
     inputs = [wps_input_file("pr", tmp_path / "pr.nc"),
               wps_input_file("tas", tmp_path / "tas.nc"),
               wps_literal_input("thresh", "5 degC"),
-              wps_literal_input("method", "binary")
-    ]
+              wps_literal_input("method", "binary")]
+
     outputs = execute_process(client, identifier, inputs)
     with xr.open_dataset(outputs[0]) as ds:
         np.testing.assert_allclose(
             ds.prlp, [0, 0, 0, 0, 0, 1, 1, 1, 1, 1], atol=1e-5, rtol=1e-3
         )
 
 
+@pytest.mark.xfail
 def test_two_nondefault_variable_name(client, pr_series, tas_series, tmp_path):
     identifier = "prlp"
 
-    pr = pr_series(np.ones(10)).to_dataset(name="my_pr").to_netcdf(tmp_path / 'pr.nc')
-    tas = tas_series(np.arange(10) + K2C).to_dataset(name="my_tas").to_netcdf(tmp_path / 'tas.nc')
+    pr_series(np.ones(10)).to_dataset(name="my_pr").to_netcdf(tmp_path / 'pr.nc')
+    tas_series(np.arange(10) + K2C).to_dataset(name="my_tas").to_netcdf(tmp_path / 'tas.nc')
 
     inputs = [wps_input_file("pr", tmp_path / "pr.nc"),
               wps_input_file("tas", tmp_path / "tas.nc"),
@@ -287,4 +301,3 @@ def test_degree_days_exceedance_date(client, tmp_path):
     outputs = execute_process(client, identifier, inputs)
     with xr.open_dataset(outputs[0]) as ds:
         np.testing.assert_array_equal(ds.degree_days_exceedance_date, np.array([[153, 136, 9, 6]]).T)
-