Merge branch 'main' into groupby-aggs-using-numpy-groupies

* main:
  [pre-commit.ci] pre-commit autoupdate (pydata#6115)
  Make CI pass by limiting dask version (pydata#6111)
  Fix mypy precommit (pydata#6110)
  [pre-commit.ci] pre-commit autoupdate (pydata#6088)
  Add type definitions in prep for pydata#6086 (pydata#6090)
  Replace distutils.version with packaging.version (pydata#6096)
  Attempt datetime coding using cftime when pandas fails (pydata#6049)
  fix tests for h5netcdf v0.12 (pydata#6097)
  disable pytest-xdist (to check CI failure) (pydata#6077)
  cftime: 'gregorian' -> 'standard' [test-upstream] (pydata#6082)
  Add release note skeleton for 0.21 (pydata#6061)

.github/workflows/ci-additional.yaml

@@ -96,7 +96,7 @@ jobs:
           python -c "import xarray"
       - name: Run tests
         run: |
-          python -m pytest -n 4 \
+          python -m pytest \
              --cov=xarray \
              --cov-report=xml \
              $PYTEST_EXTRA_FLAGS

.github/workflows/ci.yaml

@@ -87,7 +87,7 @@ jobs:
         run: |
           python -c "import xarray"
       - name: Run tests
-        run: python -m pytest -n 4
+        run: python -m pytest
           --cov=xarray
           --cov-report=xml
           --junitxml=pytest.xml

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # https://pre-commit.com/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.1.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -32,7 +32,7 @@ repos:
   #     - id: velin
   #       args: ["--write", "--compact"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
+    rev: v0.930
     hooks:
       - id: mypy
         # `properies` & `asv_bench` are copied from setup.cfg.

ci/install-upstream-wheels.sh

@@ -15,6 +15,7 @@ conda uninstall -y --force \
     bottleneck \
     sparse \
     flox \
+    h5netcdf \
     xarray
 # to limit the runtime of Upstream CI
 python -m pip install pytest-timeout
@@ -45,4 +46,5 @@ python -m pip install \
     git+https://github.com/pydata/sparse \
     git+https://github.com/intake/filesystem_spec \
     git+https://github.com/SciTools/nc-time-axis \
-    git+https://github.com/dcherian/flox
+    git+https://github.com/dcherian/flox \
+    git+https://github.com/h5netcdf/h5netcdf

ci/requirements/environment-windows.yml

@@ -8,15 +8,15 @@ dependencies:
   # - cdms2  # Not available on Windows
   # - cfgrib  # Causes Python interpreter crash on Windows: https://github.com/pydata/xarray/pull/3340
   - cftime
-  - dask-core
+  - dask-core != 2021.12.0 # https://github.com/pydata/xarray/pull/6111, can remove on next release
   - distributed
   - fsspec!=2021.7.0
   - h5netcdf
   - h5py
   - hdf5
   - hypothesis
   - iris
-  - lxml    # Optional dep of pydap
+  - lxml # Optional dep of pydap
   - matplotlib-base
   - nc-time-axis
   - netcdf4
@@ -43,5 +43,5 @@ dependencies:
   - typing_extensions
   - zarr
   - pip:
-    - numbagg
-    - git+https://github.com/dcherian/flox.git
+      - numbagg
+      - git+https://github.com/dcherian/flox.git

ci/requirements/environment.yml

@@ -10,15 +10,15 @@ dependencies:
   - cdms2
   - cfgrib
   - cftime
-  - dask-core
+  - dask-core != 2021.12.0 # https://github.com/pydata/xarray/pull/6111, can remove on next release
   - distributed
   - fsspec!=2021.7.0
   - h5netcdf
   - h5py
   - hdf5
   - hypothesis
   - iris
-  - lxml    # Optional dep of pydap
+  - lxml # Optional dep of pydap
   - matplotlib-base
   - nc-time-axis
   - netcdf4
@@ -47,5 +47,5 @@ dependencies:
   - typing_extensions
   - zarr
   - pip:
-    - numbagg
-    - git+https://github.com/dcherian/flox.git
+      - numbagg
+      - git+https://github.com/dcherian/flox.git

doc/whats-new.rst

@@ -14,6 +14,37 @@ What's New
 
     np.random.seed(123456)
 
+.. _whats-new.0.X.Y+1:
+
+v0.21.0 (unreleased)
+---------------------
+
+New Features
+~~~~~~~~~~~~
+
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+
+Deprecations
+~~~~~~~~~~~~
+
+
+Bug fixes
+~~~~~~~~~
+
+
+Documentation
+~~~~~~~~~~~~~
+
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+- Replace ``distutils.version`` with ``packaging.version``  (:issue:`6092`).
+  By `Mathias Hauser <https://github.com/mathause>`_.
+
 
 .. _whats-new.0.20.2:
 
@@ -51,6 +82,8 @@ Bug fixes
   By `Sebastian Weigand <https://github.com/s-weigand>`_.
 - Fix a regression in the removal of duplicate backend entrypoints (:issue:`5944`, :pull:`5959`)
   By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
+- Fix an issue that datasets from being saved when time variables with units that ``cftime`` can parse but pandas can not were present (:pull:`6049`).
+  By `Tim Heap <https://github.com/mx-moth>`_.
 
 Documentation
 ~~~~~~~~~~~~~

setup.cfg

@@ -200,6 +200,8 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 [mypy-h5py.*]
 ignore_missing_imports = True
+[mypy-importlib_metadata.*]
+ignore_missing_imports = True
 [mypy-iris.*]
 ignore_missing_imports = True
 [mypy-matplotlib.*]

xarray/backends/h5netcdf_.py

@@ -1,9 +1,9 @@
 import functools
 import io
 import os
-from distutils.version import LooseVersion
 
 import numpy as np
+from packaging.version import Version
 
 from ..core import indexing
 from ..core.utils import (
@@ -156,16 +156,16 @@ def open(
 
         kwargs = {"invalid_netcdf": invalid_netcdf}
         if phony_dims is not None:
-            if LooseVersion(h5netcdf.__version__) >= LooseVersion("0.8.0"):
+            if Version(h5netcdf.__version__) >= Version("0.8.0"):
                 kwargs["phony_dims"] = phony_dims
             else:
                 raise ValueError(
                     "h5netcdf backend keyword argument 'phony_dims' needs "
                     "h5netcdf >= 0.8.0."
                 )
-        if LooseVersion(h5netcdf.__version__) >= LooseVersion(
-            "0.10.0"
-        ) and LooseVersion(h5netcdf.core.h5py.__version__) >= LooseVersion("3.0.0"):
+        if Version(h5netcdf.__version__) >= Version("0.10.0") and Version(
+            h5netcdf.core.h5py.__version__
+        ) >= Version("3.0.0"):
             kwargs["decode_vlen_strings"] = decode_vlen_strings
 
         if lock is None:

xarray/coding/cftimeindex.py

@@ -42,11 +42,11 @@
 import re
 import warnings
 from datetime import timedelta
-from distutils.version import LooseVersion
 from typing import Tuple, Type
 
 import numpy as np
 import pandas as pd
+from packaging.version import Version
 
 from xarray.core.utils import is_scalar
 
@@ -193,15 +193,13 @@ def f(self, min_cftime_version=min_cftime_version):
         if cftime is None:
             raise ModuleNotFoundError("No module named 'cftime'")
 
-        version = cftime.__version__
-
-        if LooseVersion(version) >= LooseVersion(min_cftime_version):
+        if Version(cftime.__version__) >= Version(min_cftime_version):
             return get_date_field(self._data, name)
         else:
             raise ImportError(
-                "The {!r} accessor requires a minimum "
-                "version of cftime of {}. Found an "
-                "installed version of {}.".format(name, min_cftime_version, version)
+                f"The {name:!r} accessor requires a minimum "
+                f"version of cftime of {min_cftime_version}. Found an "
+                f"installed version of {cftime.__version__}."
             )
 
     f.__name__ = name

xarray/coding/times.py

@@ -400,8 +400,9 @@ def _cleanup_netcdf_time_units(units):
     delta, ref_date = _unpack_netcdf_time_units(units)
     try:
         units = "{} since {}".format(delta, format_timestamp(ref_date))
-    except OutOfBoundsDatetime:
-        # don't worry about reifying the units if they're out of bounds
+    except (OutOfBoundsDatetime, ValueError):
+        # don't worry about reifying the units if they're out of bounds or
+        # formatted badly
         pass
     return units
 
@@ -482,7 +483,7 @@ def encode_cf_datetime(dates, units=None, calendar=None):
             num = time_deltas / time_delta
         num = num.values.reshape(dates.shape)
 
-    except (OutOfBoundsDatetime, OverflowError):
+    except (OutOfBoundsDatetime, OverflowError, ValueError):
         num = _encode_datetime_with_cftime(dates, units, calendar)
 
     num = cast_to_int_if_safe(num)

xarray/core/dask_array_compat.py

@@ -1,6 +1,7 @@
 import warnings
 
 import numpy as np
+from packaging.version import Version
 
 from .pycompat import dask_version
 
@@ -56,7 +57,7 @@ def pad(array, pad_width, mode="constant", **kwargs):
     return padded
 
 
-if dask_version > "2.30.0":
+if dask_version > Version("2.30.0"):
     ensure_minimum_chunksize = da.overlap.ensure_minimum_chunksize
 else:
 
@@ -113,7 +114,7 @@ def ensure_minimum_chunksize(size, chunks):
         return tuple(output)
 
 
-if dask_version > "2021.03.0":
+if dask_version > Version("2021.03.0"):
     sliding_window_view = da.lib.stride_tricks.sliding_window_view
 else:
 
@@ -179,7 +180,7 @@ def sliding_window_view(x, window_shape, axis=None):
             axis=axis,
         )
         # map_overlap's signature changed in https://github.com/dask/dask/pull/6165
-        if dask_version > "2.18.0":
+        if dask_version > Version("2.18.0"):
             return map_overlap(_np_sliding_window_view, x, align_arrays=False, **kwargs)
         else:
             return map_overlap(x, _np_sliding_window_view, **kwargs)

xarray/core/dataset.py

@@ -1069,14 +1069,14 @@ def persist(self, **kwargs) -> "Dataset":
     @classmethod
     def _construct_direct(
         cls,
-        variables,
-        coord_names,
-        dims=None,
-        attrs=None,
-        indexes=None,
-        encoding=None,
-        close=None,
-    ):
+        variables: Dict[Any, Variable],
+        coord_names: Set[Hashable],
+        dims: Dict[Any, int] = None,
+        attrs: Dict = None,
+        indexes: Dict[Any, Index] = None,
+        encoding: Dict = None,
+        close: Callable[[], None] = None,
+    ) -> "Dataset":
         """Shortcut around __init__ for internal use when we want to skip
         costly validation
         """
@@ -2360,7 +2360,7 @@ def isel(
         indexers = drop_dims_from_indexers(indexers, self.dims, missing_dims)
 
         variables = {}
-        dims: Dict[Hashable, Tuple[int, ...]] = {}
+        dims: Dict[Hashable, int] = {}
         coord_names = self._coord_names.copy()
         indexes = self._indexes.copy() if self._indexes is not None else None
 

xarray/core/duck_array_ops.py

@@ -20,6 +20,7 @@
 from numpy import stack as _stack
 from numpy import take, tensordot, transpose, unravel_index  # noqa
 from numpy import where as _where
+from packaging.version import Version
 
 from . import dask_array_compat, dask_array_ops, dtypes, npcompat, nputils
 from .nputils import nanfirst, nanlast
@@ -175,7 +176,7 @@ def cumulative_trapezoid(y, x, axis):
 def astype(data, dtype, **kwargs):
     if (
         isinstance(data, sparse_array_type)
-        and sparse_version < "0.11.0"
+        and sparse_version < Version("0.11.0")
         and "casting" in kwargs
     ):
         warnings.warn(

xarray/core/indexing.py

@@ -8,6 +8,7 @@
 
 import numpy as np
 import pandas as pd
+from packaging.version import Version
 
 from . import duck_array_ops, nputils, utils
 from .npcompat import DTypeLike
@@ -1234,7 +1235,7 @@ def __getitem__(self, key):
                 return value
 
     def __setitem__(self, key, value):
-        if dask_version >= "2021.04.1":
+        if dask_version >= Version("2021.04.1"):
             if isinstance(key, BasicIndexer):
                 self.array[key.tuple] = value
             elif isinstance(key, VectorizedIndexer):

xarray/core/missing.py

@@ -6,6 +6,7 @@
 
 import numpy as np
 import pandas as pd
+from packaging.version import Version
 
 from . import utils
 from .common import _contains_datetime_like_objects, ones_like
@@ -741,7 +742,7 @@ def interp_func(var, x, new_x, method, kwargs):
         else:
             dtype = var.dtype
 
-        if dask_version < "2020.12":
+        if dask_version < Version("2020.12"):
             # Using meta and dtype at the same time doesn't work.
             # Remove this whenever the minimum requirement for dask is 2020.12:
             meta = None

xarray/core/npcompat.py

@@ -29,10 +29,10 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import sys
-from distutils.version import LooseVersion
 from typing import TYPE_CHECKING, Any, Sequence, TypeVar, Union
 
 import numpy as np
+from packaging.version import Version
 
 # Type annotations stubs
 try:
@@ -79,7 +79,7 @@ def __array__(self) -> np.ndarray:
     DTypeLike = Union[np.dtype, str]  # type: ignore[misc]
 
 
-if LooseVersion(np.__version__) >= "1.20.0":
+if Version(np.__version__) >= Version("1.20.0"):
     sliding_window_view = np.lib.stride_tricks.sliding_window_view
 else:
     from numpy.core.numeric import normalize_axis_tuple  # type: ignore[attr-defined]

xarray/core/pdcompat.py

@@ -37,12 +37,11 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from distutils.version import LooseVersion
-
 import pandas as pd
+from packaging.version import Version
 
 # allow ourselves to type checks for Panel even after it's removed
-if LooseVersion(pd.__version__) < "0.25.0":
+if Version(pd.__version__) < Version("0.25.0"):
     Panel = pd.Panel
 else: