From 1ba2401c767b53edcd8e8674d5da9197239b7147 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 17:34:56 -0700 Subject: [PATCH 01/12] Fix min_deps_check; revert to support numpy=1.14 and pandas=0.24 Fixes the issue noticed in: https://github.com/pydata/xarray/pull/4175#issuecomment-649135372 --- ci/min_deps_check.py | 21 +++++++++++---------- ci/requirements/py36-bare-minimum.yml | 4 ++-- ci/requirements/py36-min-all-deps.yml | 4 ++-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 527093cf5bc..636e60ecdf7 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -141,21 +141,22 @@ def process_pkg( policy_months = POLICY_MONTHS.get(pkg, POLICY_MONTHS_DEFAULT) policy_published = datetime.now() - timedelta(days=policy_months * 30) - policy_major = req_major - policy_minor = req_minor - policy_published_actual = req_published - for (major, minor), published in reversed(sorted(versions.items())): - if published < policy_published: - break - policy_major = major - policy_minor = minor - policy_published_actual = published + policy_versions = { + version: date for version, date in versions.items() if date < policy_published + } + version_and_date = lambda version: (version, policy_versions[version]) + policy_major, policy_minor = max(policy_versions, key=version_and_date) + policy_published_actual = versions[policy_major, policy_minor] if (req_major, req_minor) < (policy_major, policy_minor): status = "<" elif (req_major, req_minor) > (policy_major, policy_minor): status = "> (!)" - error("Package is too new: " + pkg) + error( + f"Package is too new: {pkg}={req_major}.{req_minor} was " + f"published on {versions[req_major, req_minor]:%Y-%m-%d} " + f"(within the past {policy_months} months)" + ) else: status = "=" diff --git a/ci/requirements/py36-bare-minimum.yml b/ci/requirements/py36-bare-minimum.yml index 00fef672855..3689e7c4b85 100644 --- a/ci/requirements/py36-bare-minimum.yml +++ b/ci/requirements/py36-bare-minimum.yml @@ -8,6 +8,6 @@ dependencies: - pytest - pytest-cov - pytest-env - - numpy=1.15 - - pandas=0.25 + - numpy=1.14 + - pandas=0.24 - setuptools=41.2 diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index a72cd000680..30f3f2e143f 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -31,8 +31,8 @@ dependencies: - nc-time-axis=1.2 - netcdf4=1.4 - numba=0.44 - - numpy=1.15 - - pandas=0.25 + - numpy=1.14 + - pandas=0.24 # - pint # See py36-min-nep18.yml - pip - pseudonetcdf=3.0 From c61f0bb1ff248e4b35adace8c6e2492357d2868f Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 18:12:30 -0700 Subject: [PATCH 02/12] Fix tests on numpy=1.14 --- setup.cfg | 4 ++-- xarray/core/nputils.py | 12 +++++++++++- xarray/core/variable.py | 12 +++++++++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index 42dc53bb882..92c905fbb8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -74,8 +74,8 @@ zip_safe = False # https://mypy.readthedocs.io/en/latest/installed_packages.htm include_package_data = True python_requires = >=3.6 install_requires = - numpy >= 1.15 - pandas >= 0.25 + numpy >= 1.14 + pandas >= 0.24 setuptools >= 41.2 # For pkg_resources setup_requires = setuptools >= 41.2 diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index fa6df63e0ea..d170f7fa9db 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -2,7 +2,6 @@ import numpy as np import pandas as pd -from numpy.core.multiarray import normalize_axis_index try: import bottleneck as bn @@ -14,6 +13,17 @@ _USE_BOTTLENECK = False +def normalize_axis_index(data, axis): + # matches numpy.core.multiarray.normalize_axis_index + # duplicated here because the NumPy function is not a public API + ndim = data.ndim + if not -ndim <= axis < ndim: + raise np.AxisError(f"axis {axis!r} out of bounds [-{ndim}, {ndim})") + if axis < 0: + axis += ndim + return axis + + def _select_along_axis(values, idx, axis): other_ind = np.ix_(*[np.arange(s) for s in idx.shape]) sl = other_ind[:axis] + (idx,) + other_ind[axis:] diff --git a/xarray/core/variable.py b/xarray/core/variable.py index e19132b1b06..c855b1bbf45 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1774,7 +1774,9 @@ def quantile( from .computation import apply_ufunc - _quantile_func = np.nanquantile if skipna else np.quantile + # TODO: switch to nanquantile/quantile once numpy >= 1.15.0 is the + # minimum requirement + _percentile_func = np.nanpercentile if skipna else np.percentile if keep_attrs is None: keep_attrs = _get_keep_attrs(default=False) @@ -1782,6 +1784,10 @@ def quantile( scalar = utils.is_scalar(q) q = np.atleast_1d(np.asarray(q, dtype=np.float64)) + # TODO: remove once numpy >= 1.15.0 is the minimum requirement + if np.count_nonzero(q < 0.0) or np.count_nonzero(q > 1.0): + raise ValueError("Quantiles must be in the range [0, 1]") + if dim is None: dim = self.dims @@ -1790,7 +1796,7 @@ def quantile( def _wrapper(npa, **kwargs): # move quantile axis to end. required for apply_ufunc - return np.moveaxis(_quantile_func(npa, **kwargs), 0, -1) + return np.moveaxis(_percentile_func(npa, **kwargs), 0, -1) axis = np.arange(-1, -1 * len(dim) - 1, -1) result = apply_ufunc( @@ -1802,7 +1808,7 @@ def _wrapper(npa, **kwargs): output_dtypes=[np.float64], output_sizes={"quantile": len(q)}, dask="parallelized", - kwargs={"q": q, "axis": axis, "interpolation": interpolation}, + kwargs={"q": q * 100, "axis": axis, "interpolation": interpolation}, ) # for backward compatibility From 6e611f75a3e7d9ed282b64de6ea4eef285310377 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 18:13:07 -0700 Subject: [PATCH 03/12] doc older required versions --- doc/installing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/installing.rst b/doc/installing.rst index a25bf65e342..eb06291237b 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -8,8 +8,8 @@ Required dependencies - Python (3.6 or later) - setuptools -- `numpy `__ (1.15 or later) -- `pandas `__ (0.25 or later) +- `numpy `__ (1.14 or later) +- `pandas `__ (0.24 or later) .. _optional-dependencies: From 68f21bdc0588050144f242338d084daf02288d9d Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 18:22:40 -0700 Subject: [PATCH 04/12] fix normalize_axis_index --- xarray/core/nputils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index d170f7fa9db..2bf1d019e4e 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -16,7 +16,7 @@ def normalize_axis_index(data, axis): # matches numpy.core.multiarray.normalize_axis_index # duplicated here because the NumPy function is not a public API - ndim = data.ndim + ndim = np.ndim(data) if not -ndim <= axis < ndim: raise np.AxisError(f"axis {axis!r} out of bounds [-{ndim}, {ndim})") if axis < 0: From c374b420afb876ed2cfdcbf58c0a13a0752e741e Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 18:25:51 -0700 Subject: [PATCH 05/12] Simplify policy version check --- ci/min_deps_check.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 636e60ecdf7..afb066df819 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -141,11 +141,9 @@ def process_pkg( policy_months = POLICY_MONTHS.get(pkg, POLICY_MONTHS_DEFAULT) policy_published = datetime.now() - timedelta(days=policy_months * 30) - policy_versions = { - version: date for version, date in versions.items() if date < policy_published - } - version_and_date = lambda version: (version, policy_versions[version]) - policy_major, policy_minor = max(policy_versions, key=version_and_date) + policy_major, policy_minor = max( + version for version, date in versions.items() if date < policy_published + ) policy_published_actual = versions[policy_major, policy_minor] if (req_major, req_minor) < (policy_major, policy_minor): From d129bc695b625cd42f09cf9f98cd8eb5c30a7b8f Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 18:34:03 -0700 Subject: [PATCH 06/12] Fix normalize_axis_index --- xarray/core/nputils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 2bf1d019e4e..3b6c19fbf9c 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -13,10 +13,9 @@ _USE_BOTTLENECK = False -def normalize_axis_index(data, axis): +def normalize_axis_index(axis, ndim): # matches numpy.core.multiarray.normalize_axis_index # duplicated here because the NumPy function is not a public API - ndim = np.ndim(data) if not -ndim <= axis < ndim: raise np.AxisError(f"axis {axis!r} out of bounds [-{ndim}, {ndim})") if axis < 0: From 432b07792b84cc0906669daec1738c9375fba512 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 19:06:25 -0700 Subject: [PATCH 07/12] Use utcnow() so version checks are independent of timezone --- ci/min_deps_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index afb066df819..62ceab94de6 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -139,7 +139,7 @@ def process_pkg( return pkg, fmt_version(req_major, req_minor, req_patch), "-", "-", "-", "(!)" policy_months = POLICY_MONTHS.get(pkg, POLICY_MONTHS_DEFAULT) - policy_published = datetime.now() - timedelta(days=policy_months * 30) + policy_published = datetime.utcnow() - timedelta(days=policy_months * 30.44) policy_major, policy_minor = max( version for version, date in versions.items() if date < policy_published From 67bcd5081d6722d89f144bdf615de01de921a6c5 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 19:55:51 -0700 Subject: [PATCH 08/12] Use relativedelta for policy months --- ci/min_deps_check.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 62ceab94de6..c0e52a0621c 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -6,6 +6,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta +from dateutil.relativedelta import relativedelta from typing import Dict, Iterator, Optional, Tuple import yaml @@ -139,7 +140,7 @@ def process_pkg( return pkg, fmt_version(req_major, req_minor, req_patch), "-", "-", "-", "(!)" policy_months = POLICY_MONTHS.get(pkg, POLICY_MONTHS_DEFAULT) - policy_published = datetime.utcnow() - timedelta(days=policy_months * 30.44) + policy_published = datetime.utcnow() - relativedelta(months=policy_months) policy_major, policy_minor = max( version for version, date in versions.items() if date < policy_published @@ -188,9 +189,9 @@ def main() -> None: ] rows = [f.result() for f in futures] - print("Package Required Policy Status") - print("------------- -------------------- -------------------- ------") - fmt = "{:13} {:7} ({:10}) {:7} ({:10}) {}" + print("Package Required Policy Status") + print("-------------- -------------------- -------------------- ------") + fmt = "{:14} {:7} ({:10}) {:7} ({:10}) {}" for row in rows: print(fmt.format(*row)) From f2a09245272a6bc26cb554a9b7c256d777542b8d Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 20:45:08 -0700 Subject: [PATCH 09/12] Use sys.exit for failures --- ci/min_deps_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index c0e52a0621c..3f746768dbc 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -5,7 +5,7 @@ import subprocess import sys from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timedelta +from datetime import datetime from dateutil.relativedelta import relativedelta from typing import Dict, Iterator, Optional, Tuple @@ -195,8 +195,8 @@ def main() -> None: for row in rows: print(fmt.format(*row)) - assert not has_errors + return has_errors if __name__ == "__main__": - main() + sys.exit(main()) From c69460a02527aadea5962c850c2fbd0c351f66d8 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 20:45:19 -0700 Subject: [PATCH 10/12] Test an invalid minimum version --- ci/requirements/py36-min-all-deps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 30f3f2e143f..b343f93d830 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -6,7 +6,7 @@ dependencies: # Run ci/min_deps_check.py to verify that this file respects the policy. # When upgrading python, numpy, or pandas, must also change # doc/installing.rst and setup.py. - - python=3.6 + - python=3.8 - black - boto3=1.9 - bottleneck=1.2 From 16833b55a4ec8c6127063f4688062e74dc058446 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 20:47:56 -0700 Subject: [PATCH 11/12] Require python-dateutil --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e04c8f74f68..564dcf8b63e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -104,7 +104,7 @@ jobs: steps: - template: ci/azure/add-conda-to-path.yml - bash: | - conda install -y pyyaml + conda install -y pyyaml python-dateutil python ci/min_deps_check.py ci/requirements/py36-bare-minimum.yml python ci/min_deps_check.py ci/requirements/py36-min-all-deps.yml displayName: minimum versions policy From 19a561c55b60b6d56750c8141af186f62977042e Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 24 Jun 2020 20:56:53 -0700 Subject: [PATCH 12/12] Fix failure --- ci/min_deps_check.py | 2 +- ci/requirements/py36-min-all-deps.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 3f746768dbc..4316c01b912 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -6,10 +6,10 @@ import sys from concurrent.futures import ThreadPoolExecutor from datetime import datetime -from dateutil.relativedelta import relativedelta from typing import Dict, Iterator, Optional, Tuple import yaml +from dateutil.relativedelta import relativedelta IGNORE_DEPS = { "black", diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index b343f93d830..30f3f2e143f 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -6,7 +6,7 @@ dependencies: # Run ci/min_deps_check.py to verify that this file respects the policy. # When upgrading python, numpy, or pandas, must also change # doc/installing.rst and setup.py. - - python=3.8 + - python=3.6 - black - boto3=1.9 - bottleneck=1.2