From 731d5f8de38a0a44e37e5f7c9b872e06fef3fb10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 09:47:37 +0200 Subject: [PATCH 1/9] Parallelize the test suite --- datashader/tests/benchmarks/__init__.py | 3 ++ datashader/tests/conftest.py | 34 +++++++++++++++++++++ datashader/tests/test_transfer_functions.py | 7 +++-- setup.py | 2 ++ tox.ini | 3 +- 5 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 datashader/tests/benchmarks/__init__.py create mode 100644 datashader/tests/conftest.py diff --git a/datashader/tests/benchmarks/__init__.py b/datashader/tests/benchmarks/__init__.py new file mode 100644 index 000000000..1a24ae6ee --- /dev/null +++ b/datashader/tests/benchmarks/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytestmark = pytest.mark.benchmark diff --git a/datashader/tests/conftest.py b/datashader/tests/conftest.py new file mode 100644 index 000000000..ed9933742 --- /dev/null +++ b/datashader/tests/conftest.py @@ -0,0 +1,34 @@ +CUSTOM_MARKS = ("benchmark",) + + +def pytest_addoption(parser): + for marker in CUSTOM_MARKS: + parser.addoption( + f"--{marker}", + action="store_true", + default=False, + help=f"Run {marker} related tests", + ) + + +def pytest_configure(config): + for marker in CUSTOM_MARKS: + config.addinivalue_line("markers", f"{marker}: {marker} test marker") + + +def pytest_collection_modifyitems(config, items): + skipped, selected = [], [] + markers = [m for m in CUSTOM_MARKS if config.getoption(f"--{m}")] + empty = not markers + for item in items: + if empty and any(m in item.keywords for m in CUSTOM_MARKS): + skipped.append(item) + elif empty: + selected.append(item) + elif not empty and any(m in item.keywords for m in markers): + selected.append(item) + else: + skipped.append(item) + + config.hook.pytest_deselected(items=skipped) + items[:] = selected diff --git a/datashader/tests/test_transfer_functions.py b/datashader/tests/test_transfer_functions.py index 8e2ae472f..574e73e36 100644 --- a/datashader/tests/test_transfer_functions.py +++ b/datashader/tests/test_transfer_functions.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from io import BytesIO import numpy as np @@ -13,6 +14,8 @@ coords = dict([('x_axis', [3, 4, 5]), ('y_axis', [0, 1, 2])]) dims = ['y_axis', 'x_axis'] +test_gpu = bool(int(os.getenv("DATASHADER_TEST_GPU", 0))) + # CPU def build_agg(array_module=np): a = array_module.arange(10, 19, dtype='u4').reshape((3, 3)) @@ -42,12 +45,12 @@ def create_dask_array_np(*args, **kwargs): return da.from_array(np.array(*args, **kwargs)) -try: +if test_gpu: import cupy aggs = [build_agg(np), build_agg(cupy), build_agg_dask()] arrays = [np.array, cupy.array, create_dask_array_np] array_modules = [np, cupy] -except ImportError: +else: cupy = None aggs = [build_agg(np), build_agg_dask()] arrays = [np.array, create_dask_array_np] diff --git a/setup.py b/setup.py index 0b599e9b7..afedbf5f3 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,8 @@ 'pytest <8', # Fails lint with IPynbFile is deprecated 'pytest-benchmark', 'pytest-cov', + 'psutil', + 'pytest-xdist', 'rasterio', 'rioxarray', 'scikit-image', diff --git a/tox.ini b/tox.ini index a6bdc77b0..d890e463f 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,8 @@ commands = flake8 [_unit] description = Run unit tests deps = .[tests] -commands = pytest datashader --cov=./datashader --cov-append +commands = pytest datashader -n logical --dist loadgroup --cov=./datashader --cov-append + pytest datashader --benchmark --cov=./datashader --cov-append [_unit_deploy] description = Run unit tests without coverage From 39ef510f76464929057ece5ebf6efeebd4d81533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 10:18:54 +0200 Subject: [PATCH 2/9] Updates --- datashader/data_libraries/cudf.py | 9 +++++++-- datashader/data_libraries/dask_cudf.py | 9 +++++++-- datashader/datashape/coretypes.py | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/datashader/data_libraries/cudf.py b/datashader/data_libraries/cudf.py index 343a0639e..ff8586a99 100644 --- a/datashader/data_libraries/cudf.py +++ b/datashader/data_libraries/cudf.py @@ -1,9 +1,14 @@ from __future__ import annotations +from contextlib import suppress from datashader.data_libraries.pandas import default from datashader.core import bypixel -import cudf -@bypixel.pipeline.register(cudf.DataFrame) def cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False): return default(glyph, df, schema, canvas, summary, antialias=antialias, cuda=True) + + +with suppress(ImportError): + import cudf + + cudf_pipeline = bypixel.pipeline.register(cudf.DataFrame)(cudf_pipeline) diff --git a/datashader/data_libraries/dask_cudf.py b/datashader/data_libraries/dask_cudf.py index 9e0f7634d..26308830a 100644 --- a/datashader/data_libraries/dask_cudf.py +++ b/datashader/data_libraries/dask_cudf.py @@ -1,9 +1,14 @@ from __future__ import annotations +from contextlib import suppress from datashader.data_libraries.dask import dask_pipeline from datashader.core import bypixel -import dask_cudf -@bypixel.pipeline.register(dask_cudf.DataFrame) def dask_cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False): return dask_pipeline(df, schema, canvas, glyph, summary, antialias=antialias, cuda=True) + + +with suppress(ImportError): + import dask_cudf + + dask_cudf_pipeline = bypixel.pipeline.register(dask_cudf.DataFrame)(dask_cudf_pipeline) diff --git a/datashader/datashape/coretypes.py b/datashader/datashape/coretypes.py index 637683e6b..6e2e2cf33 100644 --- a/datashader/datashape/coretypes.py +++ b/datashader/datashape/coretypes.py @@ -254,7 +254,7 @@ def to_numpy_dtype(self): return np.dtype('datetime64[us]') -_units = set(['ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y']) +_units = ('ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y') _unit_aliases = { From f9357e2d973ec12a4db249a9ffc06aa48dce1718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 11:25:10 +0200 Subject: [PATCH 3/9] Disable benchmark warnings for tests and add par. to nojit test --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d890e463f..3b190fdd4 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ commands = flake8 [_unit] description = Run unit tests deps = .[tests] -commands = pytest datashader -n logical --dist loadgroup --cov=./datashader --cov-append +commands = pytest datashader -n logical --dist loadgroup --cov=./datashader --cov-append --benchmark-skip pytest datashader --benchmark --cov=./datashader --cov-append [_unit_deploy] @@ -28,7 +28,7 @@ commands = pytest datashader [_unit_nojit] description = Run select unit tests with numba jit disabled deps = .[tests] -commands = pytest datashader -k "not benchmarks and not test_tiles" --cov=./datashader --cov-append +commands = pytest datashader -k "not test_tiles" -n logical --dist loadgroup --cov=./datashader --cov-append --benchmark-skip [_examples] description = Test that default examples run From 38abbb2390ec7aca50e9e5ce104c37db139ac153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 12:11:01 +0200 Subject: [PATCH 4/9] Skip failing test (for now) --- datashader/tests/test_quadmesh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datashader/tests/test_quadmesh.py b/datashader/tests/test_quadmesh.py index e5c95e9c7..c3c10ad1f 100644 --- a/datashader/tests/test_quadmesh.py +++ b/datashader/tests/test_quadmesh.py @@ -188,7 +188,8 @@ def test_raster_quadmesh_upsamplex_and_downsampley(array_module): assert_eq_xr(res, out) -@pytest.mark.parametrize('array_module', array_modules) +# FIXME: dask.array does not work for this test +@pytest.mark.parametrize('array_module', [array_modules[0], *array_modules[2:]]) def test_raster_quadmesh_autorange_reversed(array_module): c = ds.Canvas(plot_width=8, plot_height=4) da = xr.DataArray( From f2cd9a0e5f53c28da2032144c31cec2c7f02bcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 13:25:10 +0200 Subject: [PATCH 5/9] Also do it for examples --- setup.py | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afedbf5f3..3de23bb0b 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ 'nbconvert', 'nbformat', 'nbsmoke[verify] >0.5', + 'nbval', 'netcdf4', 'pyarrow', 'pytest <8', # Fails lint with IPynbFile is deprecated diff --git a/tox.ini b/tox.ini index 3b190fdd4..8f731a2a7 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,7 @@ commands = pytest datashader -k "not test_tiles" -n logical --dist loadgroup --c description = Test that default examples run deps = .[examples, tests] commands = datashader fetch-data --path=examples --use-test-data - pytest --nbsmoke-run -k ".ipynb" + pytest -n logical --dist loadscope --nbval-lax examples # could add more, to test types of example other than nbs [_examples_extra] From b6877cfee16fdc4882e01d9c929da6d85d28fb93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 14:20:22 +0200 Subject: [PATCH 6/9] Skip tiling example --- examples/conftest.py | 4 +++- pyproject.toml | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/conftest.py b/examples/conftest.py index 1eb79c649..532dabe55 100644 --- a/examples/conftest.py +++ b/examples/conftest.py @@ -1,6 +1,8 @@ import platform -collect_ignore_glob = [] +collect_ignore_glob = [ + "tiling.ipynb", +] # 2023-07-21 with following error: # nbclient.exceptions.CellTimeoutError: A cell timed out while it was being executed, after 300 seconds. diff --git a/pyproject.toml b/pyproject.toml index 83954127c..0ed1d0fdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,6 @@ norecursedirs = 'doc .git dist build _build .ipynb_checkpoints' minversion = "7" xfail_strict = true log_cli_level = "INFO" -# skipping any notebooks that require extra deps -nbsmoke_skip_run = ".*tiling.ipynb$\n.*streaming-aggregation.ipynb$\n.*8_Geography.ipynb$" filterwarnings = [ "ignore:Passing a (SingleBlockManager|BlockManager) to (Series|GeoSeries|DataFrame|GeoDataFrame) is deprecated:DeprecationWarning", # https://github.com/holoviz/spatialpandas/issues/137 "ignore:Accessing the underlying geometries through the `.data`:DeprecationWarning:dask_geopandas.core", # https://github.com/geopandas/dask-geopandas/issues/264 From 3c48b5929d834cfaeff5cb29173788d65528c821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 19:02:45 +0200 Subject: [PATCH 7/9] Fix failing test --- datashader/data_libraries/dask_xarray.py | 4 ++-- datashader/tests/test_quadmesh.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/datashader/data_libraries/dask_xarray.py b/datashader/data_libraries/dask_xarray.py index d062a1a7a..7787e9448 100644 --- a/datashader/data_libraries/dask_xarray.py +++ b/datashader/data_libraries/dask_xarray.py @@ -173,8 +173,8 @@ def dask_raster(glyph, xr_ds, schema, canvas, summary, *, antialias=False, cuda= src_y0, src_y1 = glyph._compute_bounds_from_1d_centers( xr_ds, y_name, maybe_expand=False, orient=False ) - xbinsize = float(xr_ds[x_name][1] - xr_ds[x_name][0]) - ybinsize = float(xr_ds[y_name][1] - xr_ds[y_name][0]) + xbinsize = abs(float(xr_ds[x_name][1] - xr_ds[x_name][0])) + ybinsize = abs(float(xr_ds[y_name][1] - xr_ds[y_name][0])) # Compute scale/translate out_h, out_w = shape diff --git a/datashader/tests/test_quadmesh.py b/datashader/tests/test_quadmesh.py index c3c10ad1f..e5c95e9c7 100644 --- a/datashader/tests/test_quadmesh.py +++ b/datashader/tests/test_quadmesh.py @@ -188,8 +188,7 @@ def test_raster_quadmesh_upsamplex_and_downsampley(array_module): assert_eq_xr(res, out) -# FIXME: dask.array does not work for this test -@pytest.mark.parametrize('array_module', [array_modules[0], *array_modules[2:]]) +@pytest.mark.parametrize('array_module', array_modules) def test_raster_quadmesh_autorange_reversed(array_module): c = ds.Canvas(plot_width=8, plot_height=4) da = xr.DataArray( From 79b58832ad4eb5bdb26d2c1d83a60b57cc6323d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 22 May 2024 19:03:28 +0200 Subject: [PATCH 8/9] Add benchmark skip to examples --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8f731a2a7..ec84d3968 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,7 @@ commands = pytest datashader -k "not test_tiles" -n logical --dist loadgroup --c description = Test that default examples run deps = .[examples, tests] commands = datashader fetch-data --path=examples --use-test-data - pytest -n logical --dist loadscope --nbval-lax examples + pytest -n logical --dist loadscope --nbval-lax examples --benchmark-skip # could add more, to test types of example other than nbs [_examples_extra] From a706938052593e502b7d3bc5d886ede8c5606a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Thu, 23 May 2024 13:06:32 +0200 Subject: [PATCH 9/9] Update numpy 2 deps --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 32d594cdb..fcd99ee42 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,9 @@ 'nbformat', 'nbsmoke[verify] >0.5', 'netcdf4', + 'nbval', + 'psutil', + 'pytest-xdist', # 'pyarrow', 'pytest <8', # Fails lint with IPynbFile is deprecated 'pytest-benchmark',