Parallelize the test suite and fix a test polluted bug

datashader/data_libraries/cudf.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
+from contextlib import suppress
 from datashader.data_libraries.pandas import default
 from datashader.core import bypixel
-import cudf
 
 
-@bypixel.pipeline.register(cudf.DataFrame)
 def cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False):
     return default(glyph, df, schema, canvas, summary, antialias=antialias, cuda=True)
+
+
+with suppress(ImportError):
+    import cudf
+
+    cudf_pipeline = bypixel.pipeline.register(cudf.DataFrame)(cudf_pipeline)

datashader/data_libraries/dask_cudf.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
+from contextlib import suppress
 from datashader.data_libraries.dask import dask_pipeline
 from datashader.core import bypixel
-import dask_cudf
 
 
-@bypixel.pipeline.register(dask_cudf.DataFrame)
 def dask_cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False):
     return dask_pipeline(df, schema, canvas, glyph, summary, antialias=antialias, cuda=True)
+
+
+with suppress(ImportError):
+    import dask_cudf
+
+    dask_cudf_pipeline = bypixel.pipeline.register(dask_cudf.DataFrame)(dask_cudf_pipeline)

datashader/datashape/coretypes.py

@@ -254,7 +254,7 @@ def to_numpy_dtype(self):
         return np.dtype('datetime64[us]')
 
 
-_units = set(['ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y'])
+_units = ('ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y')
 
 
 _unit_aliases = {

datashader/tests/benchmarks/__init__.py

@@ -0,0 +1,3 @@
+import pytest
+
+pytestmark = pytest.mark.benchmark

datashader/tests/conftest.py

@@ -0,0 +1,34 @@
+CUSTOM_MARKS = ("benchmark",)
+
+
+def pytest_addoption(parser):
+    for marker in CUSTOM_MARKS:
+        parser.addoption(
+            f"--{marker}",
+            action="store_true",
+            default=False,
+            help=f"Run {marker} related tests",
+        )
+
+
+def pytest_configure(config):
+    for marker in CUSTOM_MARKS:
+        config.addinivalue_line("markers", f"{marker}: {marker} test marker")
+
+
+def pytest_collection_modifyitems(config, items):
+    skipped, selected = [], []
+    markers = [m for m in CUSTOM_MARKS if config.getoption(f"--{m}")]
+    empty = not markers
+    for item in items:
+        if empty and any(m in item.keywords for m in CUSTOM_MARKS):
+            skipped.append(item)
+        elif empty:
+            selected.append(item)
+        elif not empty and any(m in item.keywords for m in markers):
+            selected.append(item)
+        else:
+            skipped.append(item)
+
+    config.hook.pytest_deselected(items=skipped)
+    items[:] = selected

datashader/tests/test_quadmesh.py

@@ -188,7 +188,8 @@ def test_raster_quadmesh_upsamplex_and_downsampley(array_module):
     assert_eq_xr(res, out)
 
 
-@pytest.mark.parametrize('array_module', array_modules)
+# FIXME: dask.array does not work for this test
+@pytest.mark.parametrize('array_module', [array_modules[0], *array_modules[2:]])
 def test_raster_quadmesh_autorange_reversed(array_module):
     c = ds.Canvas(plot_width=8, plot_height=4)
     da = xr.DataArray(

datashader/tests/test_transfer_functions.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from io import BytesIO
 
 import numpy as np
@@ -13,6 +14,8 @@
 coords = dict([('x_axis', [3, 4, 5]), ('y_axis', [0, 1, 2])])
 dims = ['y_axis', 'x_axis']
 
+test_gpu = bool(int(os.getenv("DATASHADER_TEST_GPU", 0)))
+
 # CPU
 def build_agg(array_module=np):
     a = array_module.arange(10, 19, dtype='u4').reshape((3, 3))
@@ -42,12 +45,12 @@ def create_dask_array_np(*args, **kwargs):
     return da.from_array(np.array(*args, **kwargs))
 
 
-try:
+if test_gpu:
     import cupy
     aggs = [build_agg(np), build_agg(cupy), build_agg_dask()]
     arrays = [np.array, cupy.array, create_dask_array_np]
     array_modules = [np, cupy]
-except ImportError:
+else:
     cupy = None
     aggs = [build_agg(np), build_agg_dask()]
     arrays = [np.array, create_dask_array_np]

setup.py

@@ -53,6 +53,8 @@
         'pytest <8',  # Fails lint with IPynbFile is deprecated
         'pytest-benchmark',
         'pytest-cov',
+        'psutil',
+        'pytest-xdist',
         'rasterio',
         'rioxarray',
         'scikit-image',

tox.ini

@@ -17,7 +17,8 @@ commands = flake8
 [_unit]
 description = Run unit tests
 deps = .[tests]
-commands = pytest datashader --cov=./datashader --cov-append
+commands = pytest datashader -n logical --dist loadgroup --cov=./datashader --cov-append --benchmark-skip
+           pytest datashader --benchmark --cov=./datashader --cov-append
 
 [_unit_deploy]
 description = Run unit tests without coverage
@@ -27,7 +28,7 @@ commands = pytest datashader
 [_unit_nojit]
 description = Run select unit tests with numba jit disabled
 deps = .[tests]
-commands = pytest datashader -k "not benchmarks and not test_tiles" --cov=./datashader --cov-append
+commands = pytest datashader -k "not test_tiles" -n logical --dist loadgroup --cov=./datashader --cov-append --benchmark-skip
 
 [_examples]
 description = Test that default examples run