Add gpu marker and test both Classic/dask-expr Dask DataFrames

.github/workflows/test.yaml

@@ -34,7 +34,6 @@ env:
   VECLIB_MAXIMUM_THREADS: 1
   NUMEXPR_NUM_THREADS: 1
   PYDEVD_DISABLE_FILE_VALIDATION: 1
-  DASK_DATAFRAME__QUERY_PLANNING: false
 
 jobs:
   pre_commit:
@@ -159,11 +158,6 @@ jobs:
           conda activate test-environment
           python -c "import numba; print('Numba', numba.__version__)"
           python -c "import numpy; print('Numpy', numpy.__version__)"
-      - name: doit test_lint
-        if: runner.os == 'Linux'
-        run: |
-          conda activate test-environment
-          doit test_lint
       - name: doit test_unit
         run: |
           conda activate test-environment
@@ -175,59 +169,11 @@ jobs:
         env:
           NUMBA_DISABLE_JIT: 1
       - name: doit test_examples
+        env:
+          DASK_DATAFRAME__QUERY_PLANNING: false
         run: |
           conda activate test-environment
           doit test_examples
-      - name: codecov
-        run: |
-          conda activate test-environment
-          codecov
-
-  test_pip:
-    name: Pip tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}
-    needs: [pre_commit]
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ['ubuntu-latest', 'macos-latest']
-        python-version: ["3.12"]
-    steps:
-      - name: Checkout source
-        uses: actions/checkout@v3
+      - uses: codecov/codecov-action@v4
         with:
-          fetch-depth: 0
-      - name: Install Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Update setuptools
-        run: |
-          pip install --upgrade setuptools
-      - name: Install pyctdev
-        run: |
-          pip install pyctdev
-      - name: doit develop_install
-        run: |
-          doit ecosystem=pip develop_install -o tests -o examples
-      - name: doit env_capture
-        run: |
-          doit ecosystem=pip env_capture
-      - name: doit test_lint
-        if: runner.os == 'Linux'
-        run: |
-          doit ecosystem=pip test_lint
-      - name: doit test_unit
-        run: |
-          doit ecosystem=pip test_unit
-      - name: doit test_unit_nojit
-        run: |
-          doit ecosystem=pip test_unit_nojit
-        env:
-          NUMBA_DISABLE_JIT: 1
-      - name: doit test_examples
-        run: |
-          doit ecosystem=pip test_examples
-      - name: codecov
-        run: |
-          codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

datashader/data_libraries/cudf.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
+from contextlib import suppress
 from datashader.data_libraries.pandas import default
 from datashader.core import bypixel
-import cudf
 
 
-@bypixel.pipeline.register(cudf.DataFrame)
 def cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False):
     return default(glyph, df, schema, canvas, summary, antialias=antialias, cuda=True)
+
+
+with suppress(ImportError):
+    import cudf
+
+    cudf_pipeline = bypixel.pipeline.register(cudf.DataFrame)(cudf_pipeline)

datashader/data_libraries/dask.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from contextlib import suppress
+
 import numpy as np
 import pandas as pd
 import dask
@@ -30,7 +32,6 @@ def _dask_compat(df):
     return getattr(df, 'optimize', lambda: df)()
 
 
-@bypixel.pipeline.register(dd.DataFrame)
 def dask_pipeline(df, schema, canvas, glyph, summary, *, antialias=False, cuda=False):
     dsk, name = glyph_dispatch(glyph, df, schema, canvas, summary, antialias=antialias, cuda=cuda)
 
@@ -50,6 +51,15 @@ def dask_pipeline(df, schema, canvas, glyph, summary, *, antialias=False, cuda=F
     return scheduler(dsk, name)
 
 
+# Classic Dask.DataFrame
+bypixel.pipeline.register(dd.core.DataFrame)(dask_pipeline)
+
+with suppress(ImportError):
+    import dask_expr
+
+    bypixel.pipeline.register(dask_expr.DataFrame)(dask_pipeline)
+
+
 def shape_bounds_st_and_axis(df, canvas, glyph):
     if not canvas.x_range or not canvas.y_range:
         x_extents, y_extents = glyph.compute_bounds_dask(df)

datashader/data_libraries/dask_cudf.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
+from contextlib import suppress
 from datashader.data_libraries.dask import dask_pipeline
 from datashader.core import bypixel
-import dask_cudf
 
 
-@bypixel.pipeline.register(dask_cudf.DataFrame)
 def dask_cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False):
     return dask_pipeline(df, schema, canvas, glyph, summary, antialias=antialias, cuda=True)
+
+
+with suppress(ImportError):
+    import dask_cudf
+
+    dask_cudf_pipeline = bypixel.pipeline.register(dask_cudf.DataFrame)(dask_cudf_pipeline)

datashader/data_libraries/dask_xarray.py

@@ -173,8 +173,8 @@ def dask_raster(glyph, xr_ds, schema, canvas, summary, *, antialias=False, cuda=
     src_y0, src_y1 = glyph._compute_bounds_from_1d_centers(
         xr_ds, y_name, maybe_expand=False, orient=False
     )
-    xbinsize = float(xr_ds[x_name][1] - xr_ds[x_name][0])
-    ybinsize = float(xr_ds[y_name][1] - xr_ds[y_name][0])
+    xbinsize = abs(float(xr_ds[x_name][1] - xr_ds[x_name][0]))
+    ybinsize = abs(float(xr_ds[y_name][1] - xr_ds[y_name][0]))
 
     # Compute scale/translate
     out_h, out_w = shape

datashader/datashape/coretypes.py

@@ -254,7 +254,7 @@ def to_numpy_dtype(self):
         return np.dtype('datetime64[us]')
 
 
-_units = set(['ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y'])
+_units = ('ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y')
 
 
 _unit_aliases = {

datashader/tests/benchmarks/__init__.py

@@ -0,0 +1,3 @@
+import pytest
+
+pytestmark = pytest.mark.benchmark

datashader/tests/benchmarks/test_canvas.py

@@ -1,12 +1,9 @@
 import pytest
-import os
 import numpy as np
 import pandas as pd
 
 import datashader as ds
 
-test_gpu = bool(int(os.getenv("DATASHADER_TEST_GPU", 0)))
-
 
 @pytest.fixture
 def time_series():
@@ -33,7 +30,7 @@ def test_points(benchmark, time_series):
     benchmark(cvs.points, time_series, 'x', 'y')
 
 
-@pytest.mark.skipif(not test_gpu, reason="DATASHADER_TEST_GPU not set")
+@pytest.mark.gpu
 @pytest.mark.benchmark(group="canvas")
 def test_line_gpu(benchmark, time_series):
     from cudf import from_pandas
@@ -42,7 +39,7 @@ def test_line_gpu(benchmark, time_series):
     benchmark(cvs.line, time_series, 'x', 'y')
 
 
-@pytest.mark.skipif(not test_gpu, reason="DATASHADER_TEST_GPU not set")
+@pytest.mark.gpu
 @pytest.mark.benchmark(group="canvas")
 def test_points_gpu(benchmark, time_series):
     from cudf import from_pandas

datashader/tests/conftest.py

@@ -0,0 +1,35 @@
+CUSTOM_MARKS = {"benchmark", "gpu"}
+
+
+def pytest_addoption(parser):
+    for marker in sorted(CUSTOM_MARKS):
+        parser.addoption(
+            f"--{marker}",
+            action="store_true",
+            default=False,
+            help=f"Run {marker} related tests",
+        )
+
+
+def pytest_configure(config):
+    for marker in sorted(CUSTOM_MARKS):
+        config.addinivalue_line("markers", f"{marker}: {marker} test marker")
+
+
+def pytest_collection_modifyitems(config, items):
+    skipped, selected = [], []
+    markers = {m for m in CUSTOM_MARKS if config.getoption(f"--{m}")}
+    empty = not markers
+    for item in items:
+        item_marks = set(item.keywords) & CUSTOM_MARKS
+        if empty and item_marks:
+            skipped.append(item)
+        elif empty:
+            selected.append(item)
+        elif not empty and item_marks == markers:
+            selected.append(item)
+        else:
+            skipped.append(item)
+
+    config.hook.pytest_deselected(items=skipped)
+    items[:] = selected