Ignore Kernel died failures in example tests (#5892)

holoviz · Sep 17, 2023 · 0b76098 · 0b76098
1 parent d7bf91f
commit 0b76098
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 43 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -14,6 +14,20 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  SETUPTOOLS_ENABLE_FEATURES: "legacy-editable"
+  DISPLAY: ":99.0"
+  PYTHONIOENCODING: "utf-8"
+  MPLBACKEND: "Agg"
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  OMP_NUM_THREADS: 1
+  OPENBLAS_NUM_THREADS: 1
+  MKL_NUM_THREADS: 1
+  VECLIB_MAXIMUM_THREADS: 1
+  NUMEXPR_NUM_THREADS: 1
+  NUMBA_NUM_THREADS: 1
+  PYDEVD_DISABLE_FILE_VALIDATION: 1
+
 jobs:
   pre_commit:
     name: Run pre-commit hooks
@@ -30,43 +44,35 @@ jobs:
           key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
       - name: pre-commit
         uses: pre-commit/action@v3.0.0
-  test_suite:
-    name: Pytest on ${{ matrix.python-version }}, Bokeh ${{ matrix.bokeh-version }}, ${{ matrix.os }}
+  unit_test_suite:
+    name: Unit tests on Python ${{ matrix.python-version }}, ${{ matrix.os }}
     needs: [pre_commit]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
         python-version: ['3.8', '3.11']
-        bokeh-version: ['3']
+        include:
+          - os: 'ubuntu-latest'
+            python-version: '3.9'
+          - os: 'ubuntu-latest'
+            python-version: '3.10'
     timeout-minutes: 120
     defaults:
       run:
         shell: bash -el {0}
     env:
-      DESC: "Python ${{ matrix.python-version }} - Bokeh ${{ matrix.bokeh-version }} tests"
+      DESC: "Python ${{ matrix.python-version }}, ${{ matrix.os }} unit tests"
       PYTHON_VERSION: ${{ matrix.python-version }}
-      SETUPTOOLS_ENABLE_FEATURES: "legacy-editable"
-      DISPLAY: ":99.0"
-      PYTHONIOENCODING: "utf-8"
-      MPLBACKEND: "Agg"
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      OMP_NUM_THREADS: 1
-      OPENBLAS_NUM_THREADS: 1
-      MKL_NUM_THREADS: 1
-      VECLIB_MAXIMUM_THREADS: 1
-      NUMEXPR_NUM_THREADS: 1
-      NUMBA_NUM_THREADS: 1
-      PYDEVD_DISABLE_FILE_VALIDATION: 1
     steps:
       - uses: holoviz-dev/holoviz_tasks/install@v0.1a15
         with:
-          name: unit_test_suite_bokeh${{ matrix.bokeh-version }}
+          name: unit_test_suite
           python-version: ${{ matrix.python-version }}
           channel-priority: strict
           channels: pyviz/label/dev,conda-forge,nodefaults
-          envs: "-o flakes -o tests -o examples_tests -o bokeh${{ matrix.bokeh-version }}"
+          envs: "-o flakes -o tests -o examples_tests"
           cache: true
           conda-update: true
         id: install
@@ -81,31 +87,27 @@ jobs:
       - name: test examples
         run: |
           conda activate test-environment
-          mkdir -p ~/.jupyter/
-          echo "c.ExecutePreprocessor.startup_timeout=600" >> ~/.jupyter/jupyter_nbconvert_config.py
           doit test_examples
       - name: codecov
         run: |
           conda activate test-environment
           codecov
   ui_test_suite:
-    name: UI tests on ${{ matrix.os }} with Python 3.9
+    name: UI tests on Python ${{ matrix.python-version }}, ${{ matrix.os }}
     needs: [pre_commit]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest']
+        python-version: ['3.9']
     timeout-minutes: 60
     defaults:
       run:
         shell: bash -el {0}
     env:
-      DESC: "Python ${{ matrix.python-version }} tests"
-      PYTHONIOENCODING: "utf-8"
+      DESC: "Python ${{ matrix.python-version }}, ${{ matrix.os }} UI tests"
       PANEL_LOG_LEVEL: info
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      SETUPTOOLS_ENABLE_FEATURES: "legacy-editable"
       # Without this env var `doit env_create ...` uses by default
       # the `pyviz` channel, except that we don't want to configure
       # it as one of the sources.
@@ -114,7 +116,7 @@ jobs:
       - uses: holoviz-dev/holoviz_tasks/install@v0.1a15
         with:
           name: ui_test_suite
-          python-version: 3.9
+          python-version: ${{ matrix.python-version }}
           channels: pyviz/label/dev,bokeh,conda-forge,nodefaults
           envs: "-o recommended -o tests -o build"
           cache: true
@@ -134,8 +136,8 @@ jobs:
           files: ./coverage.xml
           flags: ui-tests
           fail_ci_if_error: false # optional (default = false)
-  core_tests:
-    name: Core tests on ${{ matrix.python-version }}, ${{ matrix.os }}
+  core_test_suite:
+    name: Core tests on Python ${{ matrix.python-version }}, ${{ matrix.os }}
     needs: [pre_commit]
     runs-on: ${{ matrix.os }}
     strategy:
@@ -148,20 +150,8 @@ jobs:
       run:
         shell: bash -el {0}
     env:
-      DESC: "Python ${{ matrix.python-version }} core tests"
+      DESC: "Python ${{ matrix.python-version }}, ${{ matrix.os }} core tests"
       PYTHON_VERSION: ${{ matrix.python-version }}
-      SETUPTOOLS_ENABLE_FEATURES: "legacy-editable"
-      DISPLAY: ":99.0"
-      PYTHONIOENCODING: "utf-8"
-      MPLBACKEND: "Agg"
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      OMP_NUM_THREADS: 1
-      OPENBLAS_NUM_THREADS: 1
-      MKL_NUM_THREADS: 1
-      VECLIB_MAXIMUM_THREADS: 1
-      NUMEXPR_NUM_THREADS: 1
-      NUMBA_NUM_THREADS: 1
-      PYDEVD_DISABLE_FILE_VALIDATION: 1
     steps:
       - uses: holoviz-dev/holoviz_tasks/install@v0.1a15
         with:

diff --git a/examples/conftest.py b/examples/conftest.py
@@ -38,3 +38,23 @@
         "gallery/demos/*/bachelors_degrees_by_gender.ipynb",
         "gallery/demos/*/topographic_hillshading.ipynb",
     ]
+
+
+def pytest_runtest_makereport(item, call):
+    """
+    Skip tests that fail because "the kernel died before replying to kernel_info"
+    this is a common error when running the example tests in CI.
+
+    Inspired from: https://stackoverflow.com/questions/32451811
+
+    """
+    from _pytest.runner import pytest_runtest_makereport
+    tr = pytest_runtest_makereport(item, call)
+
+    if call.excinfo is not None:
+        msg = "Kernel died before replying to kernel_info"
+        if call.excinfo.type == RuntimeError and call.excinfo.value.args[0] == msg:
+            tr.outcome = 'skipped'
+            tr.wasxfail = f"reason: {msg}"
+
+    return tr
diff --git a/scripts/check_latest_packages.py b/scripts/check_latest_packages.py
@@ -6,7 +6,7 @@
 
 
 def main(*packages):
-    allowed_date = date.today() - timedelta(days=2)
+    allowed_date = date.today() - timedelta(days=5)
     is_latest = True
     for package in sorted(packages):
         url = f"https://pypi.org/pypi/{package}/json"
@@ -23,7 +23,7 @@ def main(*packages):
 
         version_check = Version(current) >= Version(latest)
         date_check = allowed_date >= latest_release_date
-        is_latest &= version_check and date_check
+        is_latest &= version_check or date_check
 
         print(
             f"Package: {package:<10} Current: {current:<7} ({current_release_date})\tLatest: {latest:<7} ({latest_release_date})"