From 9c2319290e335a4ba37f77ecb44cbe242e36dbc7 Mon Sep 17 00:00:00 2001 From: tonyfast Date: Thu, 7 Dec 2023 20:17:57 -0800 Subject: [PATCH 01/13] warm up the vnu-validator cache before remove existing node impl --- .github/test-environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/test-environment.yml b/.github/test-environment.yml index 26787d2c..7cf0259c 100644 --- a/.github/test-environment.yml +++ b/.github/test-environment.yml @@ -28,4 +28,5 @@ dependencies: - scipy - doit - mkdocs-material - - mkdocstrings[python] \ No newline at end of file + - mkdocstrings[python] + - vnu-validator \ No newline at end of file From 7e90753e6a84361f8e7de417444ccaa75277b54d Mon Sep 17 00:00:00 2001 From: tonyfast Date: Thu, 7 Dec 2023 20:28:47 -0800 Subject: [PATCH 02/13] swap out old invocation with conda based invocation --- tests/test_w3c.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_w3c.py b/tests/test_w3c.py index 9d10eafa..b16d8164 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -6,35 +6,35 @@ import itertools import json import operator +import os import pathlib import re import shlex +import shutil import subprocess +import sys +from pathlib import Path import exceptiongroup from tests.test_smoke import CONFIGURATIONS, get_target_html +WIN = os.name == "nt" EXCLUDE = re.compile( """or with a “role” attribute whose value is “table”, “grid”, or “treegrid”.$""" # https://github.com/validator/validator/issues/1125 ) -@functools.lru_cache(1) -def vnu_jar(): - VNU_JAR = ( - pathlib.Path(subprocess.check_output(shlex.split("npm root vnu-jar")).strip().decode()) - / "vnu-jar/build/dist/vnu.jar" - ) - assert VNU_JAR.exists() - return VNU_JAR +VNU = shutil.which("vnu") or shutil.which("vnu.cmd") +JAVA = Path(shutil.which("java") or shutil.which("java.exe")) +JAR = Path(sys.prefix) / ("Library/lib" if WIN else "lib") / "vnu.jar" def validate_html(*files: pathlib.Path) -> dict: return json.loads( subprocess.check_output( - shlex.split(f"java -jar {vnu_jar()} --stdout --format json --exit-zero-always") + shlex.split(f"{JAVA} -jar {JAR} --stdout --format json --exit-zero-always") + list(files) ).decode() ) From 88b14549dda831d4d6b09fbd9648295a2eddefe1 Mon Sep 17 00:00:00 2001 From: tonyfast Date: Thu, 7 Dec 2023 20:41:02 -0800 Subject: [PATCH 03/13] rm vnu-jar from node install --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ecfae8ec..535c9a72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,7 +59,7 @@ jobs: playwright install --with-deps chromium - name: init node & files run: | - npm install vnu-jar axe-core + npm install axe-core doit copy - name: init dev module run: | From 1ac768f269b8ac48e1b542d6ccbe62875a6e1c69 Mon Sep 17 00:00:00 2001 From: tonyfast Date: Fri, 8 Dec 2023 18:19:14 -0800 Subject: [PATCH 04/13] change vnu invocation from nicks notes --- tests/test_w3c.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_w3c.py b/tests/test_w3c.py index b16d8164..d212df4b 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -6,20 +6,17 @@ import itertools import json import operator -import os import pathlib import re import shlex import shutil import subprocess -import sys from pathlib import Path import exceptiongroup from tests.test_smoke import CONFIGURATIONS, get_target_html -WIN = os.name == "nt" EXCLUDE = re.compile( """or with a “role” attribute whose value is “table”, “grid”, or “treegrid”.$""" # https://github.com/validator/validator/issues/1125 @@ -27,15 +24,12 @@ VNU = shutil.which("vnu") or shutil.which("vnu.cmd") -JAVA = Path(shutil.which("java") or shutil.which("java.exe")) -JAR = Path(sys.prefix) / ("Library/lib" if WIN else "lib") / "vnu.jar" def validate_html(*files: pathlib.Path) -> dict: return json.loads( subprocess.check_output( - shlex.split(f"{JAVA} -jar {JAR} --stdout --format json --exit-zero-always") - + list(files) + shlex.split(f"{VNU} --stdout --format json --exit-zero-always") + list(files) ).decode() ) From 1de6f5e7154e12fa1f5767844918688594cee101 Mon Sep 17 00:00:00 2001 From: tonyfast Date: Sat, 9 Dec 2023 16:14:27 -0800 Subject: [PATCH 05/13] use list for shelling --- tests/test_w3c.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/test_w3c.py b/tests/test_w3c.py index d212df4b..132e469b 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -13,7 +13,13 @@ import subprocess from pathlib import Path +from json import dumps +from logging import getLogger +from pathlib import Path + import exceptiongroup +from pytest import mark, param + from tests.test_smoke import CONFIGURATIONS, get_target_html @@ -21,15 +27,13 @@ """or with a “role” attribute whose value is “table”, “grid”, or “treegrid”.$""" # https://github.com/validator/validator/issues/1125 ) - - VNU = shutil.which("vnu") or shutil.which("vnu.cmd") def validate_html(*files: pathlib.Path) -> dict: return json.loads( subprocess.check_output( - shlex.split(f"{VNU} --stdout --format json --exit-zero-always") + list(files) + [VNU, "--stdout", "--format=json", "--exit-zero-always", *files] ).decode() ) @@ -58,13 +62,6 @@ def raise_if_errors(results, exclude=EXCLUDE): raise exceptiongroup.ExceptionGroup("nu validator errors", exceptions) -from json import dumps -from logging import getLogger -from pathlib import Path - -import exceptiongroup -from pytest import mark, param - HERE = Path(__file__).parent NOTEBOOKS = HERE / "notebooks" EXPORTS = HERE / "exports" From ae6bc7c4775c5a9cc2c07018de7627903140252d Mon Sep 17 00:00:00 2001 From: tonyfast Date: Sat, 9 Dec 2023 22:14:09 -0800 Subject: [PATCH 06/13] split shell cmd --- tests/test_w3c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_w3c.py b/tests/test_w3c.py index 132e469b..97346834 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -33,7 +33,7 @@ def validate_html(*files: pathlib.Path) -> dict: return json.loads( subprocess.check_output( - [VNU, "--stdout", "--format=json", "--exit-zero-always", *files] + [VNU, "--stdout", "--format", "json", "--exit-zero-always", *files] ).decode() ) From efb65dc7c12435dfac218e4c4ac76f08d03f259d Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 10 Dec 2023 10:49:05 -0600 Subject: [PATCH 07/13] normalize test env --- .github/test-environment.yml | 55 ++++++++++++++++++++++-------------- .github/workflows/test.yml | 39 ++++++++++++++++--------- .gitignore | 35 +++++++++++------------ .yarnrc.yml | 25 ++++++++++++++++ package.json | 8 ++++++ yarn.lock | 21 ++++++++++++++ 6 files changed, 131 insertions(+), 52 deletions(-) create mode 100644 .yarnrc.yml create mode 100644 package.json create mode 100644 yarn.lock diff --git a/.github/test-environment.yml b/.github/test-environment.yml index 7cf0259c..a1ad9345 100644 --- a/.github/test-environment.yml +++ b/.github/test-environment.yml @@ -2,31 +2,44 @@ name: test-nbconvert-a11y channels: - conda-forge - microsoft + - nodefaults dependencies: - - python=3.11 + # run + - accessible-pygments + - exceptiongroup + - html5lib + - nbconvert + - python-slugify + - mdit-py-plugins + - linkify-it-py >=1,<3 + - markdown-it-py + # runtimes + - python =3.11 - openjdk + - nodejs =20 + # build + - doit + - hatch + - hatch-vcs - pip - - pip: - - pytest-playwright - - pytest-html - - accessible-pygments - - requests-cache - - markdown-it-py[plugins,linkify] - - python-slugify - - html5lib - - build - - nodejs + - python-build + - yarn =3.6 + # deps + - beautifulsoup4 + - ipywidgets + - matplotlib-base + - mkdocs-material + - mkdocstrings + - numpy - playwright - - nbconvert - pytest - - pytest-xdist + - pytest-html + - pytest-playwright - pytest-sugar - - matplotlib-base - - numpy - - ipywidgets - - beautifulsoup4 + - pytest-xdist + - requests-cache + # report - scipy - - doit - - mkdocs-material - - mkdocstrings[python] - - vnu-validator \ No newline at end of file + - pandas + # audit + - vnu-validator diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 535c9a72..5551a058 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,16 @@ name: pytest nbconvert-a11y, axe test exports, build docs. + on: - - push + push: + branches: [main] + pull_request: + branches: ['*'] + workflow_dispatch: + +env: + # Increase this value to reset cache if environments have not changed + CACHE_NUMBER: 2 + jobs: format: name: format @@ -26,6 +36,7 @@ jobs: echo "~~~diff" >> "${GITHUB_STEP_SUMMARY}" git diff | tee --append "${GITHUB_STEP_SUMMARY}" echo "~~~" >> "${GITHUB_STEP_SUMMARY}" + test: name: test package and accessibility defaults: @@ -38,18 +49,15 @@ jobs: runs-on: ubuntu-latest steps: - name: fetch all history and tags - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache conda - uses: actions/cache@v2 - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - CACHE_NUMBER: 2 + uses: actions/cache@v3 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('.github/test-environment.yml') }} + key: |- + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/test-environment.yml') }} - uses: mamba-org/setup-micromamba@v1 with: environment-file: .github/test-environment.yml @@ -59,11 +67,11 @@ jobs: playwright install --with-deps chromium - name: init node & files run: | - npm install axe-core + yarn doit copy - name: init dev module run: | - pip install -e. + pip install -e . --no-deps --no-build-isolation - name: smoke test run: | # the smoke generate html assets that are used in the accessibility testing. @@ -72,7 +80,7 @@ jobs: pytest tests/test_smoke.py - name: build wheel and sdist run: | - python -m build + pyproject-build - uses: actions/upload-artifact@v3 with: name: dist @@ -88,8 +96,12 @@ jobs: # always build the docs to see what the new versions look like. # continue-on-error: true run: | - pytest --deselect tests/test_smoke.py \ - -n auto --self-contained-html --html=tests/exports/pytest/report.html + pytest \ + -n auto \ + --deselect tests/test_smoke.py \ + --self-contained-html \ + --html=tests/exports/pytest/report.html + publish: name: publish the mkdocs build to github pages needs: [test] @@ -114,6 +126,7 @@ jobs: folder: site # The folder the action should deploy. single-commit: true target-folder: branch/${{ github.ref_name }} + release: name: draft release when tagged if: startsWith(github.ref, 'refs/tags/') diff --git a/.gitignore b/.gitignore index 1a6f1b33..fb0c4fa8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,23 @@ -docs -site -tests/exports -nbconvert_a11y/templates/a11y/light-code.css -nbconvert_a11y/templates/a11y/dark-code.css - -*~ -*# +__pycache__ .#* -*.pyc +.doit.* .python-version -nbconvert_a11y.egg-info -node_modules -__pycache__ -.doit.db.* -nbconvert_a11y/_version.py -tests/outputs/*.html -build/* -tests/out.html *-checkpoint* +*.pyc +*# +*~ +build/* +docs docs/**/*.html docs/**/*.json -settings.json +nbconvert_a11y.egg-info +nbconvert_a11y/_version.py nbconvert_a11y/templates/a11y/axe.js +nbconvert_a11y/templates/a11y/dark-code.css +nbconvert_a11y/templates/a11y/light-code.css +node_modules +settings.json +site +tests/exports +tests/out.html +tests/outputs/*.html diff --git a/.yarnrc.yml b/.yarnrc.yml new file mode 100644 index 00000000..2306181d --- /dev/null +++ b/.yarnrc.yml @@ -0,0 +1,25 @@ +enableImmutableInstalls: false +enableInlineBuilds: false +enableTelemetry: false +httpTimeout: 60000 +nodeLinker: node-modules +npmRegistryServer: https://registry.npmjs.org/ +installStatePath: ./build/.cache/yarn/install-state.gz +globalFolder: ./build/.cache/yarn +cacheFolder: ./build/.cache/yarn +# these messages provide no actionable information, and make non-TTY output +# almost unreadable, masking real dependency-related information +# see: https://yarnpkg.com/advanced/error-codes +logFilters: + - code: YN0006 # SOFT_LINK_BUILD + level: discard + - code: YN0007 # MUST_BUILD + level: discard + - code: YN0008 # MUST_REBUILD + level: discard + - code: YN0013 # FETCH_NOT_CACHED + level: discard + - code: YN0019 # UNUSED_CACHE_ENTRY + level: discard + - code: YN0002 # BOO_PEER_DEPS_LIKE_REACT + level: discard diff --git a/package.json b/package.json new file mode 100644 index 00000000..cb2f891d --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "@deathbeds/nbconvert-a11y", + "private": true, + "version": "0.0.1", + "dependencies": { + "axe-core": "^4.8.2" + } +} diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 00000000..379be6bd --- /dev/null +++ b/yarn.lock @@ -0,0 +1,21 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 6 + cacheKey: 8 + +"@deathbeds/nbconvert-a11y@workspace:.": + version: 0.0.0-use.local + resolution: "@deathbeds/nbconvert-a11y@workspace:." + dependencies: + axe-core: ^4.8.2 + languageName: unknown + linkType: soft + +"axe-core@npm:^4.8.2": + version: 4.8.2 + resolution: "axe-core@npm:4.8.2" + checksum: 8c19f507dabfcb8514e4280c7fc66e85143be303ddb57ec9f119338021228dc9b80560993938003837bda415fde7c07bba3a96560008ffa5f4145a248ed8f5fe + languageName: node + linkType: hard From 530c54b379a27a7b7c700f8bbcb02c78fd3578f5 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 10 Dec 2023 11:04:23 -0600 Subject: [PATCH 08/13] normalize mamba cache --- .github/workflows/test.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5551a058..64d5e4e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,16 +52,10 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v3 - with: - path: ~/conda_pkgs_dir - key: |- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/test-environment.yml') }} - uses: mamba-org/setup-micromamba@v1 with: environment-file: .github/test-environment.yml - cache-environment: true + cache-downloads-key: mamba-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/test-environment.yml') }} - name: init plawright run: | playwright install --with-deps chromium @@ -77,7 +71,7 @@ jobs: # the smoke generate html assets that are used in the accessibility testing. # we run this script to generate assets and test the nbconvert-a11y module. # failures here will stop any docs builds - pytest tests/test_smoke.py + pytest --color=yes tests/test_smoke.py - name: build wheel and sdist run: | pyproject-build @@ -97,6 +91,7 @@ jobs: # continue-on-error: true run: | pytest \ + --color=yes \ -n auto \ --deselect tests/test_smoke.py \ --self-contained-html \ From 9d0bf13c0964b9536e3de42cfba17beacdf37301 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 10 Dec 2023 11:09:55 -0600 Subject: [PATCH 09/13] whitepace, try env cache --- .github/workflows/test.yml | 335 +++++++++++++++++++------------------ 1 file changed, 170 insertions(+), 165 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64d5e4e1..8b7c08ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,165 +1,170 @@ -name: pytest nbconvert-a11y, axe test exports, build docs. - -on: - push: - branches: [main] - pull_request: - branches: ['*'] - workflow_dispatch: - -env: - # Increase this value to reset cache if environments have not changed - CACHE_NUMBER: 2 - -jobs: - format: - name: format - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: "3.12" - cache: pip - cache-dependency-path: pyproject.toml - - name: install dev dependencies - run: python -m pip install --upgrade pip hatch - - name: run formatters - run: | - echo "~~~bash" > "${GITHUB_STEP_SUMMARY}" - hatch run format:code 2>&1 | tee --append "${GITHUB_STEP_SUMMARY}" - echo "~~~" >> "${GITHUB_STEP_SUMMARY}" - - name: print diff - run: | - echo "~~~diff" >> "${GITHUB_STEP_SUMMARY}" - git diff | tee --append "${GITHUB_STEP_SUMMARY}" - echo "~~~" >> "${GITHUB_STEP_SUMMARY}" - - test: - name: test package and accessibility - defaults: - run: - shell: bash -el {0} - strategy: - matrix: - python-version: - - "3.10" - runs-on: ubuntu-latest - steps: - - name: fetch all history and tags - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: .github/test-environment.yml - cache-downloads-key: mamba-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/test-environment.yml') }} - - name: init plawright - run: | - playwright install --with-deps chromium - - name: init node & files - run: | - yarn - doit copy - - name: init dev module - run: | - pip install -e . --no-deps --no-build-isolation - - name: smoke test - run: | - # the smoke generate html assets that are used in the accessibility testing. - # we run this script to generate assets and test the nbconvert-a11y module. - # failures here will stop any docs builds - pytest --color=yes tests/test_smoke.py - - name: build wheel and sdist - run: | - pyproject-build - - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist - - name: mkdocs - run: | - mkdocs build -v - - uses: actions/upload-artifact@v3 - with: - name: site - path: site - - name: a11y tests - # always build the docs to see what the new versions look like. - # continue-on-error: true - run: | - pytest \ - --color=yes \ - -n auto \ - --deselect tests/test_smoke.py \ - --self-contained-html \ - --html=tests/exports/pytest/report.html - - publish: - name: publish the mkdocs build to github pages - needs: [test] - runs-on: ubuntu-latest - steps: - - name: checkout repo - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: site - path: site - - name: Deploy main 🚀 - uses: JamesIves/github-pages-deploy-action@v4 - if: ${{ github.ref_name == 'main' }} - with: - folder: site # The folder the action should deploy. - single-commit: true - - name: Deploy non-main 🚀 - uses: JamesIves/github-pages-deploy-action@v4 - if: ${{ github.ref_name != 'main' }} - with: - folder: site # The folder the action should deploy. - single-commit: true - target-folder: branch/${{ github.ref_name }} - - release: - name: draft release when tagged - if: startsWith(github.ref, 'refs/tags/') - needs: [test] - runs-on: ubuntu-latest - permissions: - id-token: write - contents: write - steps: - - name: fetch contents - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: dist - path: dist - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - cache: pip - cache-dependency-path: pyproject.toml - - name: install twine and pytest - run: | - pip install twine pytest - - name: Publish package distributions to TestPyPI - run: | - twine upload --repository testpypi \ - --user __token__ --password ${{secrets.HATCH_TEST_INDEX_AUTH}} \ - dist/* - - name: install nbconvert-a11y dependencies from test pip - run: | - pip install \ - --index-url 'https://test.pypi.org/simple/' \ - --extra-index-url 'https://pypi.org/simple/' \ - nbconvert-a11y - - name: test test release - run: | - pytest tests/test_smoke.py - - uses: ncipollo/release-action@v1 - with: - artifacts: "dist/.*" - draft: true # does not trigger a created event +name: pytest nbconvert-a11y, axe test exports, build docs. + +on: + push: + branches: [main] + pull_request: + branches: ['*'] + workflow_dispatch: + +env: + # Increase this value to reset cache if environments have not changed + CACHE_NUMBER: 2 + # squash some known warnings + JUPYTER_PLATFORM_DIRS: 1 + +jobs: + format: + name: format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: install dev dependencies + run: python -m pip install --upgrade pip hatch + - name: run formatters + run: | + echo "~~~bash" > "${GITHUB_STEP_SUMMARY}" + hatch run format:code 2>&1 | tee --append "${GITHUB_STEP_SUMMARY}" + echo "~~~" >> "${GITHUB_STEP_SUMMARY}" + - name: print diff + run: | + echo "~~~diff" >> "${GITHUB_STEP_SUMMARY}" + git diff | tee --append "${GITHUB_STEP_SUMMARY}" + echo "~~~" >> "${GITHUB_STEP_SUMMARY}" + + test: + name: test package and accessibility + defaults: + run: + shell: bash -el {0} + strategy: + matrix: + python-version: + - "3.10" + runs-on: ubuntu-latest + steps: + - name: fetch all history and tags + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: .github/test-environment.yml + cache-downloads-key: mamba-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/test-environment.yml') }} + - name: init plawright + run: | + playwright install --with-deps chromium + - name: init node & files + run: | + yarn + doit copy + - name: init dev module + run: | + python3 -m pip install -e . --no-deps --no-build-isolation --ignore-installed + - name: check pip env + run: | + python3 -m pip check + - name: smoke test + run: | + # the smoke generate html assets that are used in the accessibility testing. + # we run this script to generate assets and test the nbconvert-a11y module. + # failures here will stop any docs builds + pytest --color=yes tests/test_smoke.py + - name: build wheel and sdist + run: | + pyproject-build + - uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + - name: mkdocs + run: | + mkdocs build -v + - uses: actions/upload-artifact@v3 + with: + name: site + path: site + - name: a11y tests + # always build the docs to see what the new versions look like. + # continue-on-error: true + run: | + pytest \ + --color=yes \ + -n auto \ + --deselect tests/test_smoke.py \ + --self-contained-html \ + --html=tests/exports/pytest/report.html + + publish: + name: publish the mkdocs build to github pages + needs: [test] + runs-on: ubuntu-latest + steps: + - name: checkout repo + uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: site + path: site + - name: Deploy main 🚀 + uses: JamesIves/github-pages-deploy-action@v4 + if: ${{ github.ref_name == 'main' }} + with: + folder: site # The folder the action should deploy. + single-commit: true + - name: Deploy non-main 🚀 + uses: JamesIves/github-pages-deploy-action@v4 + if: ${{ github.ref_name != 'main' }} + with: + folder: site # The folder the action should deploy. + single-commit: true + target-folder: branch/${{ github.ref_name }} + + release: + name: draft release when tagged + if: startsWith(github.ref, 'refs/tags/') + needs: [test] + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + steps: + - name: fetch contents + uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: dist + path: dist + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: pyproject.toml + - name: install twine and pytest + run: | + pip install twine pytest + - name: Publish package distributions to TestPyPI + run: | + twine upload --repository testpypi \ + --user __token__ --password ${{secrets.HATCH_TEST_INDEX_AUTH}} \ + dist/* + - name: install nbconvert-a11y dependencies from test pip + run: | + pip install \ + --index-url 'https://test.pypi.org/simple/' \ + --extra-index-url 'https://pypi.org/simple/' \ + nbconvert-a11y + - name: test test release + run: | + pytest tests/test_smoke.py + - uses: ncipollo/release-action@v1 + with: + artifacts: "dist/.*" + draft: true # does not trigger a created event From b48818ede9177faa8dec760e2c41b8b95923ba84 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 10 Dec 2023 13:07:56 -0600 Subject: [PATCH 10/13] add vnu server fixtures --- .github/test-environment.yml | 6 +- .github/workflows/test.yml | 1 - nbconvert_a11y/exporter.py | 26 +- nbconvert_a11y/pytest_axe.py | 7 +- .../templates/a11y/static/theme/__main__.py | 1 - pyproject.toml | 2 +- tests/test_a11y_baseline.py | 7 +- tests/test_a11y_settings.py | 5 +- tests/test_color_themes.py | 14 +- tests/test_smoke.py | 9 +- tests/test_third.py | 9 +- tests/test_w3c.py | 257 ++++++++++++++---- 12 files changed, 250 insertions(+), 94 deletions(-) diff --git a/.github/test-environment.yml b/.github/test-environment.yml index a1ad9345..3ff5f8b6 100644 --- a/.github/test-environment.yml +++ b/.github/test-environment.yml @@ -24,6 +24,10 @@ dependencies: - pip - python-build - yarn =3.6 + # audit + - vnu-validator + # lint + - ruff # deps - beautifulsoup4 - ipywidgets @@ -41,5 +45,3 @@ dependencies: # report - scipy - pandas - # audit - - vnu-validator diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b7c08ac..f6af11b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,7 +57,6 @@ jobs: - uses: mamba-org/setup-micromamba@v1 with: environment-file: .github/test-environment.yml - cache-downloads-key: mamba-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/test-environment.yml') }} - name: init plawright run: | playwright install --with-deps chromium diff --git a/nbconvert_a11y/exporter.py b/nbconvert_a11y/exporter.py index c007cd7f..0032deba 100644 --- a/nbconvert_a11y/exporter.py +++ b/nbconvert_a11y/exporter.py @@ -4,18 +4,18 @@ """ import builtins -from io import StringIO import json from contextlib import suppress from datetime import datetime from functools import lru_cache +from io import StringIO from pathlib import Path import bs4 -from nbconvert import Exporter import nbformat.v4 import pygments from bs4 import BeautifulSoup +from nbconvert import Exporter from nbconvert.exporters.html import HTMLExporter from traitlets import Bool, CUnicode, Enum, Unicode @@ -138,7 +138,7 @@ def from_notebook_node(self, nb, resources=None, **kw): return super().from_notebook_node(nb, resources, **kw) def post_process_html(self, body): - """a final pass at the exported html to add table of contents, heading links, and other a11y affordances.""" + """A final pass at the exported html to add table of contents, heading links, and other a11y affordances.""" soup = soupify(body) describe_main(soup) heading_links(soup) @@ -180,12 +180,12 @@ def get_markdown_renderer(): def get_markdown(md, **kwargs): - """exporter markdown as html""" + """Exporter markdown as html""" return get_markdown_renderer().render("".join(md), **kwargs) def highlight(code, lang="python", attrs=None, experimental=True): - """highlight code blocks""" + """Highlight code blocks""" import html import pygments @@ -216,7 +216,7 @@ def soupify(body: str) -> BeautifulSoup: def mdtoc(html): - """create a table of contents in markdown that will be converted to html""" + """Create a table of contents in markdown that will be converted to html""" import io toc = io.StringIO() @@ -236,12 +236,12 @@ def mdtoc(html): def toc(html): - """create an html table of contents""" + """Create an html table of contents""" return get_markdown(mdtoc(html)) def heading_links(html): - """convert headings into links""" + """Convert headings into links""" for header in html.select(":is(h1,h2,h3,h4,h5,h6):not([role])"): id = header.attrs.get("id") if not id: @@ -271,22 +271,22 @@ def count_cell_loc(cell): def count_loc(nb): - """count total significant lines of code in the document""" + """Count total significant lines of code in the document""" return sum(map(count_cell_loc, nb.cells)) def count_outputs(nb): - """count total number of cell outputs""" + """Count total number of cell outputs""" return sum(map(len, (x.get("outputs", "") for x in nb.cells))) def count_code_cells(nb): - """count total number of code cells""" + """Count total number of code cells""" return len([None for x in nb.cells if x["cell_type"] == "code"]) def describe_main(soup): - """add REFIDs to aria-describedby""" + """Add REFIDs to aria-describedby""" x = soup.select_one("#toc > details > summary") if x: x.attrs["aria-describedby"] = soup.select_one("main").attrs[ @@ -295,7 +295,7 @@ def describe_main(soup): def ordered(nb) -> str: - """measure if the notebook is ordered""" + """Measure if the notebook is ordered""" start = 0 for cell in nb.cells: if cell["cell_type"] == "code": diff --git a/nbconvert_a11y/pytest_axe.py b/nbconvert_a11y/pytest_axe.py index 55aeabb4..1e963fe6 100644 --- a/nbconvert_a11y/pytest_axe.py +++ b/nbconvert_a11y/pytest_axe.py @@ -7,9 +7,8 @@ # requires node and axe # requires playwright -from ast import Not -from collections import defaultdict import dataclasses +from collections import defaultdict from functools import lru_cache, partial from json import dumps, loads from pathlib import Path @@ -93,7 +92,7 @@ class AxeOptions(Base): def get_npm_directory(package, data=False): - """get the path of an npm package in the environment""" + """Get the path of an npm package in the environment""" try: info = loads(check_output(split(f"npm ls --long --depth 0 --json {quote(package)}"))) except CalledProcessError: @@ -152,7 +151,7 @@ def configure(self, **config): return self def reset(self): - self.page.evaluate(f"""window.axe.reset()""") + self.page.evaluate("""window.axe.reset()""") return self def __enter__(self): diff --git a/nbconvert_a11y/templates/a11y/static/theme/__main__.py b/nbconvert_a11y/templates/a11y/static/theme/__main__.py index 441120eb..55b2e316 100644 --- a/nbconvert_a11y/templates/a11y/static/theme/__main__.py +++ b/nbconvert_a11y/templates/a11y/static/theme/__main__.py @@ -2,7 +2,6 @@ from pathlib import Path - HERE = Path(__file__).parent themes = """a11y-dark diff --git a/pyproject.toml b/pyproject.toml index eec6770a..5d001060 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,7 +201,7 @@ exclude = [ ] [tool.ruff.lint] -isort.known-first-party = ["importnb"] +isort.known-first-party = ["importnb", "nbconvert_a11y", "tests"] ignore = ["D203", "D213", "COM812", "ISC001"] select = [ "A", diff --git a/tests/test_a11y_baseline.py b/tests/test_a11y_baseline.py index 0c31e4b3..7d217258 100644 --- a/tests/test_a11y_baseline.py +++ b/tests/test_a11y_baseline.py @@ -4,13 +4,12 @@ * test the accessibility of nbconvert-a11y dialogs """ -from json import dumps from pathlib import Path from pytest import mark, param -from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX -from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, SKIPCI, get_target_html +from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX +from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html SA11Y = "sa11y-control-panel" @@ -41,7 +40,7 @@ ], ) def test_axe(axe, config, notebook): - """verify the baseline templates satisify all rules update AAA. + """Verify the baseline templates satisify all rules update AAA. any modifications to the template can only degrade accessibility. this baseline is critical for adding more features. all testing piles diff --git a/tests/test_a11y_settings.py b/tests/test_a11y_settings.py index 58930c12..9da1c7e8 100644 --- a/tests/test_a11y_settings.py +++ b/tests/test_a11y_settings.py @@ -10,10 +10,11 @@ LORENZ_EXECUTED = get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb") -@fixture + +@fixture() def lorenz(page): axe = Axe(page=page, url=LORENZ_EXECUTED.as_uri()) - yield axe.configure() + return axe.configure() @mark.parametrize( diff --git a/tests/test_color_themes.py b/tests/test_color_themes.py index 1f850b1e..a595e02e 100644 --- a/tests/test_color_themes.py +++ b/tests/test_color_themes.py @@ -1,9 +1,9 @@ from nbconvert import get_exporter -from pytest import fixture, mark, param, xfail -from nbconvert_a11y.exporter import THEMES +from pytest import fixture -from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, PYGMENTS, AllOf, Axe, Violation -from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html +from nbconvert_a11y.exporter import THEMES +from nbconvert_a11y.pytest_axe import Axe +from tests.test_smoke import NOTEBOOKS LORENZ = NOTEBOOKS / "lorenz-executed.ipynb" @@ -14,14 +14,14 @@ def exporter(request): e.color_theme = request.param e.include_settings = True e.wcag_priority = "AA" - yield e + return e -@fixture +@fixture() def lorenz(page, tmp_path, exporter): tmp = tmp_path / f"{exporter.color_theme}.html" tmp.write_text(exporter.from_filename(LORENZ)[0]) - yield Axe(page=page, url=tmp.absolute().as_uri()).configure() + return Axe(page=page, url=tmp.absolute().as_uri()).configure() def test_dark_themes(lorenz): diff --git a/tests/test_smoke.py b/tests/test_smoke.py index d8ff661d..768e96ef 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -12,12 +12,10 @@ from pathlib import Path from shutil import copyfile +import jupyter_core.paths import nbconvert.nbconvertapp from pytest import mark, param -import nbconvert_a11y -import jupyter_core.paths - from nbconvert_a11y.exporter import soupify SKIP_BASELINE = "baseline tests skipped locally" @@ -100,7 +98,10 @@ def test_export_notebooks(config, notebook): TARGET.write_text(html) LOGGER.debug(f"writing html to {TARGET}") -@mark.parametrize("target", [get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb")]) + +@mark.parametrize( + "target", [get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb")] +) def test_a11y_template_content(target): soup = soupify(target.read_text()) diff --git a/tests/test_third.py b/tests/test_third.py index 1b7f0e00..cf8c345d 100644 --- a/tests/test_third.py +++ b/tests/test_third.py @@ -9,8 +9,9 @@ from pathlib import Path from unittest import TestCase -from pytest import fixture, skip, mark -from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, NO_ALT, PYGMENTS, AllOf, Violation +from pytest import fixture, mark, skip + +from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, NO_ALT, PYGMENTS, AllOf, Violation from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html # only run these tests when the CI environment variables are defined. @@ -35,7 +36,7 @@ def test_all(self): @xfail(reason="the default pygments theme has priority AA and AAA color contrast issues.") def test_highlight_pygments(self): - """the default template has two serious color contrast violations. + """The default template has two serious color contrast violations. an issue needs to be opened or referenced. """ @@ -47,7 +48,7 @@ def test_highlight_pygments(self): @xfail(reason="widgets have not recieved a concerted effort.") def test_widget_display(self): - """the simple lorenz widget generates one minor and one serious accessibility violation.""" + """The simple lorenz widget generates one minor and one serious accessibility violation.""" raise self.axe.run({"include": [JUPYTER_WIDGETS], "exclude": [NO_ALT]}).raises_allof( Violation["minor-focus-order-semantics"], Violation["serious-aria-input-field-name"], diff --git a/tests/test_w3c.py b/tests/test_w3c.py index 97346834..5af80130 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -1,67 +1,30 @@ # requires node # requires jvm - import collections import functools import itertools -import json import operator -import pathlib +import os import re -import shlex import shutil -import subprocess -from pathlib import Path - +import socket +import sys +import time +import uuid from json import dumps from logging import getLogger from pathlib import Path +from subprocess import Popen +from typing import Any, Callable, Dict, Generator, Tuple +from urllib.request import urlopen import exceptiongroup +import pytest +import requests from pytest import mark, param - from tests.test_smoke import CONFIGURATIONS, get_target_html -EXCLUDE = re.compile( - """or with a “role” attribute whose value is “table”, “grid”, or “treegrid”.$""" - # https://github.com/validator/validator/issues/1125 -) -VNU = shutil.which("vnu") or shutil.which("vnu.cmd") - - -def validate_html(*files: pathlib.Path) -> dict: - return json.loads( - subprocess.check_output( - [VNU, "--stdout", "--format", "json", "--exit-zero-always", *files] - ).decode() - ) - - -def organize_validator_results(results): - collect = collections.defaultdict(functools.partial(collections.defaultdict, list)) - for (error, msg), group in itertools.groupby( - results["messages"], key=operator.itemgetter("type", "message") - ): - for item in group: - collect[error][msg].append(item) - return collect - - -def raise_if_errors(results, exclude=EXCLUDE): - collect = organize_validator_results(results) - exceptions = [] - for msg in collect["error"]: - if not exclude or not exclude.search(msg): - exceptions.append( - exceptiongroup.ExceptionGroup( - msg, [Exception(x["extract"]) for x in collect["error"][msg]] - ) - ) - if exceptions: - raise exceptiongroup.ExceptionGroup("nu validator errors", exceptions) - - HERE = Path(__file__).parent NOTEBOOKS = HERE / "notebooks" EXPORTS = HERE / "exports" @@ -75,6 +38,20 @@ def raise_if_errors(results, exclude=EXCLUDE): INVALID_MARKUP = mark.xfail(reason="invalid html markup") + +ENV_JAVA_PATH = "NBA11Y_JAVA_PATH" +ENV_VNU_JAR_PATH = "NBA11Y_VNU_PATH" +ENV_VNU_SERVER_URL = "NBA11Y_VNU_SERVER_URL" + +TVnuResults = Dict[str, Any] +TVnuValidator = Callable[[Path], TVnuResults] + + +EXCLUDE = re.compile( + """or with a “role” attribute whose value is “table”, “grid”, or “treegrid”.$""" + # https://github.com/validator/validator/issues/1125 +) + htmls = mark.parametrize( "html", [ @@ -98,11 +75,189 @@ def raise_if_errors(results, exclude=EXCLUDE): @htmls -def test_baseline_w3c(page, html): - result = validate_html(html) +def test_baseline_w3c(html: Path, an_html_validator: "TVnuValidator") -> None: + result = an_html_validator(html) VALIDATOR.mkdir(parents=True, exist_ok=True) audit = VALIDATOR / html.with_suffix(".json").name - LOGGER.info(f"""writing {audit} with {len(result.get("violations", ""))} violations""") + violations = result.get("violations", "") + LOGGER.info(f"""writing {audit} with {len(violations)} violations""") audit.write_text(dumps(result)) - raise_if_errors(result) + + +# fixtures +@pytest.fixture(scope="session") +def an_html_validator(a_vnu_server_url: str) -> TVnuValidator: + """Wrap the nvu validator REST API in a synchronous request + + https://github.com/validator/validator/wiki/Service-%C2%BB-Input-%C2%BB-POST-body + """ + + def post(path: Path) -> TVnuResults: + url = f"{a_vnu_server_url}?out=json" + data = path.read_bytes() + headers = {"Content-Type": "text/html"} + res = requests.post(url, data, headers=headers) + return res.json() + + return post + + +@pytest.fixture(scope="session") +def a_vnu_server_url( + worker_id: str, tmp_path_factory: pytest.TempPathFactory +) -> Generator[None, None, str]: + """Get the URL for a running VNU server.""" + url: str | None = os.environ.get(ENV_VNU_SERVER_URL) + + if url is not None: + return url + + proc: Popen | None = None + owns_lock = False + proto = "http" + host = "127.0.0.1" + root_tmp_dir = tmp_path_factory.getbasetemp().parent + lock_dir = root_tmp_dir / "vnu_server" + needs_lock = lock_dir / f"test-{uuid.uuid4()}" + + if worker_id == "master": + port, url, proc = _start_vnu_server(proto, host) + owns_lock = True + else: + port = None + retries = 10 + + try: + lock_dir.mkdir() + owns_lock = True + except: + pass + + needs_lock.mkdir() + + if owns_lock: + port, url, proc = _start_vnu_server(proto, host) + (lock_dir / f"port-{port}").mkdir() + else: + while retries: + retries -= 1 + try: + port = int(next(lock_dir.glob("port-*")).name.split("-")[-1]) + url = f"{proto}://{host}:{port}" + except: + time.sleep(1) + if port is None and not retries: + raise RuntimeError("Never started vnu server") + + yield url + + shutil.rmtree(needs_lock) + + if owns_lock: + while True: + needs = [*lock_dir.glob("test-*")] + if needs: + time.sleep(1) + continue + break + + print(f"... tearing down vnu server at {url}") + proc.terminate() + shutil.rmtree(lock_dir) + + +# utilities +def organize_validator_results(results): + collect = collections.defaultdict(functools.partial(collections.defaultdict, list)) + for (error, msg), group in itertools.groupby( + results["messages"], key=operator.itemgetter("type", "message") + ): + for item in group: + collect[error][msg].append(item) + return collect + + +def raise_if_errors(results, exclude=EXCLUDE): + collect = organize_validator_results(results) + exceptions = [] + for msg in collect["error"]: + if not exclude or not exclude.search(msg): + exceptions.append( + exceptiongroup.ExceptionGroup( + msg, [Exception(x["extract"]) for x in collect["error"][msg]] + ) + ) + if exceptions: + raise exceptiongroup.ExceptionGroup("nu validator errors", exceptions) + + +def _start_vnu_server(proto: str, host: str) -> Tuple[str, Popen]: + """Start a vnu HTTP server.""" + port = get_an_unused_port() + url = f"{proto}://{host}:{port}/" + server_args = get_vnu_args(host, port) + url = f"{proto}://{host}:{port}" + print(f"... starting vnu server at {url}") + print(">>>", "\t".join(server_args)) + proc = Popen(server_args) + wait_for_vnu_to_start(url) + print(f"... vnu server started at {url}") + + return port, url, proc + + +def wait_for_vnu_to_start(url: str, retries: int = 5, warmup: int = 5, sleep: int = 1): + last_error = None + + time.sleep(warmup) + + while retries: + retries -= 1 + try: + return urlopen(url, timeout=warmup) + except Exception as err: + last_error = err + time.sleep(sleep) + + raise RuntimeError(f"{last_error}") + + +def get_vnu_args(host: str, port: int): + win = os.name == "nt" + + java = Path(os.environ.get(ENV_JAVA_PATH, shutil.which("java") or shutil.which("java.exe"))) + jar = Path( + os.environ.get( + ENV_JAVA_PATH, (Path(sys.prefix) / ("Library/lib" if win else "lib") / "vnu.jar") + ) + ) + + if any(not j.exists() for j in [java, jar]): + raise RuntimeError( + "Failed to find java or vnu.jar:\b" + f" - {java.exists()} {java}" + "\n" + f" - {jar.exists()} {jar}" + ) + + server_args = [ + java, + "-cp", + jar, + f"-Dnu.validator.servlet.bind-address={host}", + "nu.validator.servlet.Main", + port, + ] + + return list(map(str, server_args)) + + +def get_an_unused_port() -> Callable[[], int]: + """Find an unused network port (could still create race conditions).""" + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + s.listen(1) + port = s.getsockname()[1] + s.close() + return port From 5cf0f2ceb797ad61bb95b5eaec5610782ffd8262 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 10 Dec 2023 13:07:56 -0600 Subject: [PATCH 11/13] add vnu server fixtures --- .github/test-environment.yml | 6 +- .github/workflows/test.yml | 1 - nbconvert_a11y/audit.py | 12 +- nbconvert_a11y/exporter.py | 26 +- nbconvert_a11y/pytest_axe.py | 7 +- .../templates/a11y/static/theme/__main__.py | 1 - pyproject.toml | 2 +- tests/test_a11y_baseline.py | 7 +- tests/test_a11y_settings.py | 5 +- tests/test_color_themes.py | 14 +- tests/test_smoke.py | 9 +- tests/test_third.py | 9 +- tests/test_w3c.py | 257 ++++++++++++++---- 13 files changed, 260 insertions(+), 96 deletions(-) diff --git a/.github/test-environment.yml b/.github/test-environment.yml index a1ad9345..3ff5f8b6 100644 --- a/.github/test-environment.yml +++ b/.github/test-environment.yml @@ -24,6 +24,10 @@ dependencies: - pip - python-build - yarn =3.6 + # audit + - vnu-validator + # lint + - ruff # deps - beautifulsoup4 - ipywidgets @@ -41,5 +45,3 @@ dependencies: # report - scipy - pandas - # audit - - vnu-validator diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b7c08ac..f6af11b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,7 +57,6 @@ jobs: - uses: mamba-org/setup-micromamba@v1 with: environment-file: .github/test-environment.yml - cache-downloads-key: mamba-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/test-environment.yml') }} - name: init plawright run: | playwright install --with-deps chromium diff --git a/nbconvert_a11y/audit.py b/nbconvert_a11y/audit.py index 197f5c54..166f1b25 100644 --- a/nbconvert_a11y/audit.py +++ b/nbconvert_a11y/audit.py @@ -1,5 +1,6 @@ """accessibility auditing tools.""" import asyncio +import os import traceback from contextlib import AsyncExitStack, asynccontextmanager from json import dumps @@ -12,6 +13,9 @@ logger = getLogger("a11y-tasks") +ENV_CHROMIUM_CHANNEL = "NBA117_CHROMIUM_CHANNEL" +DEFAULT_CHROMIUM_CHANNEL = "chrome-beta" + async def _main( ids: list, @@ -32,7 +36,7 @@ async def _main( browser = await play.chromium.launch( args=['--enable-blink-features="AccessibilityObjectModel"'], headless=True, - channel="chrome-beta", + channel=get_chrome_channel(), ) page = await browser.new_page() @@ -41,13 +45,17 @@ async def _main( await task(browser, page, output) +def get_chrome_channel(): + return os.environ.get(ENV_CHROMIUM_CHANNEL, DEFAULT_CHROMIUM_CHANNEL) + + @asynccontextmanager async def get_browser(): async with playwright.async_api.async_playwright() as play: yield await play.chromium.launch( args=['--enable-blink-features="AccessibilityObjectModel"'], headless=True, - channel="chrome-beta", + channel=get_chrome_channel(), ) diff --git a/nbconvert_a11y/exporter.py b/nbconvert_a11y/exporter.py index c007cd7f..0032deba 100644 --- a/nbconvert_a11y/exporter.py +++ b/nbconvert_a11y/exporter.py @@ -4,18 +4,18 @@ """ import builtins -from io import StringIO import json from contextlib import suppress from datetime import datetime from functools import lru_cache +from io import StringIO from pathlib import Path import bs4 -from nbconvert import Exporter import nbformat.v4 import pygments from bs4 import BeautifulSoup +from nbconvert import Exporter from nbconvert.exporters.html import HTMLExporter from traitlets import Bool, CUnicode, Enum, Unicode @@ -138,7 +138,7 @@ def from_notebook_node(self, nb, resources=None, **kw): return super().from_notebook_node(nb, resources, **kw) def post_process_html(self, body): - """a final pass at the exported html to add table of contents, heading links, and other a11y affordances.""" + """A final pass at the exported html to add table of contents, heading links, and other a11y affordances.""" soup = soupify(body) describe_main(soup) heading_links(soup) @@ -180,12 +180,12 @@ def get_markdown_renderer(): def get_markdown(md, **kwargs): - """exporter markdown as html""" + """Exporter markdown as html""" return get_markdown_renderer().render("".join(md), **kwargs) def highlight(code, lang="python", attrs=None, experimental=True): - """highlight code blocks""" + """Highlight code blocks""" import html import pygments @@ -216,7 +216,7 @@ def soupify(body: str) -> BeautifulSoup: def mdtoc(html): - """create a table of contents in markdown that will be converted to html""" + """Create a table of contents in markdown that will be converted to html""" import io toc = io.StringIO() @@ -236,12 +236,12 @@ def mdtoc(html): def toc(html): - """create an html table of contents""" + """Create an html table of contents""" return get_markdown(mdtoc(html)) def heading_links(html): - """convert headings into links""" + """Convert headings into links""" for header in html.select(":is(h1,h2,h3,h4,h5,h6):not([role])"): id = header.attrs.get("id") if not id: @@ -271,22 +271,22 @@ def count_cell_loc(cell): def count_loc(nb): - """count total significant lines of code in the document""" + """Count total significant lines of code in the document""" return sum(map(count_cell_loc, nb.cells)) def count_outputs(nb): - """count total number of cell outputs""" + """Count total number of cell outputs""" return sum(map(len, (x.get("outputs", "") for x in nb.cells))) def count_code_cells(nb): - """count total number of code cells""" + """Count total number of code cells""" return len([None for x in nb.cells if x["cell_type"] == "code"]) def describe_main(soup): - """add REFIDs to aria-describedby""" + """Add REFIDs to aria-describedby""" x = soup.select_one("#toc > details > summary") if x: x.attrs["aria-describedby"] = soup.select_one("main").attrs[ @@ -295,7 +295,7 @@ def describe_main(soup): def ordered(nb) -> str: - """measure if the notebook is ordered""" + """Measure if the notebook is ordered""" start = 0 for cell in nb.cells: if cell["cell_type"] == "code": diff --git a/nbconvert_a11y/pytest_axe.py b/nbconvert_a11y/pytest_axe.py index 55aeabb4..1e963fe6 100644 --- a/nbconvert_a11y/pytest_axe.py +++ b/nbconvert_a11y/pytest_axe.py @@ -7,9 +7,8 @@ # requires node and axe # requires playwright -from ast import Not -from collections import defaultdict import dataclasses +from collections import defaultdict from functools import lru_cache, partial from json import dumps, loads from pathlib import Path @@ -93,7 +92,7 @@ class AxeOptions(Base): def get_npm_directory(package, data=False): - """get the path of an npm package in the environment""" + """Get the path of an npm package in the environment""" try: info = loads(check_output(split(f"npm ls --long --depth 0 --json {quote(package)}"))) except CalledProcessError: @@ -152,7 +151,7 @@ def configure(self, **config): return self def reset(self): - self.page.evaluate(f"""window.axe.reset()""") + self.page.evaluate("""window.axe.reset()""") return self def __enter__(self): diff --git a/nbconvert_a11y/templates/a11y/static/theme/__main__.py b/nbconvert_a11y/templates/a11y/static/theme/__main__.py index 441120eb..55b2e316 100644 --- a/nbconvert_a11y/templates/a11y/static/theme/__main__.py +++ b/nbconvert_a11y/templates/a11y/static/theme/__main__.py @@ -2,7 +2,6 @@ from pathlib import Path - HERE = Path(__file__).parent themes = """a11y-dark diff --git a/pyproject.toml b/pyproject.toml index eec6770a..5d001060 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,7 +201,7 @@ exclude = [ ] [tool.ruff.lint] -isort.known-first-party = ["importnb"] +isort.known-first-party = ["importnb", "nbconvert_a11y", "tests"] ignore = ["D203", "D213", "COM812", "ISC001"] select = [ "A", diff --git a/tests/test_a11y_baseline.py b/tests/test_a11y_baseline.py index 0c31e4b3..7d217258 100644 --- a/tests/test_a11y_baseline.py +++ b/tests/test_a11y_baseline.py @@ -4,13 +4,12 @@ * test the accessibility of nbconvert-a11y dialogs """ -from json import dumps from pathlib import Path from pytest import mark, param -from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX -from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, SKIPCI, get_target_html +from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX +from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html SA11Y = "sa11y-control-panel" @@ -41,7 +40,7 @@ ], ) def test_axe(axe, config, notebook): - """verify the baseline templates satisify all rules update AAA. + """Verify the baseline templates satisify all rules update AAA. any modifications to the template can only degrade accessibility. this baseline is critical for adding more features. all testing piles diff --git a/tests/test_a11y_settings.py b/tests/test_a11y_settings.py index 58930c12..9da1c7e8 100644 --- a/tests/test_a11y_settings.py +++ b/tests/test_a11y_settings.py @@ -10,10 +10,11 @@ LORENZ_EXECUTED = get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb") -@fixture + +@fixture() def lorenz(page): axe = Axe(page=page, url=LORENZ_EXECUTED.as_uri()) - yield axe.configure() + return axe.configure() @mark.parametrize( diff --git a/tests/test_color_themes.py b/tests/test_color_themes.py index 1f850b1e..a595e02e 100644 --- a/tests/test_color_themes.py +++ b/tests/test_color_themes.py @@ -1,9 +1,9 @@ from nbconvert import get_exporter -from pytest import fixture, mark, param, xfail -from nbconvert_a11y.exporter import THEMES +from pytest import fixture -from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, PYGMENTS, AllOf, Axe, Violation -from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html +from nbconvert_a11y.exporter import THEMES +from nbconvert_a11y.pytest_axe import Axe +from tests.test_smoke import NOTEBOOKS LORENZ = NOTEBOOKS / "lorenz-executed.ipynb" @@ -14,14 +14,14 @@ def exporter(request): e.color_theme = request.param e.include_settings = True e.wcag_priority = "AA" - yield e + return e -@fixture +@fixture() def lorenz(page, tmp_path, exporter): tmp = tmp_path / f"{exporter.color_theme}.html" tmp.write_text(exporter.from_filename(LORENZ)[0]) - yield Axe(page=page, url=tmp.absolute().as_uri()).configure() + return Axe(page=page, url=tmp.absolute().as_uri()).configure() def test_dark_themes(lorenz): diff --git a/tests/test_smoke.py b/tests/test_smoke.py index d8ff661d..768e96ef 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -12,12 +12,10 @@ from pathlib import Path from shutil import copyfile +import jupyter_core.paths import nbconvert.nbconvertapp from pytest import mark, param -import nbconvert_a11y -import jupyter_core.paths - from nbconvert_a11y.exporter import soupify SKIP_BASELINE = "baseline tests skipped locally" @@ -100,7 +98,10 @@ def test_export_notebooks(config, notebook): TARGET.write_text(html) LOGGER.debug(f"writing html to {TARGET}") -@mark.parametrize("target", [get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb")]) + +@mark.parametrize( + "target", [get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb")] +) def test_a11y_template_content(target): soup = soupify(target.read_text()) diff --git a/tests/test_third.py b/tests/test_third.py index 1b7f0e00..cf8c345d 100644 --- a/tests/test_third.py +++ b/tests/test_third.py @@ -9,8 +9,9 @@ from pathlib import Path from unittest import TestCase -from pytest import fixture, skip, mark -from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, NO_ALT, PYGMENTS, AllOf, Violation +from pytest import fixture, mark, skip + +from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, NO_ALT, PYGMENTS, AllOf, Violation from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html # only run these tests when the CI environment variables are defined. @@ -35,7 +36,7 @@ def test_all(self): @xfail(reason="the default pygments theme has priority AA and AAA color contrast issues.") def test_highlight_pygments(self): - """the default template has two serious color contrast violations. + """The default template has two serious color contrast violations. an issue needs to be opened or referenced. """ @@ -47,7 +48,7 @@ def test_highlight_pygments(self): @xfail(reason="widgets have not recieved a concerted effort.") def test_widget_display(self): - """the simple lorenz widget generates one minor and one serious accessibility violation.""" + """The simple lorenz widget generates one minor and one serious accessibility violation.""" raise self.axe.run({"include": [JUPYTER_WIDGETS], "exclude": [NO_ALT]}).raises_allof( Violation["minor-focus-order-semantics"], Violation["serious-aria-input-field-name"], diff --git a/tests/test_w3c.py b/tests/test_w3c.py index 97346834..5af80130 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -1,67 +1,30 @@ # requires node # requires jvm - import collections import functools import itertools -import json import operator -import pathlib +import os import re -import shlex import shutil -import subprocess -from pathlib import Path - +import socket +import sys +import time +import uuid from json import dumps from logging import getLogger from pathlib import Path +from subprocess import Popen +from typing import Any, Callable, Dict, Generator, Tuple +from urllib.request import urlopen import exceptiongroup +import pytest +import requests from pytest import mark, param - from tests.test_smoke import CONFIGURATIONS, get_target_html -EXCLUDE = re.compile( - """or with a “role” attribute whose value is “table”, “grid”, or “treegrid”.$""" - # https://github.com/validator/validator/issues/1125 -) -VNU = shutil.which("vnu") or shutil.which("vnu.cmd") - - -def validate_html(*files: pathlib.Path) -> dict: - return json.loads( - subprocess.check_output( - [VNU, "--stdout", "--format", "json", "--exit-zero-always", *files] - ).decode() - ) - - -def organize_validator_results(results): - collect = collections.defaultdict(functools.partial(collections.defaultdict, list)) - for (error, msg), group in itertools.groupby( - results["messages"], key=operator.itemgetter("type", "message") - ): - for item in group: - collect[error][msg].append(item) - return collect - - -def raise_if_errors(results, exclude=EXCLUDE): - collect = organize_validator_results(results) - exceptions = [] - for msg in collect["error"]: - if not exclude or not exclude.search(msg): - exceptions.append( - exceptiongroup.ExceptionGroup( - msg, [Exception(x["extract"]) for x in collect["error"][msg]] - ) - ) - if exceptions: - raise exceptiongroup.ExceptionGroup("nu validator errors", exceptions) - - HERE = Path(__file__).parent NOTEBOOKS = HERE / "notebooks" EXPORTS = HERE / "exports" @@ -75,6 +38,20 @@ def raise_if_errors(results, exclude=EXCLUDE): INVALID_MARKUP = mark.xfail(reason="invalid html markup") + +ENV_JAVA_PATH = "NBA11Y_JAVA_PATH" +ENV_VNU_JAR_PATH = "NBA11Y_VNU_PATH" +ENV_VNU_SERVER_URL = "NBA11Y_VNU_SERVER_URL" + +TVnuResults = Dict[str, Any] +TVnuValidator = Callable[[Path], TVnuResults] + + +EXCLUDE = re.compile( + """or with a “role” attribute whose value is “table”, “grid”, or “treegrid”.$""" + # https://github.com/validator/validator/issues/1125 +) + htmls = mark.parametrize( "html", [ @@ -98,11 +75,189 @@ def raise_if_errors(results, exclude=EXCLUDE): @htmls -def test_baseline_w3c(page, html): - result = validate_html(html) +def test_baseline_w3c(html: Path, an_html_validator: "TVnuValidator") -> None: + result = an_html_validator(html) VALIDATOR.mkdir(parents=True, exist_ok=True) audit = VALIDATOR / html.with_suffix(".json").name - LOGGER.info(f"""writing {audit} with {len(result.get("violations", ""))} violations""") + violations = result.get("violations", "") + LOGGER.info(f"""writing {audit} with {len(violations)} violations""") audit.write_text(dumps(result)) - raise_if_errors(result) + + +# fixtures +@pytest.fixture(scope="session") +def an_html_validator(a_vnu_server_url: str) -> TVnuValidator: + """Wrap the nvu validator REST API in a synchronous request + + https://github.com/validator/validator/wiki/Service-%C2%BB-Input-%C2%BB-POST-body + """ + + def post(path: Path) -> TVnuResults: + url = f"{a_vnu_server_url}?out=json" + data = path.read_bytes() + headers = {"Content-Type": "text/html"} + res = requests.post(url, data, headers=headers) + return res.json() + + return post + + +@pytest.fixture(scope="session") +def a_vnu_server_url( + worker_id: str, tmp_path_factory: pytest.TempPathFactory +) -> Generator[None, None, str]: + """Get the URL for a running VNU server.""" + url: str | None = os.environ.get(ENV_VNU_SERVER_URL) + + if url is not None: + return url + + proc: Popen | None = None + owns_lock = False + proto = "http" + host = "127.0.0.1" + root_tmp_dir = tmp_path_factory.getbasetemp().parent + lock_dir = root_tmp_dir / "vnu_server" + needs_lock = lock_dir / f"test-{uuid.uuid4()}" + + if worker_id == "master": + port, url, proc = _start_vnu_server(proto, host) + owns_lock = True + else: + port = None + retries = 10 + + try: + lock_dir.mkdir() + owns_lock = True + except: + pass + + needs_lock.mkdir() + + if owns_lock: + port, url, proc = _start_vnu_server(proto, host) + (lock_dir / f"port-{port}").mkdir() + else: + while retries: + retries -= 1 + try: + port = int(next(lock_dir.glob("port-*")).name.split("-")[-1]) + url = f"{proto}://{host}:{port}" + except: + time.sleep(1) + if port is None and not retries: + raise RuntimeError("Never started vnu server") + + yield url + + shutil.rmtree(needs_lock) + + if owns_lock: + while True: + needs = [*lock_dir.glob("test-*")] + if needs: + time.sleep(1) + continue + break + + print(f"... tearing down vnu server at {url}") + proc.terminate() + shutil.rmtree(lock_dir) + + +# utilities +def organize_validator_results(results): + collect = collections.defaultdict(functools.partial(collections.defaultdict, list)) + for (error, msg), group in itertools.groupby( + results["messages"], key=operator.itemgetter("type", "message") + ): + for item in group: + collect[error][msg].append(item) + return collect + + +def raise_if_errors(results, exclude=EXCLUDE): + collect = organize_validator_results(results) + exceptions = [] + for msg in collect["error"]: + if not exclude or not exclude.search(msg): + exceptions.append( + exceptiongroup.ExceptionGroup( + msg, [Exception(x["extract"]) for x in collect["error"][msg]] + ) + ) + if exceptions: + raise exceptiongroup.ExceptionGroup("nu validator errors", exceptions) + + +def _start_vnu_server(proto: str, host: str) -> Tuple[str, Popen]: + """Start a vnu HTTP server.""" + port = get_an_unused_port() + url = f"{proto}://{host}:{port}/" + server_args = get_vnu_args(host, port) + url = f"{proto}://{host}:{port}" + print(f"... starting vnu server at {url}") + print(">>>", "\t".join(server_args)) + proc = Popen(server_args) + wait_for_vnu_to_start(url) + print(f"... vnu server started at {url}") + + return port, url, proc + + +def wait_for_vnu_to_start(url: str, retries: int = 5, warmup: int = 5, sleep: int = 1): + last_error = None + + time.sleep(warmup) + + while retries: + retries -= 1 + try: + return urlopen(url, timeout=warmup) + except Exception as err: + last_error = err + time.sleep(sleep) + + raise RuntimeError(f"{last_error}") + + +def get_vnu_args(host: str, port: int): + win = os.name == "nt" + + java = Path(os.environ.get(ENV_JAVA_PATH, shutil.which("java") or shutil.which("java.exe"))) + jar = Path( + os.environ.get( + ENV_JAVA_PATH, (Path(sys.prefix) / ("Library/lib" if win else "lib") / "vnu.jar") + ) + ) + + if any(not j.exists() for j in [java, jar]): + raise RuntimeError( + "Failed to find java or vnu.jar:\b" + f" - {java.exists()} {java}" + "\n" + f" - {jar.exists()} {jar}" + ) + + server_args = [ + java, + "-cp", + jar, + f"-Dnu.validator.servlet.bind-address={host}", + "nu.validator.servlet.Main", + port, + ] + + return list(map(str, server_args)) + + +def get_an_unused_port() -> Callable[[], int]: + """Find an unused network port (could still create race conditions).""" + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + s.listen(1) + port = s.getsockname()[1] + s.close() + return port From cb516ef3b0c38c9b0aa6d436973227663ec673d2 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 10 Dec 2023 13:23:11 -0600 Subject: [PATCH 12/13] replace long sleep with more retries --- tests/test_w3c.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_w3c.py b/tests/test_w3c.py index 5af80130..460b80ea 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -207,15 +207,15 @@ def _start_vnu_server(proto: str, host: str) -> Tuple[str, Popen]: return port, url, proc -def wait_for_vnu_to_start(url: str, retries: int = 5, warmup: int = 5, sleep: int = 1): +def wait_for_vnu_to_start(url: str, retries: int = 10, sleep: int = 1): last_error = None - time.sleep(warmup) + time.sleep(sleep) while retries: retries -= 1 try: - return urlopen(url, timeout=warmup) + return urlopen(url, timeout=sleep) except Exception as err: last_error = err time.sleep(sleep) From e6959f1a7cf5bd17971fcaf56efb051c654420e7 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 10 Dec 2023 13:35:43 -0600 Subject: [PATCH 13/13] fix env var names --- nbconvert_a11y/audit.py | 2 +- tests/test_w3c.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nbconvert_a11y/audit.py b/nbconvert_a11y/audit.py index 166f1b25..90a47de3 100644 --- a/nbconvert_a11y/audit.py +++ b/nbconvert_a11y/audit.py @@ -13,7 +13,7 @@ logger = getLogger("a11y-tasks") -ENV_CHROMIUM_CHANNEL = "NBA117_CHROMIUM_CHANNEL" +ENV_CHROMIUM_CHANNEL = "NBA11Y_CHROMIUM_CHANNEL" DEFAULT_CHROMIUM_CHANNEL = "chrome-beta" diff --git a/tests/test_w3c.py b/tests/test_w3c.py index 460b80ea..55e3d64e 100644 --- a/tests/test_w3c.py +++ b/tests/test_w3c.py @@ -40,7 +40,7 @@ ENV_JAVA_PATH = "NBA11Y_JAVA_PATH" -ENV_VNU_JAR_PATH = "NBA11Y_VNU_PATH" +ENV_VNU_JAR_PATH = "NBA11Y_VNU_JAR_PATH" ENV_VNU_SERVER_URL = "NBA11Y_VNU_SERVER_URL" TVnuResults = Dict[str, Any]