Merge pull request jupyter#83 from Quansight-Labs/axe-gitpod

Axe Gitpod JupyterLab

.gitignore

@@ -121,3 +121,13 @@ logs/
 
 # reports
 reports/
+
+# nox
+**.nox/
+
+# playwright stuff
+**/playwright-*/
+**/test-results/
+
+# mac
+.DS_Store

.gitpod.yml

@@ -0,0 +1,13 @@
+image:
+  file: testing/tools/gitpod/Dockerfile
+ports:
+- port: 9323
+- port: 8888
+tasks:
+  - name: Install JupyterLab Node.js testing dependencies
+    before: |
+      cd testing/jupyterlab
+    init: |
+      yarn install
+    command: |
+      npx playwright install

testing/README.md

@@ -0,0 +1,26 @@
+# Testing - Jupyter Accessibility
+
+## WIP - Work in Progress
+
+This directory is a place for accessibility testing within
+[Jupyter](https://jupyter.org).
+
+As of May 10, 2022, it is a very early work in progress.
+
+## Plans for the future
+
+Automated accessibility tests cannot address accessibility issues on their own,
+but used correctly they can be a useful tool.
+
+This directory is modeled after the
+[JupyterLab Benchmarks](https://github.com/jupyterlab/benchmarks/) repo.
+
+As described in the
+[Jupyter Accessibility Roadmap](https://github.com/jupyter/accessibility/blob/master/docs/funding/czi-grant-roadmap.md),
+the plan is to start with JupyterLab. We are begininng with the web app UI, then
+we will add tests for the docs. (The rationale to that sequence is to start with
+the harder problem first.)
+
+After JupyterLab, though it is not within scope of the grant driving the
+roadmap, we hope to extend this testing to other parts of the Juptyer ecosystem
+beyond JupyterLab.

testing/jupyterlab/README.md

@@ -0,0 +1,82 @@
+# JupyterLab accessibility tests
+
+## :package: Requirements
+
+To run the tests in this directory you need the following pre-requisites:
+
+- mamba (or conda)
+- Your system must also meet the [Playwright system requirements](https://playwright.dev/docs/library#system-requirements)
+
+## :zap: Running the accessibility tests
+
+You can run tests locally on your machine, or remotely in the cloud.
+
+At the time of this writing, we have configured the tests to only run with
+Chromium, but you can
+[extend these tests to cover other browsers](https://github.com/MarcusFelling/demo.playwright/blob/main/accessibility/playwright.config.ts)
+by modifying the [`playwright.config.ts`](testing/jupyterlab/playwright.config.ts) file.
+
+### :laptop: Locally
+
+1. Make sure you are in the correct directory:
+
+    ```bash
+    cd testing/jupyterlab
+    ```
+
+2. Install the Python dependencies:
+
+    ```bash
+    # if using conda
+    conda env create -f environment.yml
+
+    # if using mamba
+    mamba env create -f environment.yml
+    ```
+
+3. Install the Node.js dependencies:
+
+    ```bash
+    yarn install
+    npx playwright install
+    ```
+
+4. Run the tests:
+
+    ```bash
+    yarn test
+    ```
+
+Your console should output a local url that you can open in your browser to see
+the test results:
+    ```
+    	http://127.0.0.1:9323
+    ```
+
+### :cloud: Running in Gitpod
+
+As an alternative to running the tests locally on your own machine, you can run
+them in a cloud environment on Gitpod.
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/jupyter/accessibility)
+
+Once you are in the Gitpod workspace, run:
+
+```bash
+yarn test
+```
+
+The Gitpod console should output a local url that you can open in your browser
+to see the test results:
+    ```
+    	http://127.0.0.1:9323
+    ```
+
+In case you're wondering how you can open a local url from a cloud environment,
+Gitpod opens a remote session in VS Code (or other supported editor) and sets it
+up to proxy the local url to its remote server address.
+
+## Troubleshooting
+
+If a test fails, Playwright should attach a video and possibly other files to
+that test, which could help debug or explain why the test failed.

testing/jupyterlab/environment.yml

@@ -0,0 +1,14 @@
+name: a11y-tests
+channels:
+  - conda-forge
+dependencies:
+  # for base a11y testing
+  - nodejs=16.*
+  - yarn
+  - jupyterlab=3.*
+  # for the notebook example
+  - ipython
+  - ipywidgets
+  - matplotlib
+  - numpy
+  - scipy

testing/jupyterlab/jupyter_server_config.py

@@ -0,0 +1,13 @@
+from tempfile import mkdtemp
+
+# These settings modeled after https://github.com/jupyterlab/jupyterlab/blob/%40jupyterlab/galata%404.0.2/galata/jupyter_server_test_config.py
+
+c.ServerApp.port = 8888
+c.ServerApp.port_retries = 0
+c.ServerApp.open_browser = False
+
+c.ServerApp.root_dir = mkdtemp(prefix="galata-test-")
+c.ServerApp.token = ""
+c.ServerApp.password = ""
+c.ServerApp.disable_check_xsrf = True
+c.LabApp.expose_app_in_browser = True

testing/jupyterlab/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@jupyterlab/a11ytests",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Run automated accessibility checks against JupyterLab",
+  "license": "BSD-3-Clause",
+  "author": "Project Jupyter",
+  "scripts": {
+    "start": "jupyter lab --config ./jupyter_server_config.py",
+    "test": "playwright test",
+    "test:debug": "PWDEBUG=1 playwright test"
+  },
+  "dependencies": {
+    "@jupyterlab/galata": "4.0.2",
+    "expect-axe-playwright": "^2.2.0"
+  }
+}

testing/jupyterlab/playwright.config.ts

@@ -0,0 +1,87 @@
+// playwright.config.ts
+import galataConfig from "@jupyterlab/galata/lib/playwright-config";
+import { PlaywrightTestConfig, devices } from "@playwright/test";
+import { expect } from "@playwright/test";
+import matchers from "expect-axe-playwright";
+
+expect.extend(matchers);
+
+/**
+ * Modified from https://github.com/MarcusFelling/demo.playwright/blob/main/accessibility/playwright.config.ts
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  ...galataConfig,
+
+  testDir: "./tests",
+
+  /* Maximum time one test can run for. */
+  timeout: 20 * 1000,
+
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000,
+  },
+
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: process.env.BASEURL,
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on",
+
+    acceptDownloads: true,
+
+    video: "retain-on-failure",
+
+    axeOptions: {
+      runOnly: {
+        type: "tag",
+        // all tags and standards listed here:
+        // https://www.deque.com/axe/core-documentation/api-documentation/#axe-core-tags
+        values: ["wcag2a", "best-practice"],
+      },
+    },
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+
+      /* Project-specific settings. */
+      use: {
+        ...devices["Desktop Chrome"],
+      },
+    },
+  ],
+
+  /* Run a server. The tests will open urls to this server in the browser. */
+  webServer: {
+    command: "yarn start",
+    port: 8888,
+    timeout: 120 * 1000,
+    reuseExistingServer: !process.env.CI,
+  },
+};
+
+export default config;

testing/jupyterlab/tests/a11y.test.ts

@@ -0,0 +1,58 @@
+import { expect } from "@playwright/test";
+import { test } from "@jupyterlab/galata";
+import * as path from "path";
+
+const notebooksDirectory = path.resolve(__dirname, "../../notebooks/");
+// Lorenz notebook file name
+const LORENZ = "Lorenz.ipynb";
+
+// playwright records the browser window while running, and then uploads the
+// video with the test results to help with debugging. So here we scroll to the
+// bottom of the notebook so that if somebody needs to watch the video, they can
+// see the whole notebook from beginning to end.
+async function scrollNotebookToBottom(page) {
+  await page.evaluate(() => {
+    const notebookEl = document.querySelector(".jp-Notebook");
+    const scrollBottom = notebookEl.scrollHeight - notebookEl.clientHeight;
+    notebookEl.scroll({
+      top: scrollBottom,
+      behavior: "smooth",
+    });
+  });
+}
+
+/**
+ * In these tests we use the Axe accessibility testing engine to run analysis on
+ * page
+ * @see https://github.com/abhinaba-ghosh/axe-playwright
+ */
+test.describe("JupyterLab accessibility checks", () => {
+  test.beforeEach(async ({ page, tmpPath }) => {
+    await page.contents.uploadDirectory(notebooksDirectory, tmpPath);
+  });
+
+  test("Lorenz notebook before running all cells", async ({
+    page,
+    tmpPath,
+  }) => {
+    await page.notebook.openByPath(`${tmpPath}/${LORENZ}`);
+    try {
+      await expect(page).toBeAccessible();
+    } finally {
+      await scrollNotebookToBottom(page);
+    }
+  });
+
+  test("Lorenz notebook after running all cells", async ({
+    page,
+    tmpPath,
+  }) => {
+    await page.notebook.openByPath(`${tmpPath}/${LORENZ}`);
+    await page.notebook.run();
+    try {
+      await expect(page).toBeAccessible();
+    } finally {
+      await scrollNotebookToBottom(page);
+    }
+  });
+});