Selenium Lab Tests #813
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Selenium Lab Tests | |
| on: | |
| workflow_dispatch: | |
| # Allows for manual triggering on PRs. They should be reviewed first, to | |
| # avoid malicious code executing in the lab. | |
| inputs: | |
| pr: | |
| description: "A PR number to build and test in the lab. If empty, will build and test from main." | |
| required: false | |
| test_filter: | |
| description: "A regex filter to run a subset of the tests. If empty, all tests will run." | |
| required: false | |
| browser_filter: | |
| description: "A list of browsers to run the tests. If empty, all browsers will run." | |
| required: false | |
| workflow_call: | |
| # Allows for reuse from other workflows, such as "Update All Screenshots" | |
| # workflow. | |
| inputs: | |
| pr: | |
| description: "A PR number to build and test in the lab. If empty, will build and test from main." | |
| required: false | |
| type: string | |
| test_filter: | |
| description: "A regex filter to run a subset of the tests. If empty, all tests will run." | |
| required: false | |
| type: string | |
| browser_filter: | |
| description: "A list of browsers to run the tests. If empty, all browsers will run." | |
| required: false | |
| type: string | |
| ignore_test_status: | |
| description: "If true, ignore test success or failure, never set the commit status, and always upload screenshots." | |
| required: false | |
| type: boolean | |
| schedule: | |
| # Runs every night at 2am PST / 10am UTC, testing against the main branch. | |
| - cron: '0 10 * * *' | |
| # Only one run of this workflow is allowed at a time, since it uses physical | |
| # resources in our lab. | |
| concurrency: selenium-lab | |
| jobs: | |
| compute-ref: | |
| name: Compute ref | |
| runs-on: ubuntu-latest | |
| outputs: | |
| REF: ${{ steps.compute.outputs.REF }} | |
| steps: | |
| - name: Compute ref | |
| id: compute | |
| run: | | |
| if [[ "${{ inputs.pr }}" != "" ]]; then | |
| LAB_TEST_REF="refs/pull/${{ inputs.pr }}/head" | |
| else | |
| LAB_TEST_REF="main" | |
| fi | |
| echo "REF=$LAB_TEST_REF" | tee -a $GITHUB_OUTPUT | |
| # Configure the build matrix based on our grid's YAML config. | |
| # The matrix contents will be computed by this first job and deserialized | |
| # into the second job's config. | |
| matrix-config: | |
| name: Matrix config | |
| needs: compute-ref | |
| runs-on: ubuntu-latest | |
| outputs: | |
| INCLUDE: ${{ steps.configure.outputs.INCLUDE }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.compute-ref.outputs.REF }} | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Configure build matrix | |
| id: configure | |
| shell: node {0} | |
| run: | | |
| const fs = require('fs'); | |
| const yaml = require( | |
| '${{ github.workspace }}/node_modules/js-yaml/index.js'); | |
| // Convert the input "browser_filter" into a set of strings. Take | |
| // care to filter so that the empty string turns into an empty set. | |
| const browserFilter = new Set( "${{ inputs.browser_filter }}" | |
| .split(/\s+/) | |
| .map(x => x.toLowerCase()) | |
| .filter(x => !!x) | |
| ); | |
| const gridBrowserYaml = | |
| fs.readFileSync('build/shaka-lab.yaml', 'utf8'); | |
| const gridBrowserMetadata = yaml.load(gridBrowserYaml); | |
| const include = []; | |
| for (const name in gridBrowserMetadata) { | |
| if (name == 'vars') { | |
| // Skip variable defs in the YAML file | |
| continue; | |
| } | |
| // A browser is enabled if it's not disabled and (either the browser | |
| // filter is empty or it contains the browser name). | |
| const enabled = !gridBrowserMetadata[name].disabled && | |
| (browserFilter.size == 0 || | |
| browserFilter.has(name.toLowerCase())); | |
| if (enabled) { | |
| include.push({browser: name}); | |
| } | |
| } | |
| // Output JSON object consumed by the build matrix below. | |
| fs.appendFileSync( | |
| process.env['GITHUB_OUTPUT'], | |
| `INCLUDE=${ JSON.stringify(include) }\n`); | |
| // Log the output, for the sake of debugging this script. | |
| console.log({include}); | |
| # Build Shaka Player once, then distribute that build to the runners in the | |
| # build matrix. For N runners, runs N times faster (since all the | |
| # self-hosted Selenium jobs are run in containers on one machine). | |
| build-shaka: | |
| name: Pre-build Player | |
| needs: compute-ref | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.compute-ref.outputs.REF }} | |
| - name: Set commit status to pending | |
| if: ${{ inputs.ignore_test_status == false }} | |
| uses: ./.github/workflows/custom-actions/set-commit-status | |
| with: | |
| context: Selenium / Build | |
| state: pending | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build Player | |
| run: python3 build/all.py | |
| - name: Preprocess with Babel | |
| run: | | |
| # Run the test preprocessor without running the actual tests. | |
| # This lets us cache Babel's output and run it only once. | |
| # Ignore the exit code, since you get an error code if the filter | |
| # excludes all tests. | |
| ./build/test.py \ | |
| --use-xvfb --browsers Chrome \ | |
| --filter ThisFilterMatchesNoTests || true | |
| - name: Cache dependencies | |
| uses: actions/cache@v3 | |
| id: npm-cache | |
| with: | |
| path: node_modules/ | |
| key: node-${{ hashFiles('package-lock.json') }} | |
| - name: Cache Babel output | |
| uses: actions/cache@v3 | |
| id: babel-cache | |
| with: | |
| path: .babel-cache | |
| key: babel-${{ hashFiles('*.js', 'demo/**.js', 'lib/**.js', 'ui/**.js', 'test/**.js', 'third_party/**.js') }} | |
| - name: Store Player build | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: shaka-player | |
| path: dist/ | |
| retention-days: 1 | |
| - name: Report final commit status | |
| # Will run on success or failure, but not if the workflow is cancelled | |
| # or if we were asked to ignore the test status. | |
| if: ${{ (success() || failure()) && inputs.ignore_test_status == false }} | |
| uses: ./.github/workflows/custom-actions/set-commit-status | |
| with: | |
| context: Selenium / Build | |
| state: ${{ job.status }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| lab-tests: | |
| # This is a self-hosted runner in a Docker container, with access to our | |
| # lab's Selenium grid on port 4444. | |
| runs-on: self-hosted-selenium | |
| needs: [compute-ref, build-shaka, matrix-config] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.matrix-config.outputs.INCLUDE) }} | |
| name: ${{ matrix.browser }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.compute-ref.outputs.REF }} | |
| - name: Set commit status to pending | |
| if: ${{ inputs.ignore_test_status == false }} | |
| uses: ./.github/workflows/custom-actions/set-commit-status | |
| with: | |
| context: Selenium / ${{ matrix.browser }} | |
| state: pending | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| registry-url: 'https://registry.npmjs.org' | |
| # The Docker image for this self-hosted runner doesn't contain java. | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: zulu | |
| java-version: 11 | |
| - name: Cache dependencies | |
| uses: actions/cache@v3 | |
| id: npm-cache | |
| with: | |
| path: node_modules/ | |
| key: node-${{ hashFiles('package-lock.json') }} | |
| fail-on-cache-miss: true # Cached by the build-shaka job above | |
| enableCrossOsArchive: true # Share archives from Linux to Windows | |
| - name: Cache Babel output | |
| uses: actions/cache@v3 | |
| id: babel-cache | |
| with: | |
| path: .babel-cache | |
| key: babel-${{ hashFiles('*.js', 'demo/**.js', 'lib/**.js', 'ui/**.js', 'test/**.js', 'third_party/**.js') }} | |
| fail-on-cache-miss: true # Cached by the build-shaka job above | |
| enableCrossOsArchive: true # Share archives from Linux to Windows | |
| - name: Install dependencies | |
| if: steps.npm-cache.outputs.cache-hit != 'true' | |
| run: npm ci | |
| # Instead of building Shaka N times, build it once and fetch the build to | |
| # each Selenium runner in the matrix. | |
| - name: Fetch Player build | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: shaka-player | |
| path: dist/ | |
| # Run tests on the Selenium grid in our lab. This uses a private | |
| # hostname and TLS cert to get EME tests working on all platforms | |
| # (since EME only works on https or localhost). The variable | |
| # ALLOCATED_PORT must be defined by the self-hosted runner, and mapped | |
| # from the host to the container. | |
| - name: Test Player | |
| run: | | |
| # Use of an array keeps elements intact, and allows an element to | |
| # contain spaces without being expanded into multiple arguments in a | |
| # shell command. | |
| extra_flags=() | |
| # Generate a coverage report from uncompiled code on ChromeLinux. | |
| # It should be the uncompiled build, or else we won't execute any | |
| # coverage instrumentation on full-stack player integration tests. | |
| if [[ "${{ matrix.browser }}" == "Edge" ]]; then | |
| extra_flags+=(--html-coverage-report --uncompiled) | |
| fi | |
| if [[ "${{ inputs.test_filter }}" != "" ]]; then | |
| echo "Adding filter: ${{ inputs.test_filter }}" | |
| extra_flags+=(--filter "${{ inputs.test_filter }}") | |
| fi | |
| # Do not automatically fail when a command fails. This allows us to | |
| # implement the ignore_test_status input by capturing the exit code | |
| # and examining it. | |
| set +e | |
| # Run the tests with any extra flags. | |
| python3 build/test.py \ | |
| --no-build \ | |
| --reporters spec --spec-hide-passed \ | |
| --tls-key /etc/letsencrypt/live/karma.shakalab.rocks/privkey.pem \ | |
| --tls-cert /etc/letsencrypt/live/karma.shakalab.rocks/fullchain.pem \ | |
| --hostname karma.shakalab.rocks \ | |
| --port $ALLOCATED_PORT \ | |
| --grid-config build/shaka-lab.yaml \ | |
| --grid-address selenium-grid.lab.shaka:4444 \ | |
| --browsers ${{ matrix.browser }} \ | |
| "${extra_flags[@]}" | |
| # Capture the test exit code immediately after running the tests. | |
| # There cannot be any other command between test.py and here. | |
| exit_code=$? | |
| # If ignoring test status, treat this as an exit code of 0 (success). | |
| if [[ "${{ inputs.ignore_test_status }}" == "true" ]]; then | |
| exit_code=0 | |
| fi | |
| # Report the captured (and possibly overridden) exit status. | |
| exit $exit_code | |
| - name: Find coverage report (Edge only) | |
| id: coverage | |
| # Run even if an earlier step fails, but only on Edge. | |
| if: ${{ always() && matrix.browser == 'Edge' }} | |
| shell: bash | |
| run: | | |
| # Find the path to the coverage report specifically for Chrome on | |
| # Linux. It includes the exact browser version in the path, so it | |
| # will vary. Having a single path will make the artifact zip | |
| # simpler, whereas using a wildcard in the upload step will result | |
| # in a zip file with internal directories. | |
| coverage_report="$( (ls coverage/Edge*/coverage.json || true) | head -1 )" | |
| # Show what's there, for debugging purposes. | |
| ls -l coverage/ | |
| if [ -f "$coverage_report" ]; then | |
| echo "Found coverage report: $coverage_report" | |
| echo "coverage_report=$coverage_report" >> $GITHUB_OUTPUT | |
| else | |
| echo "Could not locate coverage report!" | |
| exit 1 | |
| fi | |
| - name: Upload coverage report (Edge only) | |
| uses: actions/upload-artifact@v3 | |
| # If there's a coverage report, upload it, even if a previous step | |
| # failed. | |
| if: ${{ always() && steps.coverage.outputs.coverage_report }} | |
| with: | |
| # This will create a download called coverage.zip containing only | |
| # coverage.json. | |
| path: ${{ steps.coverage.outputs.coverage_report }} | |
| name: coverage | |
| # Since we've already filtered this step for instances where there is | |
| # an environment variable set for this, the file should definitely be | |
| # there. | |
| if-no-files-found: error | |
| # Upload new screenshots and diffs on failure; ignore if missing | |
| - name: Upload screenshots | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || inputs.ignore_test_status }} | |
| with: | |
| # In this workflow, "browser" is the selenium node name, which can | |
| # contain both browser and OS, such as "ChromeLinux". | |
| name: screenshots-${{ matrix.browser }} | |
| path: | | |
| test/test/assets/screenshots/*/*.png-new | |
| test/test/assets/screenshots/*/*.png-diff | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| - name: Report final commit status | |
| # Will run on success or failure, but not if the workflow is cancelled | |
| # or if we were asked to ignore the test status. | |
| if: ${{ (success() || failure()) && inputs.ignore_test_status == false }} | |
| uses: ./.github/workflows/custom-actions/set-commit-status | |
| with: | |
| context: Selenium / ${{ matrix.browser }} | |
| state: ${{ job.status }} | |
| token: ${{ secrets.GITHUB_TOKEN }} |