Skip to content
Tests

Add Playwright coverage to CI (#555) #1852

Sign in to view logs

Add Playwright coverage to CI (#555)

Add Playwright coverage to CI (#555) #1852

Workflow file for this run

.github/workflows/pipeline.yaml at 5643b0c
name: Tests
on:
push:
# branches-ignore:
# - main
# - dev
workflow_dispatch:
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
static-analysis:
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
fail-fast: false
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install Dependencies and library
shell: bash
run: |
set -ux
python -m pip install --upgrade pip
pip install -e ".[submodules,docs,lint,testing]"
- name: Run mypy
shell: bash
run: mypy fastagency tests
- name: Run bandit
shell: bash
run: bandit -c pyproject.toml -r fastagency
- name: Run Semgrep
shell: bash
run: semgrep scan --config auto --error
test-without-llms:
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
fail-fast: false
uses: ./.github/workflows/test.yaml
with:
python-version: ${{ matrix.python-version }}
environment: null
use-llms: ""
secrets: inherit # pragma: allowlist secret
test-playwright-without-llms:
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12"]
fail-fast: false
uses: ./.github/workflows/test-playwright.yaml
with:
python-version: ${{ matrix.python-version }}
environment: null
use-llms: ""
secrets: inherit # pragma: allowlist secret
test-with-anthropic:
uses: ./.github/workflows/test.yaml
with:
python-version: "3.9"
environment: testing
use-llms: "anthropic"
secrets: inherit # pragma: allowlist secret
test-with-azure_oai:
uses: ./.github/workflows/test.yaml
with:
python-version: "3.9"
environment: testing
use-llms: "azure_oai"
secrets: inherit # pragma: allowlist secret
test-with-openai:
uses: ./.github/workflows/test.yaml
with:
python-version: "3.9"
environment: testing
use-llms: "openai"
secrets: inherit # pragma: allowlist secret
test-with-togetherai:
uses: ./.github/workflows/test.yaml
with:
python-version: "3.9"
environment: testing
use-llms: "togetherai"
secrets: inherit # pragma: allowlist secret
test-with-llm:
uses: ./.github/workflows/test.yaml
with:
python-version: "3.9"
environment: testing
use-llms: "llm"
secrets: inherit # pragma: allowlist secret
test-macos-latest:
if: github.event.pull_request.draft == false
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: pyproject.toml
- name: Install Dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: pip install .[submodules,docs,testing]
- name: Test
run: bash scripts/test.sh -m "not (nats or anthropic or azure_oai or openai or togetherai or llm)"
test-windows-latest:
if: github.event.pull_request.draft == false
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: pyproject.toml
- name: Install Dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: pip install .[submodules,docs,testing]
- name: Test
run: bash scripts/test.sh -m "not (nats or anthropic or azure_oai or openai or togetherai or llm)"
coverage-combine:
needs:
- test-without-llms
- test-with-llm
- test-playwright-without-llms
- test-with-anthropic
- test-with-azure_oai
- test-with-openai
- test-with-togetherai
- test-windows-latest
- test-macos-latest
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.9"
cache: "pip"
cache-dependency-path: pyproject.toml
- name: Get coverage files
uses: actions/download-artifact@v4
with:
pattern: .coverage*
path: coverage
merge-multiple: true
- run: pip install coverage[toml]
- run: ls -la coverage
- run: coverage combine coverage
- run: coverage report -i
- run: coverage html -i --show-contexts --title "FastAgency coverage for ${{ github.sha }}"
- name: Store coverage html
uses: actions/upload-artifact@v4
with:
name: coverage-html
path: htmlcov
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
slug: airtai/fastagency
pre-commit-check:
runs-on: ubuntu-latest
env:
SKIP: "static-analysis, test"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Set $PY environment variable
run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
- name: Install Dependencies
run: pip install .[submodules,docs,testing]
- uses: pre-commit/action@v3.0.1
# https://github.com/marketplace/actions/alls-green#why
check: # This job does nothing and is only used for the branch protection
# from: https://github.com/re-actors/alls-green
# Important: For this to work properly, it is a must to have the job always
# run, otherwise GitHub will make it skipped when any of the dependencies
# fail. In some contexts, skipped is interpreted as success which may lead
# to undersired, unobvious and even dangerous (as in security breach
# "dangerous") side-effects.
if: always()
needs:
- static-analysis
- pre-commit-check
- coverage-combine
- test-macos-latest
- test-windows-latest
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1 # nosemgrep
with:
# allowed-failures: docs, linters
# allowed-skips: non-voting-flaky-job
jobs: ${{ toJSON(needs) }}