-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Playwright coverage to CI (#555)
* playwright coverage WIP(1) * playwright coverage WIP(2) * playwright coverage WIP(3) * playwright coverage WIP(4) * playwright coverage WIP(5) * playwright coverage WIP(6) * playwright coverage WIP(7) * playwright coverage WIP(8) * playwright coverage WIP(9) * playwright coverage WIP(10) * playwright coverage WIP(11) * playwright coverage WIP(12) * playwright coverage WIP(12+1) * playwright coverage WIP(14) * playwright coverage WIP(15) * playwright coverage WIP(16) * playwright coverage WIP(17) * playwright coverage WIP(18) * playwright coverage WIP(19) * playwright coverage WIP(20) * playwright coverage WIP(21) * playwright coverage WIP(22) * playwright coverage WIP(23) * playwright coverage WIP(24) * added sleep after starting fastagency server in CI * wip * wip * wip * playwright coverage WIP2(1) * playwright coverage WIP2(2) * playwright coverage WIP2(3) * playwright coverage WIP2(3) * playwright coverage WIP2(5) * playwright coverage WIP2(6) * playwright coverage WIP2(7) * playwright coverage WIP2(8) * playwright coverage WIP2(10) --------- Co-authored-by: Davor Runje <davor@airt.ai>
- Loading branch information
1 parent
c4b0305
commit 5643b0c
Showing
10 changed files
with
256 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| name: Internal test runner | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| environment: | ||
| description: 'Environment to run the tests in' | ||
| required: false | ||
| default: null | ||
| type: string | ||
| python-version: | ||
| description: 'Python version to run the tests in' | ||
| required: true | ||
| type: string | ||
| use-llms: | ||
| description: 'Use LLM in the tests' | ||
| required: false | ||
| type: string | ||
| default: "" | ||
|
|
||
| jobs: | ||
| test: | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 15 | ||
| environment: ${{ inputs.environment }} | ||
| services: | ||
| nats: | ||
| image: diementros/nats:js | ||
| ports: | ||
| - 4222:4222 | ||
| env: | ||
| NATS_URL: nats://localhost:4222 | ||
| steps: | ||
| - name: Set up environment variables | ||
| run: | | ||
| # check if an environment var or secret is defined and set env var to its value | ||
| # vars | ||
| if [ -n "${{ vars.AZURE_API_VERSION }}" ]; then | ||
| echo "AZURE_API_VERSION=${{ vars.AZURE_API_VERSION }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ vars.AZURE_API_ENDPOINT }}" ]; then | ||
| echo "AZURE_API_ENDPOINT=${{ vars.AZURE_API_ENDPOINT }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ vars.AZURE_GPT35_MODEL }}" ]; then | ||
| echo "AZURE_GPT35_MODEL=${{ vars.AZURE_GPT35_MODEL }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ vars.AZURE_GPT4_MODEL }}" ]; then | ||
| echo "AZURE_GPT4_MODEL=${{ vars.AZURE_GPT4_MODEL }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ vars.AZURE_GPT4o_MODEL }}" ]; then | ||
| echo "AZURE_GPT4o_MODEL=${{ vars.AZURE_GPT4o_MODEL }}" >> $GITHUB_ENV | ||
| fi | ||
| # secrets | ||
| if [ -n "${{ secrets.AZURE_OPENAI_API_KEY }}" ]; then | ||
| echo "AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ secrets.TOGETHER_API_KEY }}" ]; then | ||
| echo "TOGETHER_API_KEY=${{ secrets.TOGETHER_API_KEY }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then | ||
| echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ secrets.ANTHROPIC_API_KEY }}" ]; then | ||
| echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ secrets.BING_API_KEY }}" ]; then | ||
| echo "BING_API_KEY=${{ secrets.BING_API_KEY }}" >> $GITHUB_ENV | ||
| fi | ||
| if [ -n "${{ secrets.POSTMAN_API_KEY }}" ]; then | ||
| echo "POSTMAN_API_KEY=${{ secrets.POSTMAN_API_KEY }}" >> $GITHUB_ENV | ||
| fi | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ inputs.python-version }} | ||
| cache: "pip" | ||
| cache-dependency-path: pyproject.toml | ||
| - uses: actions/cache@v4 | ||
| id: cache | ||
| with: | ||
| path: ${{ env.pythonLocation }} | ||
| key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-test-v03 | ||
| - name: Install Dependencies | ||
| if: steps.cache.outputs.cache-hit != 'true' | ||
| run: pip install .[docs,testing] | ||
| - name: Install Pydantic v2 | ||
| run: pip install --pre "pydantic>=2,<3" | ||
| - run: mkdir coverage | ||
| - uses: actions/setup-node@v4 | ||
| with: | ||
| node-version: lts/* | ||
| - name: Install dependencies | ||
| run: npm ci | ||
| - name: Install Playwright Browsers | ||
| run: npx playwright install --with-deps | ||
| - name: Start fastagency | ||
| id: fastagency-start | ||
| run: | | ||
| # Start fastagency and grab its pid | ||
| nohup gunicorn --workers=1 e2e.llm-sans.main:app >nohup.txt 2>nohup-error.txt & | ||
| # Get the process ID (PID) | ||
| FAST_PID=$! | ||
| echo "Started fastagency with PID: $FAST_PID" | ||
| echo "FAST_PID=$FAST_PID" >> $GITHUB_OUTPUT | ||
| env: | ||
| COVERAGE_PROCESS_START: "e2e/playwright.coverage.cfg" | ||
| - run: echo "obtained FAST_PID" ${{ steps.fastagency-start.outputs.FAST_PID}} | ||
| - name: Run Playwright tests without LLMs | ||
| if: ${{ inputs.python-version != '3.9' }} | ||
| run: npx playwright test -c "playwright.coverage.config.ts" | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ !cancelled() }} | ||
| with: | ||
| name: playwright-report.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} | ||
| path: playwright-report/ | ||
| retention-days: 30 | ||
|
|
||
| - name: Kill the program | ||
| run: | | ||
| PID="${{ steps.fastagency-start.outputs.FAST_PID}}" | ||
| # Send SIGTERM to the program (graceful shutdown) | ||
| echo "killing the fastagency:" $PID | ||
| ps -ef | grep $PID | ||
| kill -s SIGTERM $PID | ||
| wait_count=0 | ||
| while ps -p "$PID" > /dev/null && ((wait_count < 5)); do | ||
| echo "Fastagency with PID $PID is still running..." | ||
| sleep 10 | ||
| wait_count=$((wait_count + 1)) | ||
| done | ||
| if ((wait_count == 5)); then | ||
| echo "Fastagency with PID $PID did not exit after 5 timeouts. Terminating..." | ||
| kill -9 $PID | ||
| else | ||
| echo "Fastagency with PID $PID has exited." | ||
| fi | ||
| - run: ls -al .coverage | ||
| - run: cat nohup.txt | ||
| - run: cat nohup-error.txt | ||
| - name: Report | ||
| run: coverage report --no-skip-covered | ||
| - name: Move coverage file | ||
| run: mv .coverage coverage/.coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} | ||
| - name: Check coverage file | ||
| run: ls -al coverage | ||
| - name: Store coverage files | ||
| if: ${{ inputs.python-version != '3.9' }} | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: .coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} | ||
| path: coverage/.coverage.playwright.* | ||
| if-no-files-found: error | ||
| overwrite: true | ||
| include-hidden-files: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| [coverage:run] | ||
| branch = True | ||
| sigterm = True | ||
| context = playwright | ||
|
|
||
| [coverage:report] | ||
| omit = | ||
| /tmp/* | ||
|
|
||
| exclude_also = | ||
| ; Don't complain about missing debug-only code: | ||
| def __repr__ | ||
| if self\.debug | ||
|
|
||
| ; Don't complain if tests don't hit defensive assertion code: | ||
| raise AssertionError | ||
| raise NotImplementedError | ||
|
|
||
| ; Don't complain if non-runnable code isn't run: | ||
| if 0: | ||
| if __name__ == .__main__.: | ||
|
|
||
| ; Don't complain about abstract methods, they aren't run: | ||
| @(abc\.)?abstractmethod | ||
|
|
||
| ignore_errors = True |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| // playwright setting for tests that run with unicorn | ||
| import { defineConfig } from "./e2e/playwright.base.config.ts"; | ||
|
|
||
| export default defineConfig( | ||
| { | ||
| testDir: './e2e/llm-sans', | ||
| use: { | ||
| /* Base URL to use in actions like `await page.goto('/')`. */ | ||
| baseURL: 'http://127.0.0.1:8000', | ||
|
|
||
| /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ | ||
| trace: 'on-first-retry', | ||
| }, | ||
| } | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters