Add Playwright coverage to CI

.github/workflows/pipeline.yaml

@@ -57,6 +57,19 @@ jobs:
       use-llms: ""
     secrets: inherit  # pragma: allowlist secret
 
+  test-playwright-without-llms:
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+      fail-fast: false
+    uses: ./.github/workflows/test-playwright.yaml
+    with:
+      python-version: ${{ matrix.python-version }}
+      environment: null
+      use-llms: ""
+    secrets: inherit  # pragma: allowlist secret
+
+
   test-with-anthropic:
     uses: ./.github/workflows/test.yaml
     with:
@@ -137,6 +150,7 @@ jobs:
     needs:
       - test-without-llms
       - test-with-llm
+      - test-playwright-without-llms
       - test-with-anthropic
       - test-with-azure_oai
       - test-with-openai
@@ -165,8 +179,8 @@ jobs:
 
       - run: ls -la coverage
       - run: coverage combine coverage
-      - run: coverage report
-      - run: coverage html --show-contexts --title "FastAgency coverage for ${{ github.sha }}"
+      - run: coverage report -i
+      - run: coverage html -i --show-contexts --title "FastAgency coverage for ${{ github.sha }}"
 
       - name: Store coverage html
         uses: actions/upload-artifact@v4

.github/workflows/test-playwright.yaml

@@ -0,0 +1,162 @@
+name: Internal test runner
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: 'Environment to run the tests in'
+        required: false
+        default: null
+        type: string
+      python-version:
+        description: 'Python version to run the tests in'
+        required: true
+        type: string
+      use-llms:
+        description: 'Use LLM in the tests'
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment: ${{ inputs.environment }}
+    services:
+      nats:
+        image: diementros/nats:js
+        ports:
+          - 4222:4222
+    env:
+      NATS_URL: nats://localhost:4222
+    steps:
+      - name: Set up environment variables
+        run: |
+          # check if an environment var or secret is defined and set env var to its value
+
+          # vars
+
+          if [ -n "${{ vars.AZURE_API_VERSION }}" ]; then
+            echo "AZURE_API_VERSION=${{ vars.AZURE_API_VERSION }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ vars.AZURE_API_ENDPOINT }}" ]; then
+            echo "AZURE_API_ENDPOINT=${{ vars.AZURE_API_ENDPOINT }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ vars.AZURE_GPT35_MODEL }}" ]; then
+            echo "AZURE_GPT35_MODEL=${{ vars.AZURE_GPT35_MODEL }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ vars.AZURE_GPT4_MODEL }}" ]; then
+            echo "AZURE_GPT4_MODEL=${{ vars.AZURE_GPT4_MODEL }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ vars.AZURE_GPT4o_MODEL }}" ]; then
+            echo "AZURE_GPT4o_MODEL=${{ vars.AZURE_GPT4o_MODEL }}" >> $GITHUB_ENV
+          fi
+
+          # secrets
+
+          if [ -n "${{ secrets.AZURE_OPENAI_API_KEY }}" ]; then
+            echo "AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ secrets.TOGETHER_API_KEY }}" ]; then
+            echo "TOGETHER_API_KEY=${{ secrets.TOGETHER_API_KEY }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then
+            echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ secrets.ANTHROPIC_API_KEY }}" ]; then
+            echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ secrets.BING_API_KEY }}" ]; then
+            echo "BING_API_KEY=${{ secrets.BING_API_KEY }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ secrets.POSTMAN_API_KEY }}" ]; then
+            echo "POSTMAN_API_KEY=${{ secrets.POSTMAN_API_KEY }}" >> $GITHUB_ENV
+          fi
+
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+          cache: "pip"
+          cache-dependency-path: pyproject.toml
+      - uses: actions/cache@v4
+        id: cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-test-v03
+      - name: Install Dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: pip install .[docs,testing]
+      - name: Install Pydantic v2
+        run: pip install --pre "pydantic>=2,<3"
+      - run: mkdir coverage
+      - uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+      - name: Install dependencies
+        run: npm ci
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+      - name: Start fastagency
+        id: fastagency-start
+        run: |
+            # Start fastagency and grab its pid
+            nohup gunicorn --workers=1  e2e.llm-sans.main:app >nohup.txt 2>nohup-error.txt &
+            # Get the process ID (PID)
+            FAST_PID=$!
+            echo "Started fastagency with PID: $FAST_PID"
+            echo "FAST_PID=$FAST_PID" >> $GITHUB_OUTPUT
+        env:
+            COVERAGE_PROCESS_START: "e2e/playwright.coverage.cfg"
+      - run: echo "obtained FAST_PID" ${{ steps.fastagency-start.outputs.FAST_PID}}
+      - name: Run Playwright tests without LLMs
+        if: ${{ inputs.python-version != '3.9' }}
+        run: npx playwright test -c "playwright.coverage.config.ts"
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }}
+          path: playwright-report/
+          retention-days: 30
+
+      - name: Kill the program
+        run: |
+            PID="${{ steps.fastagency-start.outputs.FAST_PID}}"
+            # Send SIGTERM to the program (graceful shutdown)
+            echo "killing the fastagency:" $PID
+            ps -ef | grep $PID
+            kill -s SIGTERM $PID
+
+            wait_count=0
+            while ps -p "$PID" > /dev/null && ((wait_count < 5)); do
+              echo "Fastagency with PID $PID is still running..."
+              sleep 10
+              wait_count=$((wait_count + 1))
+            done
+            if ((wait_count == 5)); then
+              echo "Fastagency with PID $PID did not exit after 5 timeouts.  Terminating..."
+              kill -9 $PID
+            else
+              echo "Fastagency with PID $PID has exited."
+            fi
+
+      - run: ls -al .coverage
+      - run: cat nohup.txt
+      - run: cat nohup-error.txt
+      - name: Report
+        run: coverage report  --no-skip-covered
+      - name: Move coverage file
+        run: mv .coverage coverage/.coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }}
+      - name: Check coverage file
+        run: ls -al coverage
+      - name: Store coverage files
+        if: ${{ inputs.python-version != '3.9' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: .coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }}
+          path: coverage/.coverage.playwright.*
+          if-no-files-found: error
+          overwrite: true
+          include-hidden-files: true

.github/workflows/test.yaml

@@ -92,10 +92,6 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: lts/*
-      - name: Install dependencies
-        run: npm ci
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
       - name: Test without LLMs
         if: ${{ inputs.use-llms == ''}}
         run: bash scripts/test.sh -vv -m "not (anthropic or azure_oai or openai or togetherai or llm)"
@@ -108,15 +104,6 @@ jobs:
         env:
           COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }}
           CONTEXT: ${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }}
-      - name: Run Playwright tests without LLMs
-        if: ${{ inputs.python-version != '3.9' }}
-        run: npx playwright test -c "playwright.llm-sans.config.ts"
-      - uses: actions/upload-artifact@v4
-        if: ${{ !cancelled() }}
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
       - name: Check coverage file
         run: ls -al coverage
       - name: Store coverage files

e2e/llm-sans/messages.spec.ts

@@ -86,7 +86,7 @@ test('workflow started', async ({ page }) => {
     await page.goto('/');
     const startWorkflow = await page.getByRole('button', { name: 'Workflow started' })
     await startWorkflow.click()
-    const started = await page.getByText("Workflow started")
+    const started = await page.getByText("Workflow started: AutoGenWorkflows")
     await expect(started).toBeVisible()
     const name = await page.getByText('_workflow_started_')
     await expect(name).toBeVisible()

e2e/playwright.base.config.ts

@@ -17,18 +17,21 @@ import { devices, defineConfig as originalDefineConfig, PlaywrightTestConfig } f
  * See https://playwright.dev/docs/test-configuration.
  */
 
+//const testEnv = { ...process.env, COVERAGE_PROCESS_START: 'playwright.coverage.cfg' } as { [key: string]: string }
+
+
 const baseConfig: PlaywrightTestConfig = {
     testDir: './e2e',
     /* Run tests in files in parallel */
-    fullyParallel: true,
+    fullyParallel: false,
     /* Fail the build on CI if you accidentally left test.only in the source code. */
     forbidOnly: !!process.env.CI,
     /* Retry on CI only */
     retries: process.env.CI ? 2 : 0,
     /* Opt out of parallel tests on CI. */
-    workers: process.env.CI ? 1 : undefined,
+    workers: process.env.CI ? 1 : 1,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-    reporter: 'list',
+    reporter: 'html',
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
         /* Base URL to use in actions like `await page.goto('/')`. */
@@ -76,12 +79,15 @@ const baseConfig: PlaywrightTestConfig = {
         // },
     ],
 
-    /* Run your local dev server before starting the tests */
-    webServer: {
-        command: 'fastagency run e2e/llm-sans/main.py',
-        url: 'http://127.0.0.1:32123',
-        reuseExistingServer: !process.env.CI,
-    },
+    // /* Run your local dev server before starting the tests */
+    // webServer: {
+    //     command: 'fastagency run e2e/llm-sans/main.py',
+    //     url: 'http://127.0.0.1:32123',
+    //     //        env: testEnv,
+    //     reuseExistingServer: true,
+    //     //reuseExistingServer: !process.env.CI,
+    //     //reuseExistingServer: false,
+    // },
 }
 
 export function defineConfig(config: PlaywrightTestConfig): PlaywrightTestConfig {

e2e/playwright.coverage.cfg

@@ -0,0 +1,26 @@
+[coverage:run]
+branch = True
+sigterm = True
+context = playwright
+
+[coverage:report]
+omit =
+    /tmp/*
+
+exclude_also =
+    ; Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    ; Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    ; Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+    ; Don't complain about abstract methods, they aren't run:
+    @(abc\.)?abstractmethod
+
+ignore_errors = True

fastagency/app.py

@@ -1,5 +1,6 @@
 __all__ = ["FastAgency"]
 
+import os
 from collections.abc import Awaitable, Generator
 from contextlib import contextmanager
 from typing import Any, Callable, Optional, Union
@@ -37,6 +38,18 @@
             title (Optional[str], optional): The title of the FastAgency. If None, the default string will be used. Defaults to None.
             description (Optional[str], optional): The description of the FastAgency. If None, the default string will be used. Defaults to None.
         """
+        # check if we need to start coverage
+        logger.info("Checking if coverage is needed.")
+        coverage_process_start = os.environ.get("COVERAGE_PROCESS_START")
+        if coverage_process_start:
+            logger.info("Coverage process start detected")
+            logger.info(f"Coverage configuration file: {coverage_process_start}")
+            logger.info(
+                "To ensure coverage is written out, terminate this program with SIGTERM"
+            )
+            import coverage
+
+            coverage.process_startup()
         _self: Runnable = self
         self._title = title or "FastAgency application"
         self._description = description or "FastAgency application"

playwright.coverage.config.ts

@@ -0,0 +1,15 @@
+// playwright setting for tests that run with unicorn
+import { defineConfig } from "./e2e/playwright.base.config.ts";
+
+export default defineConfig(
+    {
+        testDir: './e2e/llm-sans',
+        use: {
+            /* Base URL to use in actions like `await page.goto('/')`. */
+            baseURL: 'http://127.0.0.1:8000',
+
+            /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+            trace: 'on-first-retry',
+        },
+    }
+)

playwright.gunicorn.config.ts

@@ -3,7 +3,7 @@ import { defineConfig } from "./e2e/playwright.base.config.ts";
 
 export default defineConfig(
     {
-        testDir: './e2e/llm',
+        testDir: './e2e/llm-sans',
         use: {
             /* Base URL to use in actions like `await page.goto('/')`. */
             baseURL: 'http://127.0.0.1:8000',

playwright.llm-sans.config.ts

@@ -3,5 +3,11 @@ import { defineConfig } from "./e2e/playwright.base.config.ts";
 export default defineConfig(
     {
         testDir: './e2e/llm-sans',
-    }
+        webServer: {
+            command: 'fastagency run e2e/llm-sans/main.py',
+            url: 'http://127.0.0.1:32123',
+            //        env: testEnv,
+            reuseExistingServer: true,
+        },
+    },
 )