daily.yml

name: Daily test

on:
  workflow_dispatch:
  schedule:
    - cron: "0 0 * * *"
  pull_request:
    paths:
      - "requirements-tests.txt"
      - ".github/workflows/daily.yml"

# Please keep the permissions minimal, as stubtest runs arbitrary code from pypi.
permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  # A few env vars to speedup brew install
  HOMEBREW_NO_ANALYTICS: 1
  HOMEBREW_NO_AUTOUPDATE: 1
  HOMEBREW_NO_INSTALL_CLEANUP: 1 # Environments are isolated, no need to cleanup old versions
  NONINTERACTIVE: 1 # Required for brew install on CI
  PIP_DISABLE_PIP_VERSION_CHECK: 1
  FORCE_COLOR: 1
  TERM: xterm-256color # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817

jobs:
  stubtest-stdlib:
    name: Check stdlib with stubtest
    if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
      fail-fast: false

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
          cache-dependency-path: requirements-tests.txt
          allow-prereleases: true
          check-latest: true
      - name: Install dependencies
        run: pip install -r requirements-tests.txt
      - name: Run stubtest
        run: python tests/stubtest_stdlib.py

  stubtest-third-party:
    name: Check third party stubs with stubtest
    if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        # As of 2024-10-18, ubuntu-latest can refer to different Ubuntu versions,
        # which causes problems when testing gdb.
        os: ["ubuntu-24.04", "windows-latest", "macos-latest"]
        shard-index: [0, 1, 2, 3]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          # TODO: Use Python 3.12. As of 2024-03-08, several third-party
          # packages fail to install with Python 3.12.
          python-version: "3.11"
          cache: pip
          cache-dependency-path: |
            requirements-tests.txt
            stubs/**/METADATA.toml
      - name: Install dependencies
        run: pip install -r requirements-tests.txt
      - name: Run stubtest
        shell: bash
        run: |
          PACKAGES=$(python tests/get_stubtest_system_requirements.py)

          if [ "${{ runner.os }}" = "Linux" ]; then
            if [ -n "$PACKAGES" ]; then
              sudo apt-get update && sudo apt-get install -y $PACKAGES
            fi

            PYTHON_EXECUTABLE="xvfb-run python"
          else
            if [ "${{ runner.os }}" = "macOS" ] && [ -n "$PACKAGES" ]; then
              brew install -q $PACKAGES
            fi

            if [ "${{ runner.os }}" = "Windows" ] && [ -n "$PACKAGES" ]; then
              choco install -y $PACKAGES
            fi

            PYTHON_EXECUTABLE="python"
          fi

          $PYTHON_EXECUTABLE tests/stubtest_third_party.py --specified-platforms-only --num-shards 4 --shard-index ${{ matrix.shard-index }}

  stub-uploader:
    name: Run the stub_uploader tests
    if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout typeshed
        uses: actions/checkout@v4
        with:
          path: typeshed
      - name: Checkout stub_uploader
        uses: actions/checkout@v4
        with:
          repository: typeshed-internal/stub_uploader
          path: stub_uploader
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Run tests
        run: |
          cd stub_uploader
          uv pip install -r requirements.txt --system
          python -m pytest tests

  # https://gh.neting.ccmunity/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2
  create-issue-on-failure:
    name: Create an issue if daily tests failed
    runs-on: ubuntu-latest
    needs: [stubtest-stdlib, stubtest-third-party, stub-uploader]
    if: ${{ github.repository == 'python/typeshed' && always() && github.event_name == 'schedule' && (needs.stubtest-stdlib.result == 'failure' || needs.stubtest-third-party.result == 'failure' || needs.stub-uploader.result == 'failure') }}
    permissions:
      issues: write
    steps:
      - uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            await github.rest.issues.create({
              owner: "python",
              repo: "typeshed",
              title: `Daily tests failed on ${new Date().toDateString()}`,
              body: "Runs listed here: https://github.com/python/typeshed/actions/workflows/daily.yml",
              labels: ["help wanted"],
            })