tests.yaml

# This file has transitioning to run almost everything, with rules defined in
# this file rather than across lots of workflow files.
name: tests

on:
  pull_request:
    # Add `labeled`, so we can trigger a new run by adding a `pr-nightly`
    # label, which we then use to trigger a `nightly` run.
    types: [opened, reopened, synchronize, labeled]
    branches:
      - main
  push:
    branches:
      - main
  schedule:
    # Pick a random time, something that others won't pick, to reduce GH's demand variance.
    - cron: "49 10 * * *"
  workflow_dispatch:
  workflow_call:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  CLICOLOR_FORCE: 1
  RUSTFLAGS: "-C debuginfo=0"
  RUSTDOCFLAGS: "-Dwarnings"

jobs:
  # This assesses whether we need to run jobs. Some of them are defined only by
  # the changes in PR, others also define a set of other criteria, such as
  # whether a label has been added, or we're on `main` branch.
  rules:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      book: ${{ steps.changes.outputs.book }}
      dotnet: ${{ steps.changes.outputs.dotnet }}
      devcontainer-push: ${{ steps.devcontainer-push.outputs.run }}
      devcontainer-build: ${{ steps.devcontainer-build.outputs.run }}
      elixir: ${{ steps.changes.outputs.elixir }}
      grammars: ${{ steps.changes.outputs.grammars }}
      java: ${{ steps.changes.outputs.java }}
      js: ${{ steps.changes.outputs.js }}
      prqlc-c: ${{ steps.changes.outputs.prqlc-c }}
      # Run tests such as rust tests for all-OSs, and bindings tests on ubuntu.
      # Somewhat a tradeoff between coverage and ensuring our CI queues stay
      # short.
      main: ${{ steps.main.outputs.run }}
      # Run all tests
      nightly: ${{ steps.nightly.outputs.run }}
      # Really run all tests (e.g. taskfile & devcontainer)
      nightly-schedule: ${{ steps.nightly-schedule.outputs.run }}
      php: ${{ steps.changes.outputs.php }}
      python: ${{ steps.changes.outputs.python }}
      rust: ${{ steps.changes.outputs.rust }}
      taskfile: ${{ steps.changes.outputs.taskfile }}
      web: ${{ steps.changes.outputs.web }}

    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: changes
        with:
          filters: |
            book:
              - .github/workflows/check-links-book.yaml
              - web/book/**
            dotnet:
              - prqlc/bindings/prql-dotnet/**
              - prqlc/bindings/prqlc-c/**
              - .github/workflows/test-dotnet.yaml
            devcontainer-push:
              - .devcontainer/**/*Dockerfile
              - Taskfile.yml
            devcontainer-build:
              - .devcontainer/**/*Dockerfile
              - .github/workflows/build-devcontainer.yaml
              - Taskfile.yml
            grammars:
              - grammars/**
            elixir:
              - prqlc/bindings/elixir/**
              - prqlc/bindings/prqlc-c/**
              - .github/workflows/test-elixir.yaml
            java:
              - prqlc/bindings/java/**
              - prqlc/bindings/prqlc-c/**
              - .github/workflows/test-java.yaml
            js:
              - prqlc/bindings/js/**
              - .github/workflows/test-js.yaml
            prqlc-c:
              - prqlc/bindings/prqlc-c/**
              - .github/workflows/test-prqlc-c.yaml
            main:
              - "**/Cargo.*"
              - .github/**
            nightly:
              - .github/workflows/nightly.yaml
              - .github/workflows/release.yaml
              - Cargo.lock
              - rust-toolchain.toml
              - .cargo/**
            php:
              - prqlc/bindings/php/**
              - prqlc/bindings/prqlc-c/**
              - .github/workflows/test-php.yaml
            python:
              - prqlc/bindings/prqlc-python/**
              - .github/workflows/test-python.yaml
            rust:
              - "**/*.rs"
              - prqlc/**
              - web/book/**
              - .github/workflows/test-rust.yaml
            taskfile:
              - Taskfile.yml
            web:
              - "web/**"
              - ".github/workflows/build-web.yaml"
              - "**.md"

      # We put a few of the more complex rules as steps here, rather than having
      # them inline. There's no strict delineation between logic here vs. inline.

      - id: nightly
        # TODO: actionlint annoyingly blocks this — try and find a way of getting
        # it back without too much trouble...
        # contains(github.event.pull_request.title, '!') ||
        run:
          echo "run=${{ steps.changes.outputs.nightly == 'true' ||
          contains(github.event.pull_request.labels.*.name, 'pr-nightly') ||
          github.event_name == 'schedule' }}" >>"$GITHUB_OUTPUT"

      - id: nightly-schedule
        run:
          echo "run=${{ github.event_name == 'schedule' }}" >>"$GITHUB_OUTPUT"

      - id: main
        run:
          echo "run=${{ steps.changes.outputs.main == 'true' || github.ref ==
          'refs/heads/main' || steps.nightly.outputs.run == 'true' }}" >>
          "$GITHUB_OUTPUT"

      - id: devcontainer-push
        # We push the devcontainer if the files have changed, and we've merged
        # to main.
        run:
          echo "run=${{ steps.changes.outputs.devcontainer-push == 'true' &&
          github.ref == 'refs/heads/main' && github.event_name == 'push' }}" >>
          "$GITHUB_OUTPUT"

      - id: devcontainer-build
        run:
          echo "run=${{ steps.devcontainer-push.outputs.run == 'true' ||
          steps.changes.outputs.devcontainer-build == 'true' ||
          steps.changes.outputs.nightly-schedule == 'true' }}" >>
          "$GITHUB_OUTPUT"

  test-rust:
    needs: rules
    if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/test-rust.yaml
    strategy:
      matrix:
        include:
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-latest
            features: default,test-dbs-external
          # Only run wasm on ubuntu, given it's the same rust target. (There is
          # a possibility of having a failure on just one platform, but it's
          # quite unlikely. If we do observe this, we can add those tests them
          # to nightly.
          - target: wasm32-unknown-unknown
            os: ubuntu-latest
            features: default
    with:
      os: ubuntu-latest
      target: ${{ matrix.target }}
      features: ${{ matrix.features }}
      nightly: ${{ needs.rules.outputs.nightly == 'true' }}

  test-python:
    needs: rules
    if:
      needs.rules.outputs.python == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/test-python.yaml
    with:
      # Only run on ubuntu unless there's a lang-specific change or we're
      # running nightly.
      #
      # An alternative to these somewhat horrible expressions would be
      # `test-python` & `test-python-more` workflows; though it would use up our
      # 20 workflow limit.
      oss:
        ${{ (needs.rules.outputs.python == 'true' || needs.rules.outputs.nightly
        == 'true') && '["ubuntu-latest", "macos-14", "windows-latest"]' ||
        '["ubuntu-latest"]' }}

  test-js:
    needs: rules
    if: needs.rules.outputs.js == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/test-js.yaml
    with:
      # Only run on ubuntu unless there's a lang-specific change or we're running nightly.
      oss:
        ${{ (needs.rules.outputs.js == 'true' || needs.rules.outputs.nightly ==
        'true') && '["ubuntu-latest", "macos-14", "windows-latest"]' ||
        '["ubuntu-latest"]' }}

  test-dotnet:
    needs: rules
    if:
      needs.rules.outputs.dotnet == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/test-dotnet.yaml

  test-php:
    needs: rules
    if: needs.rules.outputs.php == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/test-php.yaml

  test-java:
    needs: rules
    if: needs.rules.outputs.java == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/test-java.yaml
    with:
      # Currently we never run windows
      oss:
        ${{ (needs.rules.outputs.java == 'true' || needs.rules.outputs.nightly
        == 'true') && '["ubuntu-latest", "macos-14"]' || '["ubuntu-latest"]' }}

  test-elixir:
    needs: rules
    if:
      needs.rules.outputs.elixir == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/test-elixir.yaml
    with:
      # Currently we never run Mac, see prql-elixir docs for details
      oss:
        ${{ (needs.rules.outputs.elixir == 'true' || needs.rules.outputs.nightly
        == 'true') && '["ubuntu-latest", "windows-latest"]' ||
        '["ubuntu-latest"]' }}

  test-prqlc-c:
    needs: rules
    if:
      needs.rules.outputs.prqlc-c == 'true' || needs.rules.outputs.main ==
      'true'
    uses: ./.github/workflows/test-prqlc-c.yaml

  test-taskfile:
    needs: rules
    if:
      # We only run on nightly scheduled, since this is very expensive and we
      # don't want to have to run it on, for example, every dependency change.
      needs.rules.outputs.taskfile == 'true' ||
      needs.rules.outputs.nightly-schedule == 'true'
    runs-on: macos-14
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - run: ./.github/workflows/scripts/set_version.sh
      - name: 💰 Cache
        uses: Swatinem/rust-cache@v2
        with:
          prefix-key: ${{ env.version }}
          # The mac rust cache key. It's not _that_ useful since this will build
          # much more, but it's better than nothing. We can't have our own
          # cache, since we're out of cache space and this workflow takes 1.5GB.
          shared-key: rust-x86_64-apple-darwin
          save-if: false
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Install Task
        uses: arduino/setup-task@v2
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      # Required because of https://github.com/cargo-bins/cargo-binstall/issues/1254
      - run: brew install bash
      - run: task install-brew-dependencies
      - run: task setup-dev
      # This also encompasses `build-all`
      - run: task test-all
      - run: task test-rust-fast
      - run: task test-lint

  test-rust-main:
    needs: rules
    if: needs.rules.outputs.main == 'true'
    strategy:
      matrix:
        include:
          - os: macos-14
            target: aarch64-apple-darwin
            features: default,test-dbs
          - os: windows-latest
            target: x86_64-pc-windows-msvc
            # We'd like to reenable integration tests on Windows, ref https://github.com/wangfenjin/duckdb-rs/issues/179.
            features: default
          - os: ubuntu-latest
            target: x86_64-unknown-linux-gnu
            # One test with no features
            features: ""

          # TODO: potentially enable these
          # - os: ubuntu-latest
          #   target: aarch64-unknown-linux-musl

    uses: ./.github/workflows/test-rust.yaml
    with:
      os: ${{ matrix.os }}
      target: ${{ matrix.target }}
      features: ${{ matrix.features }}

  build-web:
    needs: rules
    if: needs.rules.outputs.web == 'true' || needs.rules.outputs.main == 'true'
    uses: ./.github/workflows/build-web.yaml

  lint-megalinter:
    uses: ./.github/workflows/lint-megalinter.yaml

  publish-web:
    uses: ./.github/workflows/publish-web.yaml
    if: contains(github.event.pull_request.labels.*.name, 'pr-publish-web')

  nightly:
    needs: rules
    uses: ./.github/workflows/nightly.yaml
    if: needs.rules.outputs.nightly == 'true'

  check-links-markdown:
    needs: rules
    # Another option is https://github.com/lycheeverse/lychee, but it was
    # weirdly difficult to exclude a directory, and I managed to get
    # rate-limited by GH because of it scanning node_modules.
    runs-on: ubuntu-latest
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: gaurav-nelson/github-action-markdown-link-check@v1
        with:
          # Run on all files in nightly schedule, but only on local files
          # otherwise (and only changed files in PRs)
          config-file:
            ${{ needs.rules.outputs.nightly-schedule == 'true' &&
            '.config/.markdown-link-check-all.json' ||
            '.config/.markdown-link-check-local.json' }}
          base-branch: main
          check-modified-files-only:
            ${{ needs.rules.outputs.nightly == 'true' && 'no' || 'yes' }}

  check-links-book:
    # We also have a check-links-markdown job, however it will not spot mdbook
    # mistakes such as forgetting to list an .md file in SUMMARY.md.
    # Running a link checker on the generated HTML is more reliable.
    needs: rules
    if:
      needs.rules.outputs.book == 'true' || needs.rules.outputs.nightly ==
      'true'

    runs-on: ubuntu-latest

    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: baptiste0928/cargo-install@v3
        with:
          crate: mdbook
      # the link checker
      - uses: baptiste0928/cargo-install@v3
        with:
          crate: hyperlink
      - run: ./.github/workflows/scripts/set_version.sh
      - name: Cache
        uses: Swatinem/rust-cache@v2
        with:
          prefix-key: ${{ env.version }}
          shared-key: web
          # Created by `build-web`
          save-if: false
      # Only build the book — rather than `build-web` which also builds the playground
      - name: Build the mdbook
        run: mdbook build web/book/
      - name: Check links
        run: hyperlink web/book/book/

  measure-code-cov:
    runs-on: ubuntu-latest
    needs: rules
    # Annoyingly, dependabot doesn't get the `${{ secrets.CODECOV_TOKEN }}`
    # secret, but codecov expects one given it's not a forked PR. So to avoid
    # failing, we skip this. (Ideally dependabot would either come from a fork
    # and not be this special case...). If codecov.io either recognizes this
    # case or stops requiring tokens while remaining reliable, we can remove
    # this exception.
    if: github.actor != 'dependabot[bot]'
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - run: ./.github/workflows/scripts/set_version.sh
      - uses: baptiste0928/cargo-install@v3
        with:
          crate: cargo-llvm-cov
      - name: 💰 Cache
        uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}
          prefix-key: ${{ env.version }}
      - run:
          # TODO: move to using `codecov.json` since that has branch & region
          # coverage. But the coverage is lower, and I haven't spent the time to
          # evaluate whether that's accurate or not yet.
          #
          # cargo llvm-cov --codecov --output-path=codecov.json --features=test-dbs
          cargo llvm-cov --cobertura --output-path=cobertura.xml
          --no-default-features --features=default,test-dbs
      - name: Upload to codecov.io
        uses: codecov/codecov-action@v4
        with:
          # files: codecov.json
          files: cobertura.xml
          fail_ci_if_error: true
          # This isn't strictly necessary, but as discussed in
          # https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954,
          # without this the upload has a fairly high failure rate. That also
          # suggests copying the token into the repo in plain text to ensure it
          # doesn't also fail on forks. The only thing the token allows is
          # uploading coverage, so there's no material security risk. (The token
          # is at https://app.codecov.io/gh/PRQL/prql/settings if we need to do
          # that.)
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Upload code coverage results
        uses: actions/upload-artifact@v3
        with:
          name: code-coverage-report
          # path: codecov.json
          path: cobertura.xml

  test-grammars:
    # Currently tests lezer grammars. We could split that out into a separate
    # job if we want when we add more.
    runs-on: ubuntu-latest
    needs: rules
    if:
      needs.rules.outputs.grammars == 'true' || needs.rules.outputs.main ==
      'true'
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - name: 🧅 Setup Bun
        uses: oven-sh/setup-bun@v1
        with:
          bun-version: latest
      - name: Install dependencies
        working-directory: grammars/prql-lezer/
        run: bun install
      - name: Build grammar
        working-directory: grammars/prql-lezer/
        run: bun run build
      - name: Test grammar
        working-directory: grammars/prql-lezer/
        run: bun run test

  build-devcontainer:
    needs: rules
    if: needs.rules.outputs.devcontainer-build == 'true'
    uses: ./.github/workflows/build-devcontainer.yaml
    # One problem with this setup is that if another commit is merged to main,
    # this workflow will cancel existing jobs, and so this won't get pushed. We
    # have another workflow which runs on each release, so the image should get
    # pushed eventually. The alternative is to have a separate workflow, but
    # then we can't use the nice logic of when to run the workflow that we've
    # built up here.
    with:
      # This needs to compare to the string `'true'`, because of GHA awkwardness
      push: ${{ needs.rules.outputs.devcontainer-push == 'true' }}

  time-compilation:
    runs-on: ubuntu-latest
    needs: rules
    if: needs.rules.outputs.nightly == 'true'
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: ./.github/actions/time-compilation
        with:
          use_cache: true

  test-msrv:
    runs-on: ubuntu-latest
    needs: rules
    if: needs.rules.outputs.nightly == 'true'
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: baptiste0928/cargo-install@v3
        with:
          crate: cargo-msrv
        # Note this currently uses a manually maintained key in
        # `prqlc/prqlc/Cargo.toml`, because of
        # https://github.com/foresterre/cargo-msrv/issues/590
      - name: Verify minimum rust version — prqlc
        # Ideally we'd check all crates, ref https://github.com/foresterre/cargo-msrv/issues/295
        working-directory: prqlc/prqlc
        run: cargo msrv verify

  test-deps-min-versions:
    runs-on: ubuntu-latest
    needs: rules
    if: needs.rules.outputs.nightly == 'true'
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - run: rustup override set nightly-2023-11-22
      - uses: baptiste0928/cargo-install@v3
        with:
          crate: cargo-hack
      - uses: baptiste0928/cargo-install@v3
        with:
          crate: cargo-minimal-versions
      - run: ./.github/workflows/scripts/set_version.sh
      - name: 💰 Cache
        uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}
          prefix-key: ${{ env.version }}
      - name: Verify minimum rust version
        run: cargo minimal-versions test --direct

  check-ok-to-merge:
    # This indicates to GitHub whether everything in this workflow has passed
    # and (unlike if we included each task in the branch's GitHub required
    # tests) will pass when a task is skipped.
    if: always()
    needs:
      - build-devcontainer
      - build-web
      - check-links-book
      - check-links-markdown
      - lint-megalinter
      - nightly
      - publish-web
      - test-deps-min-versions
      - test-dotnet
      - test-elixir
      - test-grammars
      - test-java
      - test-js
      - test-msrv
      - test-php
      - test-prqlc-c
      - test-python
      - test-rust
      - test-rust-main
      - test-taskfile
      - time-compilation
    runs-on: ubuntu-latest
    steps:
      - name: Decide whether the needed jobs succeeded or failed
        # https://github.com/re-actors/alls-green/issues/23
        uses: re-actors/alls-green@cf9edfcf932a0ed6b431433fa183829c68b30e3f
        with:
          jobs: ${{ toJSON(needs) }}
          # We don't include `check-links-markdown`, since occasionally we'll want to merge
          # something which temporarily fails that, such as if we're changing the
          # location of a file in this repo which is linked to.
          #
          # We're currently including `nightly` because I'm not sure whether
          # it's always reliable; e.g. `cargo-audit`
          allowed-failures: |
            [
              "check-links-markdown",
              "nightly"
            ]
          # We skip jobs deliberately, so we are OK if any are skipped.
          #
          # Copy-pasted from `needs`, since it needs to be a json list, so `${{
          # toJSON(needs) }}` (which is a map) doesn't work.
          # https://github.com/re-actors/alls-green/issues/23
          allowed-skips: |
            [
              "build-devcontainer",
              "build-web",
              "check-links-book",
              "check-links-markdown",
              "lint-megalinter",
              "measure-code-cov",
              "nightly",
              "publish-web",
              "test-deps-min-versions",
              "test-dotnet",
              "test-elixir",
              "test-grammars",
              "test-java",
              "test-js",
              "test-msrv",
              "test-php",
              "test-prqlc-c",
              "test-python",
              "test-rust",
              "test-rust-main",
              "test-taskfile",
              "time-compilation"
            ]

  build-prqlc:
    runs-on: ${{ matrix.os }}
    needs: rules
    if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true'
    strategy:
      fail-fast: false
      matrix:
        include:
          # Match the features with the available caches from tests
          - os: ubuntu-latest
            target: x86_64-unknown-linux-musl
            features: default
          # TODO: Until we have tests for these, we don't have a cache for them.
          # If we can add tests, then re-enable them. They run on `release.yaml`
          # regardless.
          #
          # - os: ubuntu-latest
          #   target: aarch64-unknown-linux-musl
          - os: macos-14
            target: aarch64-apple-darwin
            features: default,test-dbs
          - os: windows-latest
            target: x86_64-pc-windows-msvc
            features: default
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: ./.github/actions/build-prqlc
        with:
          target: ${{ matrix.target }}
          profile: dev
          features: ${{ matrix.features }}
    # These are the same env variables as in `test-rust.yaml`. Custom actions
    # don't allow setting env variables for the whole job, so we do it here.
    env:
      CARGO_TERM_COLOR: always
      CLICOLOR_FORCE: 1
      RUSTFLAGS: "-C debuginfo=0"
      RUSTDOCFLAGS: "-Dwarnings"

  build-prqlc-c:
    runs-on: ${{ matrix.os }}
    needs: rules
    if: needs.rules.outputs.main == 'true'
    strategy:
      fail-fast: false
      matrix:
        include:
          # Match the features with the available caches from tests
          - os: ubuntu-latest
            target: x86_64-unknown-linux-musl
            features: default
          - os: macos-14
            target: aarch64-apple-darwin
            features: default,test-dbs
          - os: windows-latest
            target: x86_64-pc-windows-msvc
            features: default
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: ./.github/actions/build-prqlc-c
        with:
          target: ${{ matrix.target }}
          profile: dev
          features: ${{ matrix.features }}
    # These are the same env variables as in `test-rust.yaml`. Custom actions
    # don't allow setting env variables for the whole job, so we do it here.
    env:
      CARGO_TERM_COLOR: always
      CLICOLOR_FORCE: 1
      RUSTFLAGS: "-C debuginfo=0"
      RUSTDOCFLAGS: "-Dwarnings"

  create-issue-on-nightly-failure:
    runs-on: ubuntu-latest
    needs:
      - check-ok-to-merge
    if:
      # We check that it's on a schedule as well as it running on nightly — we
      # don't want to trigger just on a `pr-nightly` label
      always() && (github.event_name == 'schedule') && contains(needs.*.result,
      'failure')
    permissions:
      contents: read
      issues: write
    steps:
      - name: 📂 Checkout code
        uses: actions/checkout@v4
      - uses: JasonEtco/create-an-issue@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          LINK:
            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{
            github.run_id }}
        with:
          filename: .github/nightly-failure.md
          update_existing: true
          search_existing: open