tests.yml

name: Tests

permissions:
  contents: read

on:
  pull_request:
    paths:
      - ".cargo/*.toml"
      - ".github/workflows/tests.yml"
      - "Cargo.*"
      - "mutants_attrs/**"
      - "src/**"
      - "testdata/**"
      - "tests/**"

  push:
    branches:
      - main
    # Actions doesn't support YAML references, so it's repeated here
    paths:
      - ".cargo/*.toml"
      - ".github/workflows/tests.yml"
      - "Cargo.*"
      - "mutants_attrs/**"
      - "src/**"
      - "testdata/**"
      - "tests/**"

# see https://matklad.github.io/2021/09/04/fast-rust-builds.html
env:
  CARGO_TERM_COLOR: always
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  CI: 1
  RUST_BACKTRACE: short
  RUSTFLAGS: "-W rust-2021-compatibility"
  RUSTUP_MAX_RETRIES: 10
  CARGO_MUTANTS_MINIMUM_TEST_TIMEOUT: 60

jobs:
  # Before anything else, run a quick test on just stable: this is significantly
  # faster than Windows or macOS and should catch most issues, and lets us get
  # started on the longer-running mutants and other tests.
  #
  # Also, build a Linux binary that we can use for the later mutants runs.
  quick-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: beta
          components: rustfmt, clippy
      - uses: Swatinem/rust-cache@v2
      - name: Show Cargo and rustc version
        run: |
          cargo --version
          rustc --version
      # TODO: Maybe also check clippy and rustfmt here.
      - name: Build
        run: cargo build --all-targets
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Test
        run: cargo test --workspace
      - name: Check rustfmt
        run: cargo fmt --all --check
      - name: Check clippy
        # TODO:  -- -D warnings
        run: cargo clippy --all-targets --all-features
      - name: Check typos
        uses: crate-ci/typos@master
      - name: Build release binary
        run: cargo build --release
      - name: Upload binary artifact
        uses: actions/upload-artifact@v4
        with:
          name: cargo-mutants-linux
          path: |
            target/release/cargo-mutants

  test:
    needs: [quick-test]
    strategy:
      matrix:
        os: [macOS-latest, ubuntu-latest, windows-latest]
        version: [stable, nightly, "1.74"]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ matrix.version }}
          components: rustfmt, clippy
      - name: Show Cargo and rustc version
        run: |
          cargo --version
          rustc --version
      - uses: Swatinem/rust-cache@v2
      - name: rustfmt
        run: cargo fmt --all -- --check
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Build
        run: cargo build --all-targets
      - name: Test
        run: cargo test --workspace
      - name: Check rustfmt
        run: cargo fmt --all --check
      - name: Check clippy
        # TODO: Deny warnings
        run: cargo clippy --all-targets --all-features
      - run: cargo update
      - name: Test after cargo update
        run: cargo test --workspace
      - name: Downgrade to minimal versions
        if: matrix.version == 'nightly'
        run: cargo +nightly -Zdirect-minimal-versions update
      - name: Test on minimal versions
        if: matrix.version == 'nightly'
        run: cargo test
      - name: Install locked
        run: cargo install --path . --locked
      - name: Install unlocked
        run: cargo install --path .

  tests-from-tarball:
    needs: [quick-test]
    strategy:
      matrix:
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@nightly
      - uses: Swatinem/rust-cache@v2
      - run: cargo package --no-verify
      - name: Upload package artifact
        uses: actions/upload-artifact@v4
        with:
          name: cargo-mutants-package
          path: |
            target/package
      - name: Unpack package
        run: |
          cd target/package
          ls -l
          tar xvf cargo-mutants*.crate
      - name: Install nextest using install-action
        uses: taiki-e/install-action@v2
        with:
          tool: nextest
      - name: Run tests from package
        run: |
          cd target/package/cargo-mutants-*.*.[0-9]
          cargo test

  pr-mutants:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    needs: [quick-test]
    strategy:
      matrix:
        test_tool: [cargo, nextest]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Relative diff
        run: |
          git branch -av
          git diff origin/${{ github.base_ref }}.. | tee git.diff
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: beta
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Download cargo-mutants binary
        uses: actions/download-artifact@v4
        with:
          name: cargo-mutants-linux
      - name: Install cargo-mutants binary
        run: |
          install cargo-mutants ~/.cargo/bin/
      - name: Mutants in-diff
        # Normally this would have --in-place, but for the sake of exercising more cases, it does not.
        run: >
          cargo mutants --no-shuffle -vV --in-diff git.diff
          --test-tool=${{matrix.test_tool}} --timeout=500 --build-timeout=500
      - name: Archive mutants.out
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: mutants-incremental-${{ matrix.test_tool}}.out
          path: mutants.out

  cargo-mutants:
    runs-on: ubuntu-latest
    needs: [quick-test]
    strategy:
      fail-fast: false # We want to get all the mutant failures
      matrix:
        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
        test_tool: [cargo]
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: beta
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Download cargo-mutants binary
        uses: actions/download-artifact@v4
        with:
          name: cargo-mutants-linux
      - name: Install cargo-mutants binary
        run: |
          install cargo-mutants ~/.cargo/bin/
      - name: Mutants
        # Skip baselines because this job only runs after the baseline has been separately run.
        run: >
          cargo mutants --no-shuffle -vV --shard ${{ matrix.shard }}/10
          --test-tool ${{ matrix.test_tool }} --baseline=skip --timeout=500
          --build-timeout=500 --in-place
      - name: Archive mutants.out
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: mutants-${{matrix.test_tool}}-shard${{matrix.shard}}.out
          path: mutants.out

  overall-result:
    needs: [quick-test, test, tests-from-tarball, pr-mutants, cargo-mutants]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Successful workflow
        if: ${{ !(contains(needs.*.result, 'failure')) }}
        run: exit 0
      - name: Failing workflow
        if: ${{ contains(needs.*.result, 'failure') }}
        run: exit 1