From fc799836e494c4875945df16e6c632a67e576833 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 19 Jan 2024 04:09:55 +0000 Subject: [PATCH 1/3] Rename CI tests to minimal/extended tests. Run minimal tests for OpenJDK --- .github/workflows/binding-tests-openjdk.yml | 47 ++++++++++++++++++ ...iew-ci.yml => extended-tests-bindings.yml} | 49 +++++-------------- .github/workflows/merge-check.yml | 4 +- .github/workflows/minimal-tests-bindings.yml | 37 ++++++++++++++ ...e-review-ci.yml => minimal-tests-core.yml} | 6 +-- 5 files changed, 100 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/binding-tests-openjdk.yml rename .github/workflows/{post-review-ci.yml => extended-tests-bindings.yml} (83%) create mode 100644 .github/workflows/minimal-tests-bindings.yml rename .github/workflows/{pre-review-ci.yml => minimal-tests-core.yml} (93%) diff --git a/.github/workflows/binding-tests-openjdk.yml b/.github/workflows/binding-tests-openjdk.yml new file mode 100644 index 0000000000..f2f81b0d8d --- /dev/null +++ b/.github/workflows/binding-tests-openjdk.yml @@ -0,0 +1,47 @@ +name: Run OpenJDK tests + +on: + workflow_call: + inputs: + repo: + type: string + required: true + ref: + type: string + required: true + test-script: + type: string + required: true + +jobs: + test: + runs-on: ubuntu-22.04 + steps: + - name: Checkout MMTk Core + uses: actions/checkout@v2 + with: + path: mmtk-core + - name: Checkout OpenJDK Binding + uses: actions/checkout@v2 + with: + repository: ${{ inputs.repo }} + path: mmtk-openjdk + ref: ${{ inputs.ref }} + - name: Use mmtk-core Rust toolchain for bindings + run: | + cp mmtk-core/rust-toolchain mmtk-openjdk/mmtk + - name: Setup + run: | + cd mmtk-openjdk + ./.github/scripts/ci-checkout.sh + ./.github/scripts/ci-setup.sh + sed -i 's/^mmtk[[:space:]]=/#ci:mmtk=/g' mmtk/Cargo.toml + sed -i 's/^#[[:space:]]mmtk/mmtk/g' mmtk/Cargo.toml + - name: Overwrite MMTk core in openjdk binding + run: | + cp -r mmtk-core mmtk-openjdk/repos/ + - name: Test + run: | + cd mmtk-openjdk + export RUST_BACKTRACE=1 + ./.github/scripts/${{ inputs.test-script }} diff --git a/.github/workflows/post-review-ci.yml b/.github/workflows/extended-tests-bindings.yml similarity index 83% rename from .github/workflows/post-review-ci.yml rename to .github/workflows/extended-tests-bindings.yml index e87c4bbe43..43bd7b2ece 100644 --- a/.github/workflows/post-review-ci.yml +++ b/.github/workflows/extended-tests-bindings.yml @@ -1,4 +1,4 @@ -name: Binding Tests +name: Extended tests - bindings on: pull_request: @@ -19,7 +19,7 @@ jobs: with: pull_request: ${{ github.event.pull_request.number }} - v8-binding-test: + extended-tests-v8: runs-on: ubuntu-20.04 needs: binding-refs if: contains(github.event.pull_request.labels.*.name, 'PR-testing') @@ -56,42 +56,16 @@ jobs: .github/scripts/ci-test.sh .github/scripts/ci-style.sh - openjdk-binding-test: - runs-on: ubuntu-22.04 + extended-tests-openjdk: needs: binding-refs if: contains(github.event.pull_request.labels.*.name, 'PR-testing') - steps: - - name: Checkout MMTk Core - uses: actions/checkout@v2 - with: - path: mmtk-core - - name: Checkout OpenJDK Binding - uses: actions/checkout@v2 - with: - repository: ${{ needs.binding-refs.outputs.openjdk_binding_repo }} - path: mmtk-openjdk - ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }} - - name: Use mmtk-core Rust toolchain for bindings - run: | - cp mmtk-core/rust-toolchain mmtk-openjdk/mmtk - - name: Setup - run: | - cd mmtk-openjdk - ./.github/scripts/ci-checkout.sh - ./.github/scripts/ci-setup.sh - sed -i 's/^mmtk[[:space:]]=/#ci:mmtk=/g' mmtk/Cargo.toml - sed -i 's/^#[[:space:]]mmtk/mmtk/g' mmtk/Cargo.toml - - name: Overwrite MMTk core in openjdk binding - run: | - cp -r mmtk-core mmtk-openjdk/repos/ - - name: Test - run: | - cd mmtk-openjdk - export RUST_BACKTRACE=1 - ./.github/scripts/ci-test.sh - ./.github/scripts/ci-style.sh + uses: ./.github/workflows/binding-tests-openjdk.yml + with: + repo: ${{ needs.binding-refs.outputs.openjdk_binding_repo }} + ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }} + test-script: ci-test-extended.sh - jikesrvm-binding-test: + extended-tests-jikesrvm: runs-on: ubuntu-22.04 needs: binding-refs if: contains(github.event.pull_request.labels.*.name, 'PR-testing') @@ -126,7 +100,7 @@ jobs: ./.github/scripts/ci-test.sh ./.github/scripts/ci-style.sh - julia-binding-test: + extended-tests-julia: runs-on: ubuntu-22.04 needs: binding-refs if: contains(github.event.pull_request.labels.*.name, 'PR-testing') @@ -168,8 +142,7 @@ jobs: ./.github/scripts/ci-test.sh ./.github/scripts/ci-style.sh - - ruby-binding-test: + extended-tests-ruby: runs-on: ubuntu-22.04 needs: binding-refs if: contains(github.event.pull_request.labels.*.name, 'PR-testing') diff --git a/.github/workflows/merge-check.yml b/.github/workflows/merge-check.yml index b8816df2b6..a06d8b2992 100644 --- a/.github/workflows/merge-check.yml +++ b/.github/workflows/merge-check.yml @@ -18,9 +18,9 @@ jobs: with: # Ignore some actions (based on what merge_group triggers): # - this action - # - pre code review checks for stable Rust (we allow them to fail) + # - minimal tests for stable Rust (we allow them to fail) # - binding tests (it may take long to run) - ignoreActions: "ready-to-merge,check-public-api-changes,pre-code-review-checks/x86_64-unknown-linux-gnu/stable,pre-code-review-checks/i686-unknown-linux-gnu/stable,pre-code-review-checks/x86_64-apple-darwin/stable,v8-binding-test,openjdk-binding-test,jikesrvm-binding-test,julia-binding-test,ruby-binding-test (release),ruby-binding-test (debug)" + ignoreActions: "ready-to-merge,check-public-api-changes,minimal-tests-core/x86_64-unknown-linux-gnu/stable,minimal-tests-core/i686-unknown-linux-gnu/stable,minimal-tests-core/x86_64-apple-darwin/stable,v8-binding-test,openjdk-binding-test,jikesrvm-binding-test,julia-binding-test,ruby-binding-test (release),ruby-binding-test (debug)" # This action uses API. We have a quota of 1000 per hour. checkInterval: 600 env: diff --git a/.github/workflows/minimal-tests-bindings.yml b/.github/workflows/minimal-tests-bindings.yml new file mode 100644 index 0000000000..4f25afd580 --- /dev/null +++ b/.github/workflows/minimal-tests-bindings.yml @@ -0,0 +1,37 @@ +name: Minimal tests - bindings + +on: + pull_request: + branches: + - master + merge_group: + branches: + - master + +concurrency: + # Cancels pending runs when a PR gets updated. + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # The workflow will be triggered once a PR is opened, and at that point, we do not have chance to specify which binding repo to use. + # This step allows 2mins before we check comments for binding repos/refs. + grace-period: + runs-on: ubuntu-latest + steps: + - run: sleep 120 + + # Figure out binding PRs. + binding-refs: + needs: grace-period + uses: ./.github/workflows/pr-binding-refs.yml + with: + pull_request: ${{ github.event.pull_request.number }} + + minimal-tests-openjdk: + needs: binding-refs + uses: ./.github/workflows/binding-tests-openjdk.yml + with: + repo: ${{ needs.binding-refs.outputs.openjdk_binding_repo }} + ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }} + test-script: ci-test-minimal.sh diff --git a/.github/workflows/pre-review-ci.yml b/.github/workflows/minimal-tests-core.yml similarity index 93% rename from .github/workflows/pre-review-ci.yml rename to .github/workflows/minimal-tests-core.yml index 23c0944d2b..2ab5e31072 100644 --- a/.github/workflows/pre-review-ci.yml +++ b/.github/workflows/minimal-tests-core.yml @@ -1,4 +1,4 @@ -name: Pre Code Review Checks +name: Minimal tests - mmtk-core on: pull_request: @@ -28,7 +28,7 @@ jobs: export TEST=`cat rust-toolchain` echo "array=[\"$MSRV\", \"$TEST\", \"stable\"]" >> $GITHUB_OUTPUT - pre-code-review-checks: + minimal-tests-core: needs: setup-test-matrix strategy: fail-fast: false @@ -39,7 +39,7 @@ jobs: - { os: macos-12, triple: x86_64-apple-darwin } rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}} - name: pre-code-review-checks/${{ matrix.target.triple }}/${{ matrix.rust }} + name: minimal-tests-core/${{ matrix.target.triple }}/${{ matrix.rust }} runs-on: ${{ matrix.target.os }} steps: From 320379fc64e1659d5d30d409f56fee2f059a95d2 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 19 Jan 2024 04:35:22 +0000 Subject: [PATCH 2/3] Doc about tests --- docs/team/ci.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/team/ci.md diff --git a/docs/team/ci.md b/docs/team/ci.md new file mode 100644 index 0000000000..d67d475c64 --- /dev/null +++ b/docs/team/ci.md @@ -0,0 +1,13 @@ +# Continuous Integration + +## Testing + +MMTK core runs CI tests *before* a pull request is merged. + +MMTk core sets up two sets of tests, the *minimal tests* and the *extended tests*. +* Minimal tests run unconditionally for every pull request and for every commit. This set of tests aims to finish within half an hour. + This include tests from the `mmtk-core` repo and integration tests from binding repos. Integration tests with a binding in the minimal tests should + focus on testing MMTk core features that is exclusively used for the binding. +* Extended tests only run for a pull request if the pull request is tagged with the label `PR-extended-testing`. This set of tests + may take hours, and usually include integration tests with bindings which run the language implementation's standard test suite + as much as possible. From 49a118926ffc7b182cf225f4c1ddf6c1fccd9338 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 19 Jan 2024 21:34:32 +1300 Subject: [PATCH 3/3] Change label to PR-extended-testing --- .github/workflows/extended-tests-bindings.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/extended-tests-bindings.yml b/.github/workflows/extended-tests-bindings.yml index 43bd7b2ece..81a3a57c48 100644 --- a/.github/workflows/extended-tests-bindings.yml +++ b/.github/workflows/extended-tests-bindings.yml @@ -14,7 +14,7 @@ concurrency: jobs: # Figure out binding PRs. binding-refs: - if: contains(github.event.pull_request.labels.*.name, 'PR-testing') + if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing') uses: ./.github/workflows/pr-binding-refs.yml with: pull_request: ${{ github.event.pull_request.number }} @@ -22,7 +22,7 @@ jobs: extended-tests-v8: runs-on: ubuntu-20.04 needs: binding-refs - if: contains(github.event.pull_request.labels.*.name, 'PR-testing') + if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing') steps: - name: Checkout MMTk Core uses: actions/checkout@v2 @@ -58,7 +58,7 @@ jobs: extended-tests-openjdk: needs: binding-refs - if: contains(github.event.pull_request.labels.*.name, 'PR-testing') + if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing') uses: ./.github/workflows/binding-tests-openjdk.yml with: repo: ${{ needs.binding-refs.outputs.openjdk_binding_repo }} @@ -68,7 +68,7 @@ jobs: extended-tests-jikesrvm: runs-on: ubuntu-22.04 needs: binding-refs - if: contains(github.event.pull_request.labels.*.name, 'PR-testing') + if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing') steps: - name: Checkout MMTk Core uses: actions/checkout@v2 @@ -103,7 +103,7 @@ jobs: extended-tests-julia: runs-on: ubuntu-22.04 needs: binding-refs - if: contains(github.event.pull_request.labels.*.name, 'PR-testing') + if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing') steps: - name: Checkout MMTk Core uses: actions/checkout@v2 @@ -145,7 +145,7 @@ jobs: extended-tests-ruby: runs-on: ubuntu-22.04 needs: binding-refs - if: contains(github.event.pull_request.labels.*.name, 'PR-testing') + if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing') strategy: fail-fast: true matrix: