Rename CI tests to minimal/extended tests

.github/workflows/binding-tests-openjdk.yml

@@ -0,0 +1,47 @@
+name: Run OpenJDK tests
+
+on:
+  workflow_call:
+    inputs:
+      repo:
+        type: string
+        required: true
+      ref:
+        type: string
+        required: true
+      test-script:
+        type: string
+        required: true
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+        - name: Checkout MMTk Core
+          uses: actions/checkout@v2
+          with:
+            path: mmtk-core
+        - name: Checkout OpenJDK Binding
+          uses: actions/checkout@v2
+          with:
+            repository: ${{ inputs.repo }}
+            path: mmtk-openjdk
+            ref: ${{ inputs.ref }}
+        - name: Use mmtk-core Rust toolchain for bindings
+          run: |
+            cp mmtk-core/rust-toolchain mmtk-openjdk/mmtk
+        - name: Setup
+          run: |
+            cd mmtk-openjdk
+            ./.github/scripts/ci-checkout.sh
+            ./.github/scripts/ci-setup.sh
+            sed -i 's/^mmtk[[:space:]]=/#ci:mmtk=/g' mmtk/Cargo.toml
+            sed -i 's/^#[[:space:]]mmtk/mmtk/g' mmtk/Cargo.toml
+        - name: Overwrite MMTk core in openjdk binding
+          run: |
+            cp -r mmtk-core mmtk-openjdk/repos/
+        - name: Test
+          run: |
+            cd mmtk-openjdk
+            export RUST_BACKTRACE=1
+            ./.github/scripts/${{ inputs.test-script }}

.github/workflows/post-review-ci.yml → .github/workflows/extended-tests-bindings.yml

@@ -1,4 +1,4 @@
-name: Binding Tests
+name: Extended tests - bindings
 
 on:
   pull_request:
@@ -14,15 +14,15 @@ concurrency:
 jobs:
   # Figure out binding PRs.
   binding-refs:
-    if: contains(github.event.pull_request.labels.*.name, 'PR-testing')
+    if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing')
     uses: ./.github/workflows/pr-binding-refs.yml
     with:
       pull_request: ${{ github.event.pull_request.number }}
 
-  v8-binding-test:
+  extended-tests-v8:
     runs-on: ubuntu-20.04
     needs: binding-refs
-    if: contains(github.event.pull_request.labels.*.name, 'PR-testing')
+    if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing')
     steps:
       - name: Checkout MMTk Core
         uses: actions/checkout@v2
@@ -56,45 +56,19 @@ jobs:
           .github/scripts/ci-test.sh
           .github/scripts/ci-style.sh
 
-  openjdk-binding-test:
-    runs-on: ubuntu-22.04
+  extended-tests-openjdk:
     needs: binding-refs
-    if: contains(github.event.pull_request.labels.*.name, 'PR-testing')
-    steps:
-      - name: Checkout MMTk Core
-        uses: actions/checkout@v2
-        with:
-          path: mmtk-core
-      - name: Checkout OpenJDK Binding
-        uses: actions/checkout@v2
-        with:
-          repository: ${{ needs.binding-refs.outputs.openjdk_binding_repo }}
-          path: mmtk-openjdk
-          ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }}
-      - name: Use mmtk-core Rust toolchain for bindings
-        run: |
-          cp mmtk-core/rust-toolchain mmtk-openjdk/mmtk
-      - name: Setup
-        run: |
-          cd mmtk-openjdk
-          ./.github/scripts/ci-checkout.sh
-          ./.github/scripts/ci-setup.sh
-          sed -i 's/^mmtk[[:space:]]=/#ci:mmtk=/g' mmtk/Cargo.toml
-          sed -i 's/^#[[:space:]]mmtk/mmtk/g' mmtk/Cargo.toml
-      - name: Overwrite MMTk core in openjdk binding
-        run: |
-          cp -r mmtk-core mmtk-openjdk/repos/
-      - name: Test
-        run: |
-          cd mmtk-openjdk
-          export RUST_BACKTRACE=1
-          ./.github/scripts/ci-test.sh
-          ./.github/scripts/ci-style.sh
+    if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing')
+    uses: ./.github/workflows/binding-tests-openjdk.yml
+    with:
+      repo: ${{ needs.binding-refs.outputs.openjdk_binding_repo }}
+      ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }}
+      test-script: ci-test-extended.sh
 
-  jikesrvm-binding-test:
+  extended-tests-jikesrvm:
     runs-on: ubuntu-22.04
     needs: binding-refs
-    if: contains(github.event.pull_request.labels.*.name, 'PR-testing')
+    if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing')
     steps:
       - name: Checkout MMTk Core
         uses: actions/checkout@v2
@@ -126,10 +100,10 @@ jobs:
           ./.github/scripts/ci-test.sh
           ./.github/scripts/ci-style.sh
 
-  julia-binding-test:
+  extended-tests-julia:
     runs-on: ubuntu-22.04
     needs: binding-refs
-    if: contains(github.event.pull_request.labels.*.name, 'PR-testing')
+    if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing')
     steps:
       - name: Checkout MMTk Core
         uses: actions/checkout@v2
@@ -168,11 +142,10 @@ jobs:
           ./.github/scripts/ci-test.sh
           ./.github/scripts/ci-style.sh
 
-
-  ruby-binding-test:
+  extended-tests-ruby:
     runs-on: ubuntu-22.04
     needs: binding-refs
-    if: contains(github.event.pull_request.labels.*.name, 'PR-testing')
+    if: contains(github.event.pull_request.labels.*.name, 'PR-extended-testing')
     strategy:
       fail-fast: true
       matrix:

.github/workflows/merge-check.yml

@@ -18,9 +18,9 @@ jobs:
         with:
           # Ignore some actions (based on what merge_group triggers):
           # - this action
-          # - pre code review checks for stable Rust (we allow them to fail)
+          # - minimal tests for stable Rust (we allow them to fail)
           # - binding tests (it may take long to run)
-          ignoreActions: "ready-to-merge,check-public-api-changes,pre-code-review-checks/x86_64-unknown-linux-gnu/stable,pre-code-review-checks/i686-unknown-linux-gnu/stable,pre-code-review-checks/x86_64-apple-darwin/stable,v8-binding-test,openjdk-binding-test,jikesrvm-binding-test,julia-binding-test,ruby-binding-test (release),ruby-binding-test (debug)"
+          ignoreActions: "ready-to-merge,check-public-api-changes,minimal-tests-core/x86_64-unknown-linux-gnu/stable,minimal-tests-core/i686-unknown-linux-gnu/stable,minimal-tests-core/x86_64-apple-darwin/stable,v8-binding-test,openjdk-binding-test,jikesrvm-binding-test,julia-binding-test,ruby-binding-test (release),ruby-binding-test (debug)"
           # This action uses API. We have a quota of 1000 per hour.
           checkInterval: 600
         env:

.github/workflows/minimal-tests-bindings.yml

@@ -0,0 +1,37 @@
+name: Minimal tests - bindings
+
+on:
+  pull_request:
+    branches:
+      - master
+  merge_group:
+    branches:
+      - master
+
+concurrency:
+  # Cancels pending runs when a PR gets updated.
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # The workflow will be triggered once a PR is opened, and at that point, we do not have chance to specify which binding repo to use.
+  # This step allows 2mins before we check comments for binding repos/refs.
+  grace-period:
+    runs-on: ubuntu-latest
+    steps:
+      - run: sleep 120
+
+  # Figure out binding PRs.
+  binding-refs:
+    needs: grace-period
+    uses: ./.github/workflows/pr-binding-refs.yml
+    with:
+      pull_request: ${{ github.event.pull_request.number }}
+
+  minimal-tests-openjdk:
+    needs: binding-refs
+    uses: ./.github/workflows/binding-tests-openjdk.yml
+    with:
+      repo: ${{ needs.binding-refs.outputs.openjdk_binding_repo }}
+      ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }}
+      test-script: ci-test-minimal.sh

.github/workflows/pre-review-ci.yml → .github/workflows/minimal-tests-core.yml

@@ -1,4 +1,4 @@
-name: Pre Code Review Checks
+name: Minimal tests - mmtk-core
 
 on:
   pull_request:
@@ -28,7 +28,7 @@ jobs:
           export TEST=`cat rust-toolchain`
           echo "array=[\"$MSRV\", \"$TEST\", \"stable\"]" >> $GITHUB_OUTPUT
 
-  pre-code-review-checks:
+  minimal-tests-core:
     needs: setup-test-matrix
     strategy:
       fail-fast: false
@@ -39,7 +39,7 @@ jobs:
           - { os: macos-12, triple: x86_64-apple-darwin }
         rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}}
 
-    name: pre-code-review-checks/${{ matrix.target.triple }}/${{ matrix.rust }}
+    name: minimal-tests-core/${{ matrix.target.triple }}/${{ matrix.rust }}
     runs-on: ${{ matrix.target.os }}
 
     steps:

docs/team/ci.md

@@ -0,0 +1,13 @@
+# Continuous Integration
+
+## Testing
+
+MMTK core runs CI tests *before* a pull request is merged.
+
+MMTk core sets up two sets of tests, the *minimal tests* and the *extended tests*.
+* Minimal tests run unconditionally for every pull request and for every commit. This set of tests aims to finish within half an hour.
+  This include tests from the `mmtk-core` repo and integration tests from binding repos. Integration tests with a binding in the minimal tests should
+  focus on testing MMTk core features that is exclusively used for the binding.
+* Extended tests only run for a pull request if the pull request is tagged with the label `PR-extended-testing`. This set of tests
+  may take hours, and usually include integration tests with bindings which run the language implementation's standard test suite
+  as much as possible.