backend.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the 'License'); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Backend
on:
  push:
  pull_request:
    branches:
      - dev
    paths-ignore:
      - 'docs/**'
      - '**/*.md'
      - 'seatunnel-ui/**'

concurrency:
  group: backend-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  license-header:
    if: github.repository == '${{github.actor}}/seatunnel'
    name: License header
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - name: Check license header
        uses: apache/skywalking-eyes@985866ce7e324454f61e22eb2db2e998db09d6f3

  code-style:
    if: github.repository == '${{github.actor}}/seatunnel'
    name: Code style
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - name: Check code style
        run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean spotless:check

  dead-link:
    if: github.repository == '${{github.actor}}/seatunnel'
    name: Dead links
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
      - run: sudo npm install -g markdown-link-check@3.8.7
      - run: |
          for file in $(find . -name "*.md"); do
            markdown-link-check -c .dlc.json -q "$file"
          done

  sanity-check:
    if: github.repository == '${{github.actor}}/seatunnel'
    name: Sanity check results
    needs: [ license-header, code-style, dead-link ]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Check results
        run: |
          [[ ${{ needs.license-header.result }} == 'success' ]] || exit 1;
          [[ ${{ needs.code-style.result }} == 'success' ]] || exit 1;
          [[ ${{ needs.dead-link.result }} == 'success' ]] || exit 1;

  changes:
    runs-on: ubuntu-latest
    if: github.repository == '${{github.actor}}/seatunnel'
    timeout-minutes: 10
    outputs:
      api: ${{ steps.filter.outputs.api }}
      engine: ${{ steps.filter.outputs.engine }}
      ut-modules: ${{ steps.ut-modules.outputs.modules }}
      it-modules: ${{ steps.it-modules.outputs.modules }}
    steps:
      - uses: actions/checkout@v3 # required for push event
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11.0'
      - name: Check for file changes
        uses: dorny/paths-filter@v2
        id: filter
        with:
          token: ${{ github.token }}
          list-files: json
          # The following is a single composite pattern that allows next CI steps,
          # the pattern is in form of [not (foo or bar)] to be safe.
          # When new files come in, the CI will NOT ignore them unless listed,
          # so remember to extend here if they do not serve functional purposes.
          # NOTE: careful with using ** in expression, keep !**/{old, new things}.
          filters: |
            cv2:
              - "seatunnel-connectors-v2/**"
            cv2-e2e:
              - "seatunnel-e2e/seatunnel-connector-v2-e2e/**"
            api:
              - "seatunnel-api/**"
              - "seatunnel-common/**"
              - "seatunnel-config/**"
              - "seatunnel-connectors/**"
              - "seatunnel-core/**"
              - "seatunnel-e2e/seatunnel-e2e-common/**"
              - "seatunnel-formats/**"
              - "seatunnel-plugin-discovery/**"
              - "seatunnel-transforms-v2/**"
              - "seatunnel-translation/**"
              - "seatunnel-e2e/seatunnel-transforms-v2-e2e/**"
              - "seatunnel-connectors/**"
              - "pom.xml"
              - "**/workflows/**"
              - "**/tools/**"
            engine:
              - "seatunnel-engine/**"
            engine-e2e:
              - "seatunnel-e2e/seatunnel-engine-e2e/**"
            deleted-poms:
              - deleted: "**/pom.xml" 

      - name: Check Connector V2 Update
        id: cv2-modules
        if: ${{ steps.filter.outputs.cv2 == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.cv2_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py cv2 $update_files`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Connector V2 E2E Update
        id: cv2-e2e-modules
        if: ${{ steps.filter.outputs.cv2-e2e == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.cv2-e2e_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py cv2-e2e $update_files`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Engine Update
        id: engine-modules
        if: ${{ steps.filter.outputs.engine == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.engine_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py engine $update_files`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Engine E2E Update
        id: engine-e2e-modules
        if: ${{ steps.filter.outputs.engine-e2e == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.engine-e2e_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py engine-e2e $update_files`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Deleted Modules
        id: deleted-modules
        if: ${{ steps.filter.outputs.deleted-poms == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.deleted-poms_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py delete $update_files`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT      

      - name: Make unit test modules
        id: ut-modules
        if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '') }}
        run: |
          modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
          modules=${modules: 1}
          pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules`
          # remove deleted modules
          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
          if [[ "zz"$delete_modules != "zz" ]];then
            pl_modules=`python tools/update_modules_check/update_modules_check.py rm $pl_modules $delete_modules`
          fi
          
          if [[ "zz"$pl_modules == "zz" ]];then
            exit 0
          fi
          
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          tree_modules="$modules$sub_modules"
          includes=`python tools/update_modules_check/update_modules_check.py tree $tree_modules`
          ./mvnw -Pci -D"e2e.dependency.skip"=false  dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
          build_modules=`python tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt`
          if [[ "zz"$build_modules == "zz" ]];then
            build_modules=$pl_modules
          fi 
          echo $build_modules
          echo "modules=$build_modules" >> $GITHUB_OUTPUT

      - name: Make integration test modules
        id: it-modules
        if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.cv2-flink-e2e-modules.outputs.modules != '' || steps.cv2-spark-e2e-modules.outputs.modules != '' || steps.engine-e2e-modules.outputs.modules != '') }}
        run: |
          modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
          modules=${modules: 1}
          pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules`
          # remove deleted modules
          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
          if [[ "zz"$delete_modules != "zz" ]];then
            pl_modules=`python tools/update_modules_check/update_modules_check.py rm $pl_modules $delete_modules`
          fi
          
          if [[ "zz"$pl_modules == "zz" ]];then
            exit 0
          fi
          
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          tree_modules="$modules$sub_modules"
          includes=`python tools/update_modules_check/update_modules_check.py tree $tree_modules`
          ./mvnw -Pci -D"e2e.dependency.skip"=false  dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
          build_modules=`python tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt`
          echo $build_modules
          echo "modules=$build_modules" >> $GITHUB_OUTPUT

  dependency-license:
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    name: Dependency licenses
    needs: [ changes, sanity-check ]
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '8'
          cache: 'maven'
      - name: Install
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 40
          max_attempts: 3
          retry_on: error
          command: |
            ./mvnw -B -q install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"license.skipAddThirdParty"
      - name: Check Dependencies Licenses
        run: tools/dependencies/checkLicense.sh

  unit-test:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest', 'windows-latest' ]
    timeout-minutes: 36
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run all modules unit test
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1 clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates
        env:
          MAVEN_OPTS: -Xmx4096m

      - name: run updated modules unit test
        if: needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != ''
        run: |
          ./mvnw -B -T 1 clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  updated-modules-integration-test-part-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run updated modules integration test (part-1)
        if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 0`
          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx2048m

  updated-modules-integration-test-part-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run updated modules integration test (part-2)
        if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 1`
          if [ ! -z $sub_modules ]; then
            ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m

  updated-modules-integration-test-part-3:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run updated modules integration test (part-3)
        if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 2`
          if [ ! -z $sub_modules ]; then
            ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m

  updated-modules-integration-test-part-4:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run updated modules integration test (part-4)
        if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 3`
          if [ ! -z $sub_modules ]; then
            ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m
  updated-modules-integration-test-part-5:
      needs: [ changes, sanity-check ]
      if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
      runs-on: ${{ matrix.os }}
      strategy:
        matrix:
          java: [ '8', '11' ]
          os: [ 'ubuntu-latest' ]
      timeout-minutes: 90
      steps:
        - uses: actions/checkout@v2
        - name: Set up JDK ${{ matrix.java }}
          uses: actions/setup-java@v3
          with:
            java-version: ${{ matrix.java }}
            distribution: 'temurin'
            cache: 'maven'
        - name: run updated modules integration test (part-5)
          if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
          run: |
            sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 4`
            if [ ! -z $sub_modules ]; then
              ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
            else
              echo "sub modules is empty, skipping"
            fi
          env:
            MAVEN_OPTS: -Xmx2048m
  engine-v2-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run seatunnel zeta integration test
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  transform-v2-it-part-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run transform-v2 integration test (part-1)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  transform-v2-it-part-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run transform-v2 integration test (part-2)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run connector-v2 integration test (part-1)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 0`
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run connector-v2 integration test (part-2)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 1`
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-3:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run connector-v2 integration test (part-3)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 2`
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-4:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 150
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run connector-v2 integration test (part-4)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 3`
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-5:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run connector-v2 integration test (part-5)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 4`
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-6:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run connector-v2 integration test (part-6)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 5`
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-7:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run connector-v2 integration test (part-7)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 6`
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run jdbc connectors integration test (part-1)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run jdbc connectors integration test (part-2)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-3:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run jdbc connectors integration test (part-3)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-4:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run jdbc connectors integration test (part-4)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-5:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run jdbc connectors integration test (part-5)
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  kafka-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run kafka connector integration test
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  rocketmq-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run rocket connector integration test
        if: needs.changes.outputs.api == 'true'
        run: |
          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m