[Improvement] [clickhouse-e2e] Enhance ClickHouse E2E testing to trigger multiple checkpoints #14156
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the 'License'); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an 'AS IS' BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| name: Backend | |
| on: | |
| push: | |
| pull_request: | |
| branches: | |
| - dev | |
| paths-ignore: | |
| - 'seatunnel-ui/**' | |
| concurrency: | |
| group: backend-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| license-header: | |
| if: github.repository == '${{github.actor}}/seatunnel' | |
| name: License header | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Check license header | |
| uses: apache/skywalking-eyes@985866ce7e324454f61e22eb2db2e998db09d6f3 | |
| code-style: | |
| if: github.repository == '${{github.actor}}/seatunnel' | |
| name: Code style | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Check code style | |
| run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean spotless:check | |
| dead-link: | |
| if: github.repository == '${{github.actor}}/seatunnel' | |
| name: Dead links | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - run: sudo npm install -g markdown-link-check@3.8.7 | |
| - run: | | |
| for file in $(find . -name "*.md"); do | |
| markdown-link-check -c .dlc.json -q "$file" | |
| done | |
| sanity-check: | |
| if: github.repository == '${{github.actor}}/seatunnel' | |
| name: Sanity check results | |
| needs: [ license-header, code-style, dead-link ] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Check results | |
| run: | | |
| [[ ${{ needs.license-header.result }} == 'success' ]] || exit 1; | |
| [[ ${{ needs.code-style.result }} == 'success' ]] || exit 1; | |
| [[ ${{ needs.dead-link.result }} == 'success' ]] || exit 1; | |
| changes: | |
| runs-on: ubuntu-latest | |
| if: github.repository == '${{github.actor}}/seatunnel' | |
| timeout-minutes: 10 | |
| outputs: | |
| api: ${{ steps.filter.outputs.api }} | |
| engine: ${{ steps.filter.outputs.engine }} | |
| ut-modules: ${{ steps.ut-modules.outputs.modules }} | |
| it-modules: ${{ steps.it-modules.outputs.modules }} | |
| steps: | |
| - uses: actions/checkout@v3 # required for push event | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11.0' | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| token: ${{ github.token }} | |
| list-files: json | |
| # The following is a single composite pattern that allows next CI steps, | |
| # the pattern is in form of [not (foo or bar)] to be safe. | |
| # When new files come in, the CI will NOT ignore them unless listed, | |
| # so remember to extend here if they do not serve functional purposes. | |
| # NOTE: careful with using ** in expression, keep !**/{old, new things}. | |
| filters: | | |
| cv2: | |
| - "seatunnel-connectors-v2/**" | |
| cv2-e2e: | |
| - "seatunnel-e2e/seatunnel-connector-v2-e2e/**" | |
| api: | |
| - "seatunnel-api/**" | |
| - "seatunnel-common/**" | |
| - "seatunnel-config/**" | |
| - "seatunnel-connectors/**" | |
| - "seatunnel-core/**" | |
| - "seatunnel-e2e/seatunnel-e2e-common/**" | |
| - "seatunnel-formats/**" | |
| - "seatunnel-plugin-discovery/**" | |
| - "seatunnel-transforms-v2/**" | |
| - "seatunnel-translation/**" | |
| - "seatunnel-e2e/seatunnel-transforms-v2-e2e/**" | |
| - "seatunnel-connectors/**" | |
| - "pom.xml" | |
| - "**/workflows/**" | |
| - "**/tools/**" | |
| engine: | |
| - "seatunnel-engine/**" | |
| engine-e2e: | |
| - "seatunnel-e2e/seatunnel-engine-e2e/**" | |
| deleted-poms: | |
| - deleted: "**/pom.xml" | |
| - name: Check Connector V2 Update | |
| id: cv2-modules | |
| if: ${{ steps.filter.outputs.cv2 == 'true' }} | |
| run: | | |
| update_files='${{ steps.filter.outputs.cv2_files }}' | |
| modules=`python tools/update_modules_check/update_modules_check.py cv2 $update_files` | |
| echo $modules | |
| echo "modules=$modules" >> $GITHUB_OUTPUT | |
| - name: Check Connector V2 E2E Update | |
| id: cv2-e2e-modules | |
| if: ${{ steps.filter.outputs.cv2-e2e == 'true' }} | |
| run: | | |
| update_files='${{ steps.filter.outputs.cv2-e2e_files }}' | |
| modules=`python tools/update_modules_check/update_modules_check.py cv2-e2e $update_files` | |
| echo $modules | |
| echo "modules=$modules" >> $GITHUB_OUTPUT | |
| - name: Check Engine Update | |
| id: engine-modules | |
| if: ${{ steps.filter.outputs.engine == 'true' }} | |
| run: | | |
| update_files='${{ steps.filter.outputs.engine_files }}' | |
| modules=`python tools/update_modules_check/update_modules_check.py engine $update_files` | |
| echo $modules | |
| echo "modules=$modules" >> $GITHUB_OUTPUT | |
| - name: Check Engine E2E Update | |
| id: engine-e2e-modules | |
| if: ${{ steps.filter.outputs.engine-e2e == 'true' }} | |
| run: | | |
| update_files='${{ steps.filter.outputs.engine-e2e_files }}' | |
| modules=`python tools/update_modules_check/update_modules_check.py engine-e2e $update_files` | |
| echo $modules | |
| echo "modules=$modules" >> $GITHUB_OUTPUT | |
| - name: Check Deleted Modules | |
| id: deleted-modules | |
| if: ${{ steps.filter.outputs.deleted-poms == 'true' }} | |
| run: | | |
| update_files='${{ steps.filter.outputs.deleted-poms_files }}' | |
| modules=`python tools/update_modules_check/update_modules_check.py delete $update_files` | |
| echo $modules | |
| echo "modules=$modules" >> $GITHUB_OUTPUT | |
| - name: Make unit test modules | |
| id: ut-modules | |
| if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '') }} | |
| run: | | |
| modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' | |
| modules=${modules: 1} | |
| pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules` | |
| # remove deleted modules | |
| delete_modules='${{ steps.deleted-modules.outputs.modules }}' | |
| if [[ "zz"$delete_modules != "zz" ]];then | |
| pl_modules=`python tools/update_modules_check/update_modules_check.py rm $pl_modules $delete_modules` | |
| fi | |
| if [[ "zz"$pl_modules == "zz" ]];then | |
| exit 0 | |
| fi | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| tree_modules="$modules$sub_modules" | |
| includes=`python tools/update_modules_check/update_modules_check.py tree $tree_modules` | |
| ./mvnw -Pci -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt | |
| build_modules=`python tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt` | |
| if [[ "zz"$build_modules == "zz" ]];then | |
| build_modules=$pl_modules | |
| fi | |
| echo $build_modules | |
| echo "modules=$build_modules" >> $GITHUB_OUTPUT | |
| - name: Make integration test modules | |
| id: it-modules | |
| if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.cv2-flink-e2e-modules.outputs.modules != '' || steps.cv2-spark-e2e-modules.outputs.modules != '' || steps.engine-e2e-modules.outputs.modules != '') }} | |
| run: | | |
| modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' | |
| modules=${modules: 1} | |
| pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules` | |
| # remove deleted modules | |
| delete_modules='${{ steps.deleted-modules.outputs.modules }}' | |
| if [[ "zz"$delete_modules != "zz" ]];then | |
| pl_modules=`python tools/update_modules_check/update_modules_check.py rm $pl_modules $delete_modules` | |
| fi | |
| if [[ "zz"$pl_modules == "zz" ]];then | |
| exit 0 | |
| fi | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| tree_modules="$modules$sub_modules" | |
| includes=`python tools/update_modules_check/update_modules_check.py tree $tree_modules` | |
| ./mvnw -Pci -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt | |
| build_modules=`python tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt` | |
| echo $build_modules | |
| echo "modules=$build_modules" >> $GITHUB_OUTPUT | |
| dependency-license: | |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' | |
| name: Dependency licenses | |
| needs: [ changes, sanity-check ] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '8' | |
| cache: 'maven' | |
| - name: Install | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 40 | |
| max_attempts: 3 | |
| retry_on: error | |
| command: | | |
| ./mvnw -B -q install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"license.skipAddThirdParty" | |
| - name: Check Dependencies Licenses | |
| run: tools/dependencies/checkLicense.sh | |
| unit-test: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '') | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest', 'windows-latest' ] | |
| timeout-minutes: 36 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run all modules unit test | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1 clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| - name: run updated modules unit test | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '' | |
| run: | | |
| ./mvnw -B -T 1 clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| updated-modules-integration-test-part-1: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run updated modules integration test (part-1) | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| run: | | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 0` | |
| ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx2048m | |
| updated-modules-integration-test-part-2: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run updated modules integration test (part-2) | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| run: | | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 1` | |
| if [ ! -z $sub_modules ]; then | |
| ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci | |
| else | |
| echo "sub modules is empty, skipping" | |
| fi | |
| env: | |
| MAVEN_OPTS: -Xmx2048m | |
| updated-modules-integration-test-part-3: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run updated modules integration test (part-3) | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| run: | | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 2` | |
| if [ ! -z $sub_modules ]; then | |
| ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci | |
| else | |
| echo "sub modules is empty, skipping" | |
| fi | |
| env: | |
| MAVEN_OPTS: -Xmx2048m | |
| updated-modules-integration-test-part-4: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run updated modules integration test (part-4) | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| run: | | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 3` | |
| if [ ! -z $sub_modules ]; then | |
| ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci | |
| else | |
| echo "sub modules is empty, skipping" | |
| fi | |
| env: | |
| MAVEN_OPTS: -Xmx2048m | |
| updated-modules-integration-test-part-5: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run updated modules integration test (part-5) | |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' | |
| run: | | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 4` | |
| if [ ! -z $sub_modules ]; then | |
| ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci | |
| else | |
| echo "sub modules is empty, skipping" | |
| fi | |
| env: | |
| MAVEN_OPTS: -Xmx2048m | |
| engine-v2-it: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run seatunnel zeta integration test | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| transform-v2-it-part-1: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run transform-v2 integration test (part-1) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| transform-v2-it-part-2: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run transform-v2 integration test (part-2) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| all-connectors-it-1: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run connector-v2 integration test (part-1) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 0` | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| all-connectors-it-2: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run connector-v2 integration test (part-2) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 1` | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| all-connectors-it-3: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run connector-v2 integration test (part-3) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 2` | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| all-connectors-it-4: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 150 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run connector-v2 integration test (part-4) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 3` | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| all-connectors-it-5: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run connector-v2 integration test (part-5) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 4` | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| all-connectors-it-6: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run connector-v2 integration test (part-6) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 5` | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| all-connectors-it-7: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run connector-v2 integration test (part-7) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` | |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 7 6` | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| jdbc-connectors-it-part-1: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run jdbc connectors integration test (part-1) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| jdbc-connectors-it-part-2: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run jdbc connectors integration test (part-2) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| jdbc-connectors-it-part-3: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run jdbc connectors integration test (part-3) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| jdbc-connectors-it-part-4: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run jdbc connectors integration test (part-4) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| jdbc-connectors-it-part-5: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run jdbc connectors integration test (part-5) | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| kafka-connector-it: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run kafka connector integration test | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m | |
| rocketmq-connector-it: | |
| needs: [ changes, sanity-check ] | |
| if: needs.changes.outputs.api == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| java: [ '8', '11' ] | |
| os: [ 'ubuntu-latest' ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: run rocket connector integration test | |
| if: needs.changes.outputs.api == 'true' | |
| run: | | |
| ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci | |
| env: | |
| MAVEN_OPTS: -Xmx4096m |