Split tests to core/providers/task-sdk/integration/system

The tests execution was traditionally using single "breeze testing
tests" command and you could select which test to run via TEST_TYPE.

However the recent move of providers and adding task_sdk necessitates
splitting the tests commands into separate commands for core, providers,
task_sdk, helm, integration and system.

This is done via introducing "TEST_GROUP" - which determines which
group of tests is being executed, and dedicated testing command for each
of the groups - with "db" and "non-db" variants where applicable.

Cleanup and small refactoring has been done to make it easier to
reason about parameters passed down from the command line to
docker and in-container pytest command.

Related: #42632

.github/workflows/check-providers.yml

@@ -40,7 +40,7 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to upgrade to newer dependencies"
         required: true
         type: string
-      affected-providers-list-as-string:
+      selected-providers-list-as-string:
         description: "List of affected providers as string"
         required: false
         type: string
@@ -54,7 +54,7 @@ on:  # yamllint disable-line rule:truthy
         description: "List of parallel provider test types as string"
         required: true
         type: string
-      skip-provider-tests:
+      skip-providers-tests:
         description: "Whether to skip provider tests (true/false)"
         required: true
         type: string
@@ -163,7 +163,7 @@ jobs:
         run: >
           breeze release-management prepare-provider-packages --include-not-ready-providers
           --version-suffix-for-pypi dev0 --package-format sdist
-          ${{ inputs.affected-providers-list-as-string }}
+          ${{ inputs.selected-providers-list-as-string }}
       - name: "Prepare airflow package: sdist"
         run: >
           breeze release-management prepare-airflow-package
@@ -187,7 +187,7 @@ jobs:
           --providers-constraints-location
           /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
           --run-in-parallel
-        if: inputs.affected-providers-list-as-string == ''
+        if: inputs.selected-providers-list-as-string == ''
       - name: "Install affected provider packages and airflow via sdist files"
         run: >
           breeze release-management install-provider-packages
@@ -198,7 +198,7 @@ jobs:
           --providers-constraints-location
           /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
           --run-in-parallel
-        if: inputs.affected-providers-list-as-string != ''
+        if: inputs.selected-providers-list-as-string != ''
 
   providers-compatibility-checks:
     timeout-minutes: 80
@@ -218,7 +218,7 @@ jobs:
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       VERBOSE: "true"
       CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
-    if: inputs.skip-provider-tests != 'true'
+    if: inputs.skip-providers-tests != 'true'
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -268,7 +268,7 @@ jobs:
           Airflow ${{ matrix.airflow-version }}:Python ${{ matrix.python-version }}
         if: matrix.run-tests == 'true'
         run: >
-          breeze testing tests --run-in-parallel
+          breeze testing providers-tests --run-in-parallel
           --parallel-test-types "${{ inputs.providers-test-types-list-as-string }}"
           --use-packages-from-dist
           --package-format wheel

.github/workflows/helm-tests.yml

@@ -75,7 +75,7 @@ jobs:
       - name: "Prepare breeze & CI image: ${{inputs.default-python-version}}:${{inputs.image-tag}}"
         uses: ./.github/actions/prepare_breeze_and_image
       - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}"
-        run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}"
+        run: breeze testing helm-tests --test-type "${{ matrix.helm-test-package }}"
 
   tests-helm-release:
     timeout-minutes: 80

.github/workflows/integration-tests.yml

@@ -28,8 +28,12 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
-      testable-integrations:
-        description: "The list of testable integrations as JSON array."
+      testable-core-integrations:
+        description: "The list of testable core integrations as JSON array."
+        required: true
+        type: string
+      testable-providers-integrations:
+        description: "The list of testable providers integrations as JSON array."
         required: true
         type: string
       default-postgres-version:
@@ -40,7 +44,7 @@ on:  # yamllint disable-line rule:truthy
         description: "Default version of MySQL to use"
         required: true
         type: string
-      skip-provider-tests:
+      skip-providers-tests:
         description: "Skip provider tests (true/false)"
         required: true
         type: string
@@ -57,22 +61,68 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  tests-integration:
+  tests-core-integration:
+    timeout-minutes: 130
+    if: inputs.testable-core-integrations != '[]'
+    name: "Integration Tests: core ${{ matrix.integration }}"
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        integration: ${{ fromJSON(inputs.testable-core-integrations) }}
+    env:
+      IMAGE_TAG: "${{ inputs.image-tag }}"
+      BACKEND: "postgres"
+      BACKEND_VERSION: ${{ inputs.default-postgres-version }}"
+      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+      JOB_ID: "integration-core-${{ matrix.integration }}"
+      SKIP_PROVIDERS_TESTS: "${{ inputs.skip-providers-tests }}"
+      ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
+      DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
+      GITHUB_REPOSITORY: ${{ github.repository }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_USERNAME: ${{ github.actor }}
+      VERBOSE: "true"
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: "Cleanup docker"
+        run: ./scripts/ci/cleanup_docker.sh
+      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
+        uses: ./.github/actions/prepare_breeze_and_image
+      - name: "Integration Tests: core ${{ matrix.integration }}"
+        # yamllint disable rule:line-length
+        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh core "${{ matrix.integration }}"
+      - name: "Post Tests success"
+        uses: ./.github/actions/post_tests_success
+        with:
+          codecov-token: ${{ secrets.CODECOV_TOKEN }}
+          python-version: ${{ inputs.default-python-version }}
+      - name: "Post Tests failure"
+        uses: ./.github/actions/post_tests_failure
+        if: failure()
+
+  tests-providers-integration:
     timeout-minutes: 130
-    if: inputs.testable-integrations != '[]'
-    name: "Integration Tests: ${{ matrix.integration }}"
+    if: inputs.testable-providers-integrations != '[]'
+    name: "Integration Tests: providers ${{ matrix.integration }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     strategy:
       fail-fast: false
       matrix:
-        integration: ${{ fromJSON(inputs.testable-integrations) }}
+        integration: ${{ fromJSON(inputs.testable-providers-integrations) }}
     env:
       IMAGE_TAG: "${{ inputs.image-tag }}"
       BACKEND: "postgres"
       BACKEND_VERSION: ${{ inputs.default-postgres-version }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      JOB_ID: "integration-${{ matrix.integration }}"
-      SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}"
+      JOB_ID: "integration-providers-${{ matrix.integration }}"
+      SKIP_PROVIDERS_TESTS: "${{ inputs.skip-providers-tests }}"
       ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
       DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
       GITHUB_REPOSITORY: ${{ github.repository }}
@@ -91,13 +141,14 @@ jobs:
         run: ./scripts/ci/cleanup_docker.sh
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
-      - name: "Integration Tests: ${{ matrix.integration }}"
-        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh ${{ matrix.integration }}
-      - name: "Post Tests success: Integration Tests ${{ matrix.integration }}"
+      - name: "Integration Tests: providers ${{ matrix.integration }}"
+        # yamllint disable rule:line-length
+        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh providers "${{ matrix.integration }}"
+      - name: "Post Tests success"
         uses: ./.github/actions/post_tests_success
         with:
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
           python-version: ${{ inputs.default-python-version }}
-      - name: "Post Tests failure: Integration Tests ${{ matrix.integration }}"
+      - name: "Post Tests failure"
         uses: ./.github/actions/post_tests_failure
         if: failure()

.github/workflows/run-unit-tests.yml

@@ -24,6 +24,10 @@ on:  # yamllint disable-line rule:truthy
         description: "The array of labels (in json form) determining default runner used for the build."
         required: true
         type: string
+      test-groups:
+        description: "The json representing list of test test groups to run"
+        required: true
+        type: string
       backend:
         description: "The backend to run the tests on"
         required: true
@@ -61,8 +65,12 @@ on:  # yamllint disable-line rule:truthy
         description: "Excluded combos (stringified JSON array of python-version/backend-version dicts)"
         required: true
         type: string
-      parallel-test-types-list-as-string:
-        description: "The list of parallel test types to run separated by spaces"
+      core-test-types-list-as-string:
+        description: "The list of core test types to run separated by spaces"
+        required: true
+        type: string
+      providers-test-types-list-as-string:
+        description: "The list of providers test types to run separated by spaces"
         required: true
         type: string
       run-migration-tests:
@@ -117,15 +125,16 @@ jobs:
   tests:
     timeout-minutes: 120
     name: "\
-      ${{ inputs.test-scope }}:\
+      ${{ inputs.test-scope }}-${{ matrix.test-group }}:\
       ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\
-      ${{matrix.python-version}}: ${{ inputs.parallel-test-types-list-as-string }}"
+      ${{matrix.python-version}}: ${{ inputs.test-types-list-as-string }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
     strategy:
       fail-fast: false
       matrix:
         python-version: "${{fromJSON(inputs.python-versions)}}"
         backend-version: "${{fromJSON(inputs.backend-versions)}}"
+        test-group: "${{fromJSON(inputs.test-groups)}}"
         exclude: "${{fromJSON(inputs.excludes)}}"
     env:
       # yamllint disable rule:line-length
@@ -145,12 +154,14 @@ jobs:
       IMAGE_TAG: "${{ inputs.image-tag }}"
       INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }}
       # yamllint disable rule:line-length
-      JOB_ID: "${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}"
+      JOB_ID: "${{ matrix.test-group }}-${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}"
       MOUNT_SOURCES: "skip"
-      PARALLEL_TEST_TYPES: "${{ inputs.parallel-test-types-list-as-string }}"
+      # yamllint disable rule:line-length
+      PARALLEL_TEST_TYPES: ${{ matrix.test-group == 'core' && inputs.core-test-types-list-as-string || inputs.providers-test-types-list-as-string }}
       PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
       UPGRADE_BOTO: "${{ inputs.upgrade-boto }}"
       AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}"
+
       VERBOSE: "true"
     steps:
       - name: "Cleanup repo"
@@ -165,33 +176,13 @@ jobs:
       - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
       - name: >
-          Migration Tests:
-          ${{ matrix.python-version }}:${{ inputs.parallel-test-types-list-as-string }}
+          Migration Tests: ${{ matrix.python-version }}:${{ inputs.core-test-types-list-as-string }}
         uses: ./.github/actions/migration_tests
-        if: inputs.run-migration-tests == 'true'
+        if: inputs.run-migration-tests == 'true' && matrix.test-group == 'core'
       - name: >
-          ${{ inputs.test-scope }} Tests ${{ inputs.test-name }} ${{ matrix.backend-version }}
-          Py${{ matrix.python-version }}:${{ inputs.parallel-test-types-list-as-string}}
-        run: |
-          if [[ "${{ inputs.test-scope }}" == "DB" ]]; then
-            breeze testing db-tests \
-              --parallel-test-types "${{ inputs.parallel-test-types-list-as-string }}"
-          elif [[ "${{ inputs.test-scope }}" == "Non-DB" ]]; then
-            breeze testing non-db-tests \
-              --parallel-test-types "${{ inputs.parallel-test-types-list-as-string }}"
-          elif [[ "${{ inputs.test-scope }}" == "All" ]]; then
-            breeze testing tests --run-in-parallel \
-              --parallel-test-types "${{ inputs.parallel-test-types-list-as-string }}"
-          elif [[ "${{ inputs.test-scope }}" == "Quarantined" ]]; then
-            breeze testing tests --test-type "All-Quarantined" || true
-          elif [[ "${{ inputs.test-scope }}" == "ARM collection" ]]; then
-            breeze testing tests --collect-only --remove-arm-packages
-          elif [[ "${{ inputs.test-scope }}" == "System" ]]; then
-            breeze testing tests tests/system/example_empty.py --system core
-          else
-            echo "Unknown test scope: ${{ inputs.test-scope }}"
-            exit 1
-          fi
+          ${{ matrix.test-group}}:${{ inputs.test-scope }} Tests ${{ inputs.test-name }} ${{ matrix.backend-version }}
+          Py${{ matrix.python-version }}:${{ inputs.test-types-list-as-string}}
+        run: ./scripts/ci/testing/run_unit_tests.sh "${{ matrix.test-group }}" "${{ inputs.test-scope }}"
       - name: "Post Tests success"
         uses: ./.github/actions/post_tests_success
         with: