chore: [proposal] de-matrix python-version in GHAs (#27906)

.asf.yaml

@@ -67,15 +67,16 @@ github:
           - cypress-matrix (2, chrome)
           - cypress-matrix (3, chrome)
           - frontend-build
-          - pre-commit (3.10)
-          - python-lint (3.10)
-          - test-mysql (3.10)
-          - test-postgres (3.10)
-          - test-postgres (3.11)
-          - test-postgres-hive (3.10)
-          - test-postgres-presto (3.10)
-          - test-sqlite (3.10)
-          - unit-tests (3.10)
+          - pre-commit
+          - python-lint
+          - test-mysql
+          - test-postgres (current)
+          - test-postgres (next)
+          - test-postgres-hive
+          - test-postgres-presto
+          - test-sqlite
+          - unit-tests (current)
+          - unit-tests (next)
 
       required_pull_request_reviews:
         dismiss_stale_reviews: false

.github/actions/setup-backend/action.yml

@@ -2,9 +2,9 @@ name: 'Setup Python Environment'
 description: 'Set up Python and install dependencies with optional configurations.'
 inputs:
   python-version:
-    description: 'Python version to set up.'
+    description: 'Python version to set up. Accepts a version number, "current", or "next".'
     required: true
-    default: '3.10'
+    default: 'current'
   cache:
     description: 'Cache dependencies. Options: pip'
     required: false
@@ -13,22 +13,39 @@ inputs:
     description: 'Type of requirements to install. Options: base, development, default'
     required: false
     default: 'dev'
+  install-superset:
+    description: 'Whether to install Superset itself. If false, only python is installed'
+    required: false
+    default: 'true'
 
 runs:
   using: 'composite'
   steps:
-    - name: Set up Python ${{ inputs.python-version }}
+    - name: Interpret Python Version
+      id: set-python-version
+      shell: bash
+      run: |
+        if [ "${{ inputs.python-version }}" = "current" ]; then
+          echo "PYTHON_VERSION=3.10" >> $GITHUB_ENV
+        elif [ "${{ inputs.python-version }}" = "next" ]; then
+          echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
+        else
+          echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV
+        fi
+    - name: Set up Python ${{ env.PYTHON_VERSION }}
       uses: actions/setup-python@v5
       with:
-        python-version: ${{ inputs.python-version }}
+        python-version: ${{ env.PYTHON_VERSION }}
         cache: ${{ inputs.cache }}
     - name: Install dependencies
       run: |
-        sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
-        pip install --upgrade pip setuptools wheel
-        if [ "${{ inputs.requirements-type }}" = "dev" ]; then
-          pip install -r requirements/development.txt
-        elif [ "${{ inputs.requirements-type }}" = "base" ]; then
-          pip install -r requirements/base.txt
+        if [ "${{ inputs.install-superset }}" = "true" ]; then
+          sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
+          pip install --upgrade pip setuptools wheel
+          if [ "${{ inputs.requirements-type }}" = "dev" ]; then
+            pip install -r requirements/development.txt
+          elif [ "${{ inputs.requirements-type }}" = "base" ]; then
+            pip install -r requirements/base.txt
+          fi
         fi
       shell: bash

.github/workflows/docker.yml

@@ -73,6 +73,8 @@ jobs:
       - name: Build Docker Image
         if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
         shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # Single platform builds in pull_request context to speed things up
           if [ "${{ github.event_name }}" = "push" ]; then

.github/workflows/no-op.yml

@@ -0,0 +1,96 @@
+# no-op.yml
+#
+# Purpose:
+# This workflow provides a workaround for the "required status checks" feature in GitHub Actions
+# when using path-specific conditions in other workflows. Required checks might remain in a "Pending"
+# state if the conditions are not met, thus blocking pull requests from being merged.
+# This no-op (no operation) workflow provides dummy success statuses for these required jobs when
+# the real jobs do not run due to path-specific conditions.
+#
+# How it works:
+# - It defines jobs with the same names as the required jobs in the main workflows.
+# - These jobs simply execute a command (`exit 0`) to succeed immediately.
+# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered.
+# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses.
+# - If the main workflows' jobs do run and fail, their failure statuses take precedence,
+#   ensuring that pull requests are not merged with failing checks.
+#
+# Usage:
+# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows.
+# - This workflow should be kept as-is, without path-specific conditions.
+
+name: no-op Checks
+on: pull_request
+
+jobs:
+  frontend-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for frontend-build
+        run: exit 0
+
+  pre-commit:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for pre-commit
+        run: exit 0
+
+  python-lint:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for python-lint
+        run: exit 0
+  test-postgres-hive:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for frontend-build
+        run: exit 0
+  test-postgres-presto:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for frontend-build
+        run: exit 0
+  unit-tests:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for frontend-build
+        run: exit 0
+  test-mysql:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for test-mysql
+        run: exit 0
+  test-postgres:
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for test-postgres
+        run: exit 0
+  test-sqlite:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op for test-sqlite
+        run: exit 0

.github/workflows/pre-commit.yml

@@ -16,9 +16,6 @@ concurrency:
 jobs:
   pre-commit:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v4

.github/workflows/superset-cli.yml

@@ -16,9 +16,6 @@ concurrency:
 jobs:
   test-load-examples:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config

.github/workflows/superset-helm-lint.yml

@@ -27,9 +27,10 @@ jobs:
         with:
           version: v3.5.4
 
-      - uses: actions/setup-python@v5
+      - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         with:
-          python-version: "3.10"
+          install-superset: 'false'
 
       - name: Set up chart-testing
         uses: ./.github/actions/chart-testing-action

.github/workflows/superset-python-integrationtest.yml

@@ -16,9 +16,6 @@ concurrency:
 jobs:
   test-mysql:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -51,14 +48,11 @@ jobs:
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
         if: steps.check.outputs.python
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: Setup MySQL
         if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            setup-mysql
+          run: setup-mysql
       - name: Run celery
         if: steps.check.outputs.python
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
@@ -74,7 +68,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["current", "next"]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -130,9 +124,6 @@ jobs:
 
   test-sqlite:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -160,8 +151,6 @@ jobs:
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
         if: steps.check.outputs.python
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies

.github/workflows/superset-python-misc.yml

@@ -17,9 +17,6 @@ concurrency:
 jobs:
   python-lint:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v4
@@ -34,18 +31,13 @@ jobs:
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
         if: steps.check.outputs.python
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: pylint
         if: steps.check.outputs.python
         # `-j 0` run Pylint in parallel
         run: pylint -j 0 superset
 
   babel-extract:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v4
@@ -60,8 +52,6 @@ jobs:
       - name: Setup Python
         if: steps.check.outputs.python
         uses: ./.github/actions/setup-backend/
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: Test babel extraction
         if: steps.check.outputs.python
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

.github/workflows/superset-python-presto-hive.yml

@@ -17,9 +17,6 @@ concurrency:
 jobs:
   test-postgres-presto:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -84,9 +81,6 @@ jobs:
 
   test-postgres-hive:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config

.github/workflows/superset-python-unittest.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["current", "next"]
     env:
       PYTHONPATH: ${{ github.workspace }}
     steps:

.github/workflows/superset-translations.yml

@@ -47,9 +47,6 @@ jobs:
 
   babel-extract:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.10"]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v4
@@ -65,8 +62,6 @@ jobs:
       - name: Setup Python
         if: steps.check.outputs.python
         uses: ./.github/actions/setup-backend/
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: Test babel extraction
         if: steps.check.outputs.python
         run: ./scripts/babel_update.sh