diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml index 80226779d4af..6de20d4d86ef 100644 --- a/.github/workflows/beam_PostCommit_Python.yml +++ b/.github/workflows/beam_PostCommit_Python.yml @@ -60,7 +60,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python] job_phrase: [Run Python PostCommit] - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml b/.github/workflows/beam_PostCommit_Python_Arm.yml index cb06e2e90bcd..b817826f332b 100644 --- a/.github/workflows/beam_PostCommit_Python_Arm.yml +++ b/.github/workflows/beam_PostCommit_Python_Arm.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python_Arm] job_phrase: [Run Python PostCommit Arm] - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml index ee3dcdeaf08a..92f760cace4a 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml @@ -74,14 +74,14 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: 3.11 + python-version: 3.12 - name: Run examplesPostCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:dataflow:examplesPostCommit arguments: | -PuseWheelDistribution \ - -PpythonVersion=3.11 \ + -PpythonVersion=3.12 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml index 0a14b423a3bd..e610301a9b68 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Direct"] job_phrase: ["Run Python Examples_Direct"] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml index f6b6a603ff89..9c87a683e677 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Flink"] job_phrase: ["Run Python Examples_Flink"] - python_version: ['3.8', '3.11'] + python_version: ['3.8', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml index 17dd383f5639..b4fa42f78694 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Spark"] job_phrase: ["Run Python Examples_Spark"] - python_version: ['3.8', '3.11'] + python_version: ['3.8', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml index b50b6711712b..efde7ac98a8f 100644 --- a/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml +++ b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml @@ -74,13 +74,13 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: 3.11 + python-version: 3.12 - name: Run mongodbioIT script uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:direct:py311:mongodbioIT + gradle-command: :sdks:python:test-suites:direct:py312:mongodbioIT arguments: | - -PpythonVersion=3.11 \ + -PpythonVersion=3.12 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml index bb3bae69806c..4c31d7bb7a44 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesContainer"] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml index c13df624e7af..bd2701950178 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC"] job_phrase: ["Run Python RC Dataflow ValidatesContainer"] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml index 7c2a46b6c91f..6e7ad5e3f2a6 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesRunner"] - python_version: ['3.8', '3.11'] + python_version: ['3.8', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml index 18c0c8d55c43..529e535f1bdc 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Flink"] job_phrase: ["Run Python Flink ValidatesRunner"] - python_version: ['3.8', '3.11'] + python_version: ['3.8', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml index 54f52992b50d..961b5390ac5b 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Samza"] job_phrase: ["Run Python Samza ValidatesRunner"] - python_version: ['3.8', '3.11'] + python_version: ['3.8', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml index 2407176bc095..546f40ecad1b 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Spark"] job_phrase: ["Run Python Spark ValidatesRunner"] - python_version: ['3.8', '3.9', '3.11'] + python_version: ['3.8', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml index e5a2c1b39134..2ee94f9e7e8e 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -75,7 +75,7 @@ jobs: with: python-version: | 3.8 - 3.11 + 3.12 - name: run PostCommit Python Xlang Gcp Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml index d4e57997da70..db82a5a94b49 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -75,7 +75,7 @@ jobs: with: python-version: | 3.8 - 3.11 + 3.12 - name: run PostCommit Python Xlang Gcp Direct script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml index 66402fad0b8d..3080d8652893 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -75,7 +75,7 @@ jobs: with: python-version: | 3.8 - 3.11 + 3.12 - name: run PostCommit Python Xlang IO Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Sickbay_Python.yml b/.github/workflows/beam_PostCommit_Sickbay_Python.yml index 1a09341adeb1..ad5f90d2bab5 100644 --- a/.github/workflows/beam_PostCommit_Sickbay_Python.yml +++ b/.github/workflows/beam_PostCommit_Sickbay_Python.yml @@ -61,7 +61,7 @@ jobs: job_name: [beam_PostCommit_Sickbay_Python] job_phrase_1: [Run Python] job_phrase_2: [PostCommit Sickbay] - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index 46d883b48681..196a9b76123c 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_TransformService_Direct"] job_phrase: ["Run TransformService_Direct PostCommit"] - python_version: ['3.8','3.11'] + python_version: ['3.8','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -77,7 +77,7 @@ jobs: java-version: 11 python-version: | 3.8 - 3.11 + ${{ matrix.python_version }} - name: run TransformService Direct script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml index de1b995bc649..a840fcc34f02 100644 --- a/.github/workflows/beam_PostCommit_XVR_Direct.yml +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Direct"] job_phrase: ["Run XVR_Direct PostCommit"] - python_version: ['3.8','3.11'] + python_version: ['3.8','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,7 +76,7 @@ jobs: with: python-version: | 3.8 - 3.11 + ${{ matrix.python_version }} - name: run PostCommit XVR Direct script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml index 134efdf0dfc3..9b4a48bb90a0 100644 --- a/.github/workflows/beam_PostCommit_XVR_Flink.yml +++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Flink"] job_phrase: ["Run XVR_Flink PostCommit"] - python_version: ['3.8','3.11'] + python_version: ['3.8','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -77,7 +77,7 @@ jobs: with: python-version: | 3.8 - 3.11 + ${{ matrix.python_version }} - name: run PostCommit XVR Flink script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml index ff2edee6f562..5a4338a615ea 100644 --- a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_JavaUsingPython_Dataflow"] job_phrase: ["Run XVR_JavaUsingPython_Dataflow PostCommit"] - python_version: ['3.8','3.11'] + python_version: ['3.8','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,7 +76,7 @@ jobs: with: python-version: | 3.8 - 3.11 + ${{ matrix.python_version }} - name: run PostCommit XVR JavaUsingPython Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml index 5cf43dbeea40..7545624c3fe5 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml @@ -73,13 +73,13 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - python-version: 3.11 + python-version: 3.12 - name: run PostCommit XVR PythonUsingJavaSQL Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingSql arguments: | - -PpythonVersion=3.11 \ + -PpythonVersion=3.12 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml index da7d88213e69..1d053aa479d5 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_PythonUsingJava_Dataflow"] job_phrase: ["Run XVR_PythonUsingJava_Dataflow PostCommit"] - python_version: ['3.8','3.11'] + python_version: ['3.8','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,7 +76,7 @@ jobs: with: python-version: | 3.8 - 3.11 + ${{ matrix.python_version }} - name: run PostCommit XVR PythonUsingJava Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml b/.github/workflows/beam_PostCommit_XVR_Samza.yml index b56872dd778b..37f7a7281719 100644 --- a/.github/workflows/beam_PostCommit_XVR_Samza.yml +++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Samza"] job_phrase: ["Run XVR_Samza PostCommit"] - python_version: ['3.8','3.11'] + python_version: ['3.8','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,7 +76,7 @@ jobs: with: python-version: | 3.8 - 3.11 + ${{ matrix.python_version }} - name: run PostCommit XVR Samza script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml index 68f9cb93b5f6..da38dca507a9 100644 --- a/.github/workflows/beam_PostCommit_XVR_Spark3.yml +++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Spark3"] job_phrase: ["Run XVR_Spark3 PostCommit"] - python_version: ['3.8','3.11'] + python_version: ['3.8','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,7 +76,7 @@ jobs: with: python-version: | 3.8 - 3.11 + ${{ matrix.python_version }} - name: run PostCommit XVR Spark3 script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} diff --git a/.github/workflows/beam_PreCommit_Portable_Python.yml b/.github/workflows/beam_PreCommit_Portable_Python.yml index 1c7bdf157f7a..09ffed626422 100644 --- a/.github/workflows/beam_PreCommit_Portable_Python.yml +++ b/.github/workflows/beam_PreCommit_Portable_Python.yml @@ -86,7 +86,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Portable_Python'] job_phrase: ['Run Portable_Python PreCommit'] - python_version: ['3.8', '3.11'] + python_version: ['3.8', '3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml index c96e7cf0bd3c..cc539b4c2f16 100644 --- a/.github/workflows/beam_PreCommit_Python.yml +++ b/.github/workflows/beam_PreCommit_Python.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python'] job_phrase: ['Run Python PreCommit'] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_PythonDocker.yml b/.github/workflows/beam_PreCommit_PythonDocker.yml index d708d58924da..5fcef7216944 100644 --- a/.github/workflows/beam_PreCommit_PythonDocker.yml +++ b/.github/workflows/beam_PreCommit_PythonDocker.yml @@ -64,9 +64,9 @@ jobs: matrix: job_name: ["beam_PreCommit_PythonDocker"] job_phrase: ["Run PythonDocker PreCommit"] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | - github.event_name == 'push' || + github.event_name == 'push' || github.event_name == 'pull_request_target' || github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml index b41da9ec3185..c5600f50b574 100644 --- a/.github/workflows/beam_PreCommit_Python_Dataframes.yml +++ b/.github/workflows/beam_PreCommit_Python_Dataframes.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Dataframes'] job_phrase: ['Run Python_Dataframes PreCommit'] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml index 2b100f05d176..62d8337ff8f1 100644 --- a/.github/workflows/beam_PreCommit_Python_Examples.yml +++ b/.github/workflows/beam_PreCommit_Python_Examples.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Examples'] job_phrase: ['Run Python_Examples PreCommit'] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Integration.yml b/.github/workflows/beam_PreCommit_Python_Integration.yml index e2531f5eea2e..a92201d83b79 100644 --- a/.github/workflows/beam_PreCommit_Python_Integration.yml +++ b/.github/workflows/beam_PreCommit_Python_Integration.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Integration'] job_phrase: ['Run Python_Integration PreCommit'] - python_version: ['3.8', '3.11'] + python_version: ['3.8', '3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml index 779f222366a2..1dd50ccd22a3 100644 --- a/.github/workflows/beam_PreCommit_Python_ML.yml +++ b/.github/workflows/beam_PreCommit_Python_ML.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_ML'] job_phrase: ['Run Python_ML PreCommit'] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml index 64609737cb25..95e59e7b4a5f 100644 --- a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml +++ b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml @@ -87,8 +87,8 @@ jobs: job_phrase: ["Run Python_PVR_Flink PreCommit"] timeout-minutes: 120 if: | - github.event_name == 'push' || - github.event_name == 'pull_request_target' || + github.event_name == 'push' || + github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || github.event_name == 'workflow_dispatch' || github.event.comment.body == 'Run Python_PVR_Flink PreCommit' @@ -104,15 +104,15 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - python-version: 3.11 + python-version: 3.12 - name: run Python PVR Flink PreCommit script uses: ./.github/actions/gradle-command-self-hosted-action env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} with: - gradle-command: :sdks:python:test-suites:portable:py311:flinkValidatesRunner + gradle-command: :sdks:python:test-suites:portable:py312:flinkValidatesRunner arguments: | - -PpythonVersion=3.11 \ + -PpythonVersion=3.12 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml index 080c191443b9..eaf565826d5f 100644 --- a/.github/workflows/beam_PreCommit_Python_Runners.yml +++ b/.github/workflows/beam_PreCommit_Python_Runners.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Runners'] job_phrase: ['Run Python_Runners PreCommit'] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml index 20d31304150f..2528aee525d7 100644 --- a/.github/workflows/beam_PreCommit_Python_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Python_Transforms.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Transforms'] job_phrase: ['Run Python_Transforms PreCommit'] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml index 8871af45b50e..4a232c4511f7 100644 --- a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml +++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml @@ -54,7 +54,7 @@ jobs: matrix: job_name: [beam_Python_ValidatesContainer_Dataflow_ARM] job_phrase: [Run Python ValidatesContainer Dataflow ARM] - python_version: ['3.8','3.9','3.10','3.11'] + python_version: ['3.8','3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || @@ -99,7 +99,7 @@ jobs: id: run_script uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:validatesContainerARM + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:validatesContainerARM arguments: | -PpythonVersion=${{ matrix.python_version }} \ -Pcontainer-architecture-list=arm64,amd64 \ diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 7b6ac3355b80..5142b0b22c30 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest env: EVENT_NAME: ${{ github.event_name }} - PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-*" + PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-* cp312-*" outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }} diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml index 1c4012029d56..fed1056b90b2 100644 --- a/.github/workflows/python_dependency_tests.yml +++ b/.github/workflows/python_dependency_tests.yml @@ -30,6 +30,7 @@ jobs: {"py_ver": "3.9", "py_env": "py39"}, {"py_ver": "3.10", "py_env": "py310" }, { "py_ver": "3.11", "py_env": "py311" }, + { "py_ver": "3.12", "py_env": "py312" }, ] steps: - name: Checkout code diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index c7840797e856..efc5af84c4bc 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -102,6 +102,7 @@ jobs: {"py_ver": "3.9", "tox_env": "py39"}, {"py_ver": "3.10", "tox_env": "py310" }, { "py_ver": "3.11", "tox_env": "py311" }, + { "py_ver": "3.12", "tox_env": "py312" }, ] steps: - name: Checkout code @@ -134,7 +135,7 @@ jobs: fail-fast: false matrix: os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest] - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Checkout code uses: actions/checkout@v4 @@ -159,7 +160,7 @@ jobs: fail-fast: false matrix: os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest] - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/update_python_dependencies.yml b/.github/workflows/update_python_dependencies.yml index b2827745148a..80cb120a78bf 100644 --- a/.github/workflows/update_python_dependencies.yml +++ b/.github/workflows/update_python_dependencies.yml @@ -16,7 +16,7 @@ # under the License. # This workflow will update python dependencies as part of the release process -# And commit the changes into the branch release, creating a PR into the branch +# And commit the changes into the branch release, creating a PR into the branch name: Update Python Depedencies @@ -43,7 +43,7 @@ jobs: properties: ${{ steps.test-properties.outputs.properties }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4 - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -62,6 +62,7 @@ jobs: 3.9 3.10 3.11 + 3.12 java-version: default go-version: default - name: Update Python Dependencies @@ -88,8 +89,8 @@ jobs: fi branchName=weekly_update_python_dependencies_$(date +%s) git checkout -b $branchName - git add -A - git diff-index --quiet HEAD || gitdiff=$? || echo $? + git add -A + git diff-index --quiet HEAD || gitdiff=$? || echo $? if [[ $gitDiff != 0 ]]; then echo "Changes are ready to commit" git commit -m "Update Python Dependencies" --quiet diff --git a/CHANGES.md b/CHANGES.md index 6947c0d5e7e5..a81ff3c28121 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -57,9 +57,8 @@ ## Highlights -* Added FlinkRunner for Flink 1.18. ([#30789](https://github.com/apache/beam/issues/30789)) -* New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)). -* New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)). +* Apache Beam adds Python 3.12 support ([#29149](https://github.com/apache/beam/issues/29149)). +* Added FlinkRunner for Flink 1.18 ([#30789](https://github.com/apache/beam/issues/30789)). ## I/Os diff --git a/build.gradle.kts b/build.gradle.kts index d443aa37b413..d1efb45b74ee 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -473,6 +473,7 @@ tasks.register("pythonPreCommit") { dependsOn(":sdks:python:test-suites:tox:py39:preCommitPy39") dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310") dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311") + dependsOn(":sdks:python:test-suites:tox:py312:preCommitPy312") } tasks.register("pythonPreCommitIT") { @@ -489,6 +490,7 @@ tasks.register("pythonDockerBuildPreCommit") { dependsOn(":sdks:python:container:py39:docker") dependsOn(":sdks:python:container:py310:docker") dependsOn(":sdks:python:container:py311:docker") + dependsOn(":sdks:python:container:py312:docker") } tasks.register("pythonLintPreCommit") { @@ -535,15 +537,22 @@ tasks.register("python311PostCommit") { dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311") } +tasks.register("python312PostCommit") { + dependsOn(":sdks:python:test-suites:dataflow:py312:postCommitIT") + dependsOn(":sdks:python:test-suites:direct:py312:postCommitIT") + dependsOn(":sdks:python:test-suites:direct:py312:hdfsIntegrationTest") + dependsOn(":sdks:python:test-suites:portable:py312:postCommitPy312") +} + tasks.register("portablePythonPreCommit") { dependsOn(":sdks:python:test-suites:portable:py38:preCommitPy38") - dependsOn(":sdks:python:test-suites:portable:py311:preCommitPy311") + dependsOn(":sdks:python:test-suites:portable:py312:preCommitPy312") } tasks.register("pythonSparkPostCommit") { dependsOn(":sdks:python:test-suites:portable:py38:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py39:sparkValidatesRunner") - dependsOn(":sdks:python:test-suites:portable:py311:sparkValidatesRunner") + dependsOn(":sdks:python:test-suites:portable:py312:sparkValidatesRunner") } tasks.register("websitePreCommit") { diff --git a/gradle.properties b/gradle.properties index 8c41a9f86c8f..f3dea792ccc1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -41,4 +41,4 @@ docker_image_default_repo_prefix=beam_ # supported flink versions flink_versions=1.14,1.15,1.16,1.17,1.18 # supported python versions -python_versions=3.8,3.9,3.10,3.11 +python_versions=3.8,3.9,3.10,3.11,3.12 diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index 85b518718528..27c2b293fbd0 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -70,7 +70,7 @@ import warnings if sys.version_info.major == 3: - if sys.version_info.minor <= 6 or sys.version_info.minor >= 12: + if sys.version_info.minor <= 7 or sys.version_info.minor >= 13: warnings.warn( 'This version of Apache Beam has not been sufficiently tested on ' 'Python %s.%s. You may encounter bugs or missing features.' % diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 1e68676c457f..e0be0c33720d 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -82,7 +82,7 @@ _LOGGER = logging.getLogger(__name__) -_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.8', '3.9', '3.10', '3.11'] +_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.8', '3.9', '3.10', '3.11', '3.12'] class Environment(object): diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index d639ad21c31c..a4e1a5253a73 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -972,7 +972,7 @@ def test_interpreter_version_check_passes_py38(self): @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info', - (3, 12, 0)) + (3, 13, 0)) @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 1c80beea23e8..4a7b57db917b 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -421,6 +421,7 @@ def get_portability_package_data(): 'pytest-xdist>=2.5.0,<4', 'pytest-timeout>=2.1.0,<3', 'scikit-learn>=0.20.0', + 'setuptools', 'sqlalchemy>=1.3,<3.0', 'psycopg2-binary>=2.8.5,<3.0.0', 'testcontainers[mysql]>=3.0.3,<4.0.0', @@ -525,6 +526,7 @@ def get_portability_package_data(): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', # When updating version classifiers, also update version warnings # above and in apache_beam/__init__.py. 'Topic :: Software Development :: Libraries', diff --git a/sdks/python/test-suites/dataflow/py312/build.gradle b/sdks/python/test-suites/dataflow/py312/build.gradle new file mode 100644 index 000000000000..ea2bacc02018 --- /dev/null +++ b/sdks/python/test-suites/dataflow/py312/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: org.apache.beam.gradle.BeamModulePlugin +applyPythonNature() + +// Required to setup a Python 3 virtualenv and task names. +pythonVersion = '3.12' +apply from: "../common.gradle" diff --git a/sdks/python/test-suites/direct/py312/build.gradle b/sdks/python/test-suites/direct/py312/build.gradle new file mode 100644 index 000000000000..e913001fd317 --- /dev/null +++ b/sdks/python/test-suites/direct/py312/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() + +// Required to setup a Python 3 virtualenv and task names. +pythonVersion = '3.12' +apply from: '../common.gradle' diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties index c4e94bed4d84..3d16ee63f12a 100644 --- a/sdks/python/test-suites/gradle.properties +++ b/sdks/python/test-suites/gradle.properties @@ -23,29 +23,29 @@ # dataflow test-suites # (TODO): https://github.com/apache/beam/issues/21971 # Add python 3.10 to dataflow test-suites -dataflow_precommit_it_task_py_versions=3.8,3.11 +dataflow_precommit_it_task_py_versions=3.8,3.12 dataflow_mongodbio_it_task_py_versions=3.8 dataflow_chicago_taxi_example_task_py_versions=3.8 # TODO: Enable following tests after making sure we have enough capacity. -dataflow_validates_runner_batch_tests=3.8,3.11 -dataflow_validates_runner_streaming_tests=3.8,3.11 -dataflow_examples_postcommit_py_versions=3.11 +dataflow_validates_runner_batch_tests=3.8,3.12 +dataflow_validates_runner_streaming_tests=3.8,3.12 +dataflow_examples_postcommit_py_versions=3.12 # TFX_BSL is not yet supported on Python 3.10. dataflow_cloudml_benchmark_tests_py_versions=3.9 # direct runner test-suites -direct_mongodbio_it_task_py_versions=3.11 +direct_mongodbio_it_task_py_versions=3.12 # flink runner test-suites -flink_validates_runner_precommit_py_versions=3.11 -flink_validates_runner_postcommit_py_versions=3.8,3.11 -flink_examples_postcommit_py_versions=3.8,3.11 +flink_validates_runner_precommit_py_versions=3.12 +flink_validates_runner_postcommit_py_versions=3.8,3.12 +flink_examples_postcommit_py_versions=3.8,3.12 # samza runner test-suites -samza_validates_runner_postcommit_py_versions=3.8,3.11 +samza_validates_runner_postcommit_py_versions=3.8,3.12 # spark runner test-suites -spark_examples_postcommit_py_versions=3.8,3.11 +spark_examples_postcommit_py_versions=3.8,3.12 # cross language postcommit python test suites -cross_language_validates_py_versions=3.8,3.11 +cross_language_validates_py_versions=3.8,3.12 diff --git a/sdks/python/test-suites/portable/py312/build.gradle b/sdks/python/test-suites/portable/py312/build.gradle new file mode 100644 index 000000000000..689713525494 --- /dev/null +++ b/sdks/python/test-suites/portable/py312/build.gradle @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: org.apache.beam.gradle.BeamModulePlugin +applyPythonNature() + +addPortableWordCountTasks() + +// Required to setup a Python 3 virtualenv and task names. +pythonVersion = '3.12' +apply from: "../common.gradle" diff --git a/sdks/python/test-suites/tox/py312/build.gradle b/sdks/python/test-suites/tox/py312/build.gradle new file mode 100644 index 000000000000..17478ec43aa8 --- /dev/null +++ b/sdks/python/test-suites/tox/py312/build.gradle @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for Python 3.12 + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() + +// Required to setup a Python 3 virtualenv and task names. +pythonVersion = '3.12' + +apply from: "../common.gradle" + diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 63bcce8adf3c..aa0200f75005 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,7 +17,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. -envlist = py38,py39,py310,py311,py38-{cloud,cloudcoverage,dask},py39-{cloud},py310-{cloud,dask},py311-{cloud,dask},docs,lint,mypy,whitespacelint +envlist = py38,py39,py310,py311,py312,py38-{cloud,cloudcoverage,dask},py39-{cloud},py310-{cloud,dask},py311-{cloud,dask},py312-{cloud,dask},docs,lint,mypy,whitespacelint toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} [pycodestyle] @@ -67,33 +67,33 @@ commands_post = commands = false {envname} is misconfigured -[testenv:py{38,39,310,311}] +[testenv:py{38,39,310,311,312}] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{38,39,310,311}-win] +[testenv:py{38,39,310,311,312}-win] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages} list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze -[testenv:py{38,39,310,311}-cloud] +[testenv:py{38,39,310,311,312}-cloud] ; extras = test,gcp,interactive,dataframe,aws,azure extras = test,gcp,interactive,dataframe,aws,azure commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{38,39,310,311}-ml] +[testenv:py{38,39,310,311,312}-ml] # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. setenv = extras = test,gcp,dataframe,ml_test commands = bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{38,39,310,311}-dask] +[testenv:py{38,39,310,311,312}-dask] extras = test,dask commands = bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" @@ -296,7 +296,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39,310,311}-pandas-{14,15,20}] +[testenv:py{38,39}-pandas-{14,15,20}] deps = 14: pandas>=1.4.3,<1.5.0 # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725 @@ -317,7 +317,7 @@ deps = commands = bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py' -[testenv:py{38,39,310,311}-pytorch-{19,110,111,112,113}] +[testenv:py{38,39}-pytorch-{19,110,111,112,113}] deps = 19: torch>=1.9.0,<1.10.0 110: torch>=1.10.0,<1.11.0 @@ -334,7 +334,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39,310}-pytorch-200] +[testenv:py{38,39}-pytorch-200] deps = 200: torch>=2.0.0,<2.1.0 @@ -350,7 +350,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' # TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py38/build.gradle once onnx supports protobuf 4.x.x -[testenv:py{38,39,310}-onnx-113] +[testenv:py{38,39}-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) # apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...} # when multiple onnx versions are tested. @@ -369,7 +369,7 @@ commands = # Run all ONNX unit tests pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_onnx {posargs} -[testenv:py{38,39,310}-tensorflow-212] +[testenv:py{38,39}-tensorflow-212] deps = 212: tensorflow>=2.12rc1,<2.13 # Help pip resolve conflict with typing-extensions for old version of TF https://github.com/apache/beam/issues/30852 @@ -382,7 +382,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39,310}-xgboost-{160,170}] +[testenv:py{38,39}-xgboost-{160,170}] deps = 160: xgboost>=1.6.0,<1.7.0 @@ -398,7 +398,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_xgboost {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39,310,311}-transformers-{428,429,430}] +[testenv:py{38,39}-transformers-{428,429,430}] deps = 428: transformers>=4.28.0,<4.29.0 429: transformers>=4.29.0,<4.30.0 @@ -428,7 +428,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39,310,311}-embeddings] +[testenv:py{38,39}-embeddings] deps = sentence-transformers==2.2.2 passenv = HF_INFERENCE_TOKEN @@ -441,7 +441,7 @@ commands = /bin/sh -c 'pytest apache_beam/ml/transforms/embeddings -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39,310,311}-TFHubEmbeddings-{014,015}] +[testenv:py{38,39}-TFHubEmbeddings-{014,015}] deps = 014: tensorflow-hub>=0.14.0,<0.15.0 # Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852 diff --git a/settings.gradle.kts b/settings.gradle.kts index c6c0140c2b56..3f0e5d631b69 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -305,21 +305,25 @@ include(":sdks:python:test-suites:dataflow:py38") include(":sdks:python:test-suites:dataflow:py39") include(":sdks:python:test-suites:dataflow:py310") include(":sdks:python:test-suites:dataflow:py311") +include(":sdks:python:test-suites:dataflow:py312") include(":sdks:python:test-suites:direct") include(":sdks:python:test-suites:direct:py38") include(":sdks:python:test-suites:direct:py39") include(":sdks:python:test-suites:direct:py310") include(":sdks:python:test-suites:direct:py311") +include(":sdks:python:test-suites:direct:py312") include(":sdks:python:test-suites:direct:xlang") include(":sdks:python:test-suites:portable:py38") include(":sdks:python:test-suites:portable:py39") include(":sdks:python:test-suites:portable:py310") include(":sdks:python:test-suites:portable:py311") +include(":sdks:python:test-suites:portable:py312") include(":sdks:python:test-suites:tox:pycommon") include(":sdks:python:test-suites:tox:py38") include(":sdks:python:test-suites:tox:py39") include(":sdks:python:test-suites:tox:py310") include(":sdks:python:test-suites:tox:py311") +include(":sdks:python:test-suites:tox:py312") include(":sdks:python:test-suites:xlang") include(":sdks:typescript") include(":sdks:typescript:container")