Generate external transform wrappers using a script

.github/workflows/README.md

@@ -271,6 +271,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour
 | [ PreCommit Website ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml) | N/A |`Run Website PreCommit`| [![.github/workflows/beam_PreCommit_Website.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml?query=event%3Aschedule) |
 | [ PreCommit Website Stage GCS ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml) | N/A |`Run Website_Stage_GCS PreCommit`| [![.github/workflows/beam_PreCommit_Website_Stage_GCS.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml?query=event%3Aschedule) |
 | [ PreCommit Whitespace ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml) | N/A |`Run Whitespace PreCommit`| [![.github/workflows/beam_PreCommit_Whitespace.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml?query=event%3Aschedule) |
+| [ PreCommit Xlang Generated Transforms ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml) | N/A |`Run Xlang_Generated_Transforms PreCommit`| [![.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml?query=event%3Aschedule) |
 
 ### PostCommit Jobs
 
@@ -355,6 +356,7 @@ PostCommit Jobs run in a schedule against master branch and generally do not get
 | [ PostCommit Python Xlang Gcp Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml) | N/A |`beam_PostCommit_Python_Xlang_Gcp_Dataflow.json`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml?query=event%3Aschedule) |
 | [ PostCommit Python Xlang Gcp Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml) | N/A |`beam_PostCommit_Python_Xlang_Gcp_Direct.json`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml?query=event%3Aschedule) |
 | [ PostCommit Python Xlang IO Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml) | N/A |`beam_PostCommit_Python_Xlang_IO_Dataflow.json`| [![.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Python Xlang IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml) | N/A |`beam_PostCommit_Python_Xlang_IO_Direct.json`| [![.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml?query=event%3Aschedule) |
 | [ PostCommit Sickbay Python ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml) | ['3.8','3.9','3.10','3.11']  |`beam_PostCommit_Sickbay_Python.json`| [![.github/workflows/beam_PostCommit_Sickbay_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml?query=event%3Aschedule) |
 | [ PostCommit SQL ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml) | N/A |`beam_PostCommit_SQL.json`| [![.github/workflows/beam_PostCommit_SQL.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml?query=event%3Aschedule) |
 | [ PostCommit TransformService Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml) | N/A |`beam_PostCommit_TransformService_Direct.json`| [![.github/workflows/beam_PostCommit_TransformService_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml?query=event%3Aschedule)

.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: PostCommit Python Xlang IO Direct
+
+on:
+  schedule:
+    - cron: '30 5/6 * * *'
+  pull_request_target:
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Xlang_IO_Direct.json']
+  workflow_dispatch:
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: write
+  checks: write
+  contents: read
+  deployments: read
+  id-token: none
+  issues: write
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
+  cancel-in-progress: true
+
+env:
+  GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+  beam_PostCommit_Python_Xlang_IO_Direct:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'pull_request_target' ||
+      (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
+      github.event.comment.body == 'Run Python_Xlang_IO_Direct PostCommit'
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    timeout-minutes: 100
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    strategy:
+      matrix:
+        job_name: ["beam_PostCommit_Python_Xlang_IO_Direct"]
+        job_phrase: ["Run Python_Xlang_IO_Direct PostCommit"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment-action
+        with:
+          python-version: |
+            3.8
+            3.11
+      - name: run PostCommit Python Xlang IO Direct script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:direct:ioCrossLanguagePostCommit
+      - name: Archive Python Test Results
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: Python Test Results
+          path: '**/pytest*.xml'
+      - name: Publish Python Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          commit: '${{ env.prsha || env.GITHUB_SHA }}'
+          comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
+          files: '**/pytest*.xml'

.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml

@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: PreCommit Xlang Generated Transforms
+
+on:
+  push:
+    tags: ['v*']
+    branches: ['master', 'release-*']
+    paths:
+      - 'model/**'
+      - 'sdks/python/**'
+      - 'sdks/java/expansion-service/**'
+      - 'sdks/java/core/**'
+      - 'sdks/java/io/**'
+      - 'sdks/java/extensions/sql/**'
+      - '!sdks/java/io/amazon-web-services/**'
+      - '!sdks/java/io/amazon-web-services2/**'
+      - '!sdks/java/io/amqp/**'
+      - '!sdks/java/io/azure/**'
+      - '!sdks/java/io/cassandra/**'
+      - '!sdks/java/io/cdap/**'
+      - '!sdks/java/io/clickhouse/**'
+      - '!sdks/java/io/debezium/**'
+      - '!sdks/java/io/elasticsearch/**'
+      - '!sdks/java/io/elasticsearch-tests/**'
+      - '!sdks/java/io/hadoop-common/**'
+      - '!sdks/java/io/hadoop-file-system/**'
+      - '!sdks/java/io/hadoop-format/**'
+      - '!sdks/java/io/hbase/**'
+      - '!sdks/java/io/hcatalog/**'
+      - '!sdks/java/io/influxdb/**'
+      - '!sdks/java/io/jms/**'
+      - '!sdks/java/io/kinesis/**'
+      - '!sdks/java/io/kudu/**'
+      - '!sdks/java/io/mqtt/**'
+      - '!sdks/java/io/mongodb/**'
+      - '!sdks/java/io/neo4j/**'
+      - '!sdks/java/io/pulsar/**'
+      - '!sdks/java/io/rabbitmq/**'
+      - '!sdks/java/io/redis/**'
+      - '!sdks/java/io/rrio/**'
+      - '!sdks/java/io/snowflake/**'
+      - '!sdks/java/io/solr/**'
+      - '!sdks/java/io/splunk/**'
+      - '!sdks/java/io/thrift/**'
+      - '!sdks/java/io/tika/**'
+      - 'release/**'
+      - '.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml'
+  pull_request_target:
+    branches: ['master', 'release-*']
+    paths:
+      - 'model/**'
+      - 'sdks/python/**'
+      - 'sdks/java/expansion-service/**'
+      - 'sdks/java/core/**'
+      - 'sdks/java/io/**'
+      - 'sdks/java/extensions/sql/**'
+      - '!sdks/java/io/amazon-web-services/**'
+      - '!sdks/java/io/amazon-web-services2/**'
+      - '!sdks/java/io/amqp/**'
+      - '!sdks/java/io/azure/**'
+      - '!sdks/java/io/cassandra/**'
+      - '!sdks/java/io/cdap/**'
+      - '!sdks/java/io/clickhouse/**'
+      - '!sdks/java/io/debezium/**'
+      - '!sdks/java/io/elasticsearch/**'
+      - '!sdks/java/io/elasticsearch-tests/**'
+      - '!sdks/java/io/hadoop-common/**'
+      - '!sdks/java/io/hadoop-file-system/**'
+      - '!sdks/java/io/hadoop-format/**'
+      - '!sdks/java/io/hbase/**'
+      - '!sdks/java/io/hcatalog/**'
+      - '!sdks/java/io/influxdb/**'
+      - '!sdks/java/io/jms/**'
+      - '!sdks/java/io/kinesis/**'
+      - '!sdks/java/io/kudu/**'
+      - '!sdks/java/io/mqtt/**'
+      - '!sdks/java/io/mongodb/**'
+      - '!sdks/java/io/neo4j/**'
+      - '!sdks/java/io/pulsar/**'
+      - '!sdks/java/io/rabbitmq/**'
+      - '!sdks/java/io/redis/**'
+      - '!sdks/java/io/rrio/**'
+      - '!sdks/java/io/snowflake/**'
+      - '!sdks/java/io/solr/**'
+      - '!sdks/java/io/splunk/**'
+      - '!sdks/java/io/thrift/**'
+      - '!sdks/java/io/tika/**'
+      - 'release/**'
+      - 'release/trigger_all_tests.json'
+      - '.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml'
+  issue_comment:
+    types: [created]
+  schedule:
+    - cron: '30 2/6 * * *'
+  workflow_dispatch:
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: read
+  checks: read
+  contents: read
+  deployments: read
+  id-token: none
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
+  cancel-in-progress: true
+
+env:
+  GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+  beam_PreCommit_Xlang_Generated_Transforms:
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+    timeout-minutes: 120
+    runs-on: ['self-hosted', ubuntu-20.04, main]
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ['beam_PreCommit_Xlang_Generated_Transforms']
+        job_phrase: ['Run Xlang_Generated_Transforms PreCommit']
+        python_version: ['3.8']
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'pull_request_target' ||
+      (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
+      startsWith(github.event.comment.body, 'Run Xlang_Generated_Transforms PreCommit')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment-action
+        with:
+          java-version: 8
+          python-version: ${{ matrix.python_version }}
+      - name: Set PY_VER_CLEAN
+        id: set_py_ver_clean
+        run: |
+          PY_VER=${{ matrix.python_version }}
+          PY_VER_CLEAN=${PY_VER//.}
+          echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT
+      - name: run Portable Python script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:direct:crossLanguageWrapperValidationPreCommit

.gitignore

@@ -52,6 +52,7 @@ sdks/python/**/*.egg
 sdks/python/LICENSE
 sdks/python/NOTICE
 sdks/python/README.md
+sdks/python/apache_beam/transforms/_external_transforms/*
 sdks/python/apache_beam/portability/api/*
 sdks/python/nosetests*.xml
 sdks/python/pytest*.xml

CHANGES.md

@@ -58,6 +58,7 @@
 
 * New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)).
 * New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)).
+* The Python SDK will now include automatically generated wrappers for external Java transforms! ([#29834](https://github.com/apache/beam/pull/29834))
 
 ## I/Os
 

build.gradle.kts

@@ -648,6 +648,23 @@ tasks.register("checkSetup") {
   dependsOn(":examples:java:wordCount")
 }
 
+// Generates external transform config and wrappers.
+// Currently only supports Python wrappers
+project.tasks.register("generateExternalTransformWrappers") {
+  dependsOn(":sdks:python:sdist")
+  // Need to build all expansion services listed in sdks/python/standard_expansion_services.yaml
+  dependsOn(":sdks:java:io:google-cloud-platform:expansion-service:build")
+  dependsOn(":sdks:java:io:expansion-service:build")
+
+  doLast {
+    exec {
+      val pythonDir = project(":sdks:python").projectDir
+      executable("python")
+      args("$pythonDir/gen_xlang_wrappers.py", "--cleanup")
+    }
+  }
+}
+
 // Configure the release plugin to do only local work; the release manager determines what, if
 // anything, to push. On failure, the release manager can reset the branch without pushing.
 release {