Merge branch 'master' into error-handler

apache · Aug 20, 2024 · 049e4b3 · 049e4b3
2 parents 5141f14 + 71e3eed
commit 049e4b3
Show file tree

Hide file tree

Showing 694 changed files with 25,593 additions and 7,680 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.58.1: {}
     release-2.58.0: {}
     release-2.57.0: {}
     release-2.56.0: {}

diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -50,6 +50,7 @@ body:
         - "Priority: 2 (default / most bugs should be filed as P2)"
         - "Priority: 1 (data loss / total loss of function)"
         - "Priority: 0 (outage / urgent vulnerability)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -68,6 +69,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

diff --git a/.github/ISSUE_TEMPLATE/failing_test.yml b/.github/ISSUE_TEMPLATE/failing_test.yml
@@ -56,6 +56,7 @@ body:
         - "Priority: 2 (backlog / disabled test but we think the product is healthy)"
         - "Priority: 1 (unhealthy code / failing or flaky postcommit so we cannot be sure the product is healthy)"
         - "Priority: 0 (outage / failing precommit test impacting development)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -74,6 +75,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml
@@ -44,6 +44,7 @@ body:
       options:
         - "Priority: 3 (nice-to-have improvement)"
         - "Priority: 2 (default / most feature requests should be filed as P2)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -62,6 +63,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

diff --git a/.github/ISSUE_TEMPLATE/task.yml b/.github/ISSUE_TEMPLATE/task.yml
@@ -45,6 +45,7 @@ body:
         - "Priority: 3 (nice-to-have improvement)"
         - "Priority: 2 (default / most normal work should be filed as P2)"
         - "Priority: 1 (urgent / mostly reserved for critical bugs)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -63,6 +64,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml
@@ -48,7 +48,7 @@ runs:
       uses: actions/setup-java@v3
       with:
         distribution: 'temurin'
-        java-version: ${{ inputs.java-version == 'default' && '8' || inputs.java-version }}
+        java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }}
     - name: Setup Gradle
       uses: gradle/gradle-build-action@v2
       with:

diff --git a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env
@@ -35,7 +35,7 @@ service_account_id = "beam-github-actions@apache-beam-testing.iam.gserviceaccoun
 runner_group = "beam"
 main_runner = {
     name = "main-runner"
-    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:21129a41e031c150c3f610639d71a95a3a941243"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:3063b55757509dad1c14751c9f2aa5905826d9a0"
     machine_type = "e2-standard-16"
     min_node_count = "1"
     max_node_count = "30"
@@ -51,7 +51,7 @@ main_runner = {
 additional_runner_pools = [{
     name = "small-runner"
     machine_type = "e2-standard-2"
-    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:21129a41e031c150c3f610639d71a95a3a941243"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:3063b55757509dad1c14751c9f2aa5905826d9a0"
     min_node_count = "1"
     max_node_count = "15"
     min_replicas = "1"
@@ -68,7 +68,7 @@ additional_runner_pools = [{
 {
     name = "highmem-runner"
     machine_type = "c3-highmem-8"
-    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:21129a41e031c150c3f610639d71a95a3a941243"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:3063b55757509dad1c14751c9f2aa5905826d9a0"
     min_node_count = "1"
     max_node_count = "15"
     min_replicas = "1"

diff --git a/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile
@@ -37,16 +37,16 @@ RUN curl -OL https://go.dev/dl/go${go_version}.linux-amd64.tar.gz && \
     rm go${go_version}.linux-amd64.tar.gz
 ENV PATH="${PATH}:/usr/local/go/bin"
 #Install Java
-RUN curl -OL https://cdn.azul.com/zulu/bin/zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \
-    tar -C /usr/local -xzf zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \
-    rm zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \
-    mv /usr/local/zulu8.70.0.23-ca-jdk8.0.372-linux_x64 /usr/local/java
+RUN curl -OL https://cdn.azul.com/zulu/bin/zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \
+    tar -C /usr/local -xzf zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \
+    rm zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \
+    mv /usr/local/zulu11.74.15-ca-jdk11.0.24-linux_x64 /usr/local/java
 ENV PATH="${PATH}:/usr/local/java/bin"
 #Install Gradle
-RUN curl -OL https://services.gradle.org/distributions/gradle-7.3.3-bin.zip && \
-    unzip gradle-7.3.3-bin.zip && \
-    rm gradle-7.3.3-bin.zip && \
-    mv gradle-7.3.3 /usr/local/gradle
+RUN curl -OL https://services.gradle.org/distributions/gradle-8.4-bin.zip && \
+    unzip gradle-8.4-bin.zip && \
+    rm gradle-8.4-bin.zip && \
+    mv gradle-8.4 /usr/local/gradle
 ENV PATH="${PATH}:/usr/local/gradle/bin"
 #Install GitHub CLI
 RUN curl -OL https://github.com/cli/cli/releases/download/v2.31.0/gh_2.31.0_linux_amd64.tar.gz && \

diff --git a/.github/issue-rules.yml b/.github/issue-rules.yml
@@ -46,6 +46,8 @@ rules:
   addLabels: ['katas']
 - contains: '[x] Component: Website'
   addLabels: ['website']
+- contains: '[x] Component: Infrastructure'
+  addLabels: ['infra']
 - contains: '[x] Component: Spark'
   addLabels: ['spark']
 - contains: '[x] Component: Flink'

diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run",
-    "modification": 1
+    "modification": 5
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_Java.json b/.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_Java.json
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Direct.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Direct.json
@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "https://github.com/apache/beam/pull/31761": "noting that PR #31761 should run this test"
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
@@ -1,4 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
+  "https://github.com/apache/beam/pull/31798": "noting that PR #31798 should run this test"
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
@@ -1,4 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
+  "https://github.com/apache/beam/pull/31798": "noting that PR #31798 should run this test"
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json
@@ -1,4 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
+  "https://github.com/apache/beam/pull/31798": "noting that PR #31798 should run this test"
 }
diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
@@ -1,4 +1,5 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run."
+  "comment": "Modify this file in a trivial way to cause this test suite to run.",
+  "modification": 1
 }
 
diff --git a/.github/workflows/IO_Iceberg_Integration_Tests.yml b/.github/workflows/IO_Iceberg_Integration_Tests.yml
@@ -72,13 +72,7 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
-      - name: Authenticate on GCP
-        id: auth
-        uses: google-github-actions/auth@v1
-        with:
-          credentials_json: ${{ secrets.GCP_SA_KEY }}
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
       - name: Run IcebergIO Integration Test
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sdks:java:io:iceberg:integrationTest
+          gradle-command: :sdks:java:io:iceberg:catalogTests
diff --git a/.github/workflows/IO_Iceberg_Performance_Tests.yml b/.github/workflows/IO_Iceberg_Performance_Tests.yml
@@ -72,12 +72,6 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
-      - name: Authenticate on GCP
-        id: auth
-        uses: google-github-actions/auth@v1
-        with:
-          credentials_json: ${{ secrets.GCP_SA_KEY }}
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
       - name: Run IcebergIO Performance Test
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:

diff --git a/.github/workflows/IO_Iceberg_Unit_Tests.yml b/.github/workflows/IO_Iceberg_Unit_Tests.yml
@@ -91,12 +91,6 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
-      - name: Authenticate on GCP
-        id: auth
-        uses: google-github-actions/auth@v1
-        with:
-          credentials_json: ${{ secrets.GCP_SA_KEY }}
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
       - name: run IcebergIO build script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with: