Merge branch 'master' into update-type-hints

.asf.yaml

@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.61.0: {}
     release-2.60.0: {}
     release-2.59.0: {}
     release-2.58.1: {}

.github/REVIEWERS.yml

@@ -61,6 +61,12 @@ labels:
     reviewers:
       - svetakvsundhar
     exclusionList: []
+  - name: kafka
+    reviewers:
+      - johnjcasey
+      - fozzie15
+      - Dippatel98
+      - sjvanrossum
   - name: Build
     reviewers:
       - damccorm

.github/trigger_files/IO_Iceberg_Integration_Tests.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run",
-    "modification": 4
+    "modification": 3
 }

.github/trigger_files/beam_PostCommit_Java.json

@@ -0,0 +1 @@
+{}

.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json

@@ -0,0 +1,4 @@
+{
+    "comment": "Modify this file in a trivial way to cause this test suite to run",
+    "modification": 1
+}

.github/trigger_files/beam_PostCommit_Java_DataflowV2.json

@@ -1,3 +1,4 @@
 {
-    "comment": "Modify this file in a trivial way to cause this test suite to run"
+    "comment": "Modify this file in a trivial way to cause this test suite to run",
+    "modification": 1
 }

.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "modification": 1
 }

.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Batch.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 2
 }

.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Docker.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "modification": "1"
 }

.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 3
 }

.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 5
 }

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json

@@ -1,5 +1,5 @@
 
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 2
 }

.github/trigger_files/beam_PostCommit_TransformService_Direct.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "revision: "2"
 }

.github/trigger_files/beam_PostCommit_XVR_Direct.json

@@ -0,0 +1,3 @@
+{
+  "https://github.com/apache/beam/pull/32648": "testing Flink 1.19 support"
+}

.github/trigger_files/beam_PostCommit_XVR_Flink.json

@@ -1,3 +1,3 @@
 {
-  "https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
+  "https://github.com/apache/beam/pull/32648": "testing Flink 1.19 support"
 }

.github/workflows/README.md

@@ -285,6 +285,7 @@ Additional PreCommit jobs running basic SDK unit test on a matrices of operating
 | [Java Tests](https://github.com/apache/beam/actions/workflows/java_tests.yml) | [![.github/workflows/java_tests.yml](https://github.com/apache/beam/actions/workflows/java_tests.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/java_tests.yml?query=event%3Aschedule) |
 | [Python Tests](https://github.com/apache/beam/actions/workflows/python_tests.yml) | [![.github/workflows/python_tests.yml](https://github.com/apache/beam/actions/workflows/python_tests.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/python_tests.yml?query=event%3Aschedule) |
 | [TypeScript Tests](https://github.com/apache/beam/actions/workflows/typescript_tests.yml) | [![.github/workflows/typescript_tests.yml](https://github.com/apache/beam/actions/workflows/typescript_tests.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/typescript_tests.yml?query=event%3Aschedule) |
+| [Build Wheels](https://github.com/apache/beam/actions/workflows/build_wheels.yml) | [![.github/workflows/build_wheels.yml](https://github.com/apache/beam/actions/workflows/build_wheels.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/build_wheels.yml?query=event%3Aschedule) |
 
 ### PostCommit Jobs
 

.github/workflows/playground_backend_precommit.yml → .github/workflows/beam_Playground_Precommit.yml

@@ -17,28 +17,43 @@ name: Playground PreCommit
 
 on:
   workflow_dispatch:
-  pull_request:
+  pull_request_target:
     paths:
       - .github/workflows/playground_backend_precommit.yml
       - playground/backend/**
+  issue_comment:
+    types: [created]
 
 env:
   DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
   GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
 
 jobs:
-  precommit_check:
-    name: precommit-check
-    runs-on: ubuntu-latest
+  beam_Playground_PreCommit:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'pull_request_target' ||
+      github.event.comment.body == 'Run Playground PreCommit'
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: [beam_Playground_PreCommit]
+        job_phrase: [Run Playground PreCommit]
     env:
       DATASTORE_EMULATOR_VERSION: '423.0.0'
       PYTHON_VERSION: '3.9'
       JAVA_VERSION: '11'
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
@@ -58,7 +73,7 @@ jobs:
           sudo chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
           sudo apt-get update --yes
           sudo apt-get install sbt --yes
-          sudo wget https://codeload.github.com/spotify/scio.g8/zip/7c1ba7c1651dfd70976028842e721da4107c0d6d -O scio.g8.zip && unzip scio.g8.zip && mv scio.g8-7c1ba7c1651dfd70976028842e721da4107c0d6d /opt/scio.g8
+          sudo wget https://codeload.github.com/spotify/scio.g8/zip/7c1ba7c1651dfd70976028842e721da4107c0d6d -O scio.g8.zip && unzip scio.g8.zip && sudo mv scio.g8-7c1ba7c1651dfd70976028842e721da4107c0d6d /opt/scio.g8
       - name: Set up Cloud SDK and its components
         uses: google-github-actions/setup-gcloud@v2
         with:

.github/workflows/beam_PostRelease_NightlySnapshot.yml

@@ -59,10 +59,17 @@ jobs:
         uses: ./.github/actions/setup-environment-action
         with:
           java-version: default
+      - name: Setup temp local maven
+        id: setup_local_maven
+        run: echo "NEW_TEMP_DIR=$(mktemp -d)" >> $GITHUB_OUTPUT
       - name: run PostRelease validation script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :release:runJavaExamplesValidationTask
           arguments: |
             -Pver='${{ github.event.inputs.RELEASE }}' \
             -Prepourl='${{ github.event.inputs.SNAPSHOT_URL }}' \
+            -PmavenLocalPath='${{ steps.setup_local_maven.outputs.NEW_TEMP_DIR }}'
+      - name: Clean up local maven
+        if: steps.setup_local_maven.outcome == 'success'
+        run: rm -rf '${{ steps.setup_local_maven.outputs.NEW_TEMP_DIR }}'

.github/workflows/beam_PreCommit_Java.yml

@@ -19,6 +19,7 @@ on:
     tags: ['v*']
     branches: ['master', 'release-*']
     paths:
+      - "buildSrc/**"
       - 'model/**'
       - 'sdks/java/**'
       - 'runners/**'

.github/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml

@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PreCommit Java PVR Prism Loopback
+
+on:
+  push:
+    tags: ['v*']
+    branches: ['master', 'release-*']
+    paths:
+      - 'model/**'
+      - 'sdks/go/pkg/beam/runners/prism/**'
+      - 'sdks/go/cmd/prism/**'
+      - 'runners/prism/**'
+      - 'runners/java-fn-execution/**'
+      - 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**'
+      - '.github/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml'
+  pull_request_target:
+    branches: ['master', 'release-*']
+    paths:
+      - 'model/**'
+      - 'sdks/go/pkg/beam/runners/prism/**'
+      - 'sdks/go/cmd/prism/**'
+      - 'runners/prism/**'
+      - 'runners/java-fn-execution/**'
+      - 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**'
+      - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_PVR_Prism_Loopback.json'
+  issue_comment:
+    types: [created]
+  schedule:
+    - cron: '22 2/6 * * *'
+  workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
+  cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: write
+  checks: write
+  contents: read
+  deployments: read
+  id-token: none
+  issues: write
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+env:
+  DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+  beam_PreCommit_Java_PVR_Prism_Loopback:
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    strategy:
+      matrix:
+        job_name: ["beam_PreCommit_Java_PVR_Prism_Loopback"]
+        job_phrase: ["Run Java_PVR_Prism_Loopback PreCommit"]
+    timeout-minutes: 240
+    runs-on: [self-hosted, ubuntu-20.04]
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'pull_request_target' ||
+      (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
+      github.event_name == 'workflow_dispatch' ||
+      github.event.comment.body == 'Run Java_PVR_Prism_Loopback PreCommit'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment-action
+      - name: run prismLoopbackValidatesRunnerTests script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :runners:prism:java:prismLoopbackValidatesRunnerTests
+      - name: Archive JUnit Test Results
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
+        with:
+          name: JUnit Test Results
+          path: "**/build/reports/tests/"
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
+        with:
+          name: java-code-coverage-report
+          path: "**/build/test-results/**/*.xml"