Merge branch 'master' into numpy-2-update-deps

apache · Nov 15, 2024 · 20fe987 · 20fe987
2 parents cdd60d2 + df3b6bb
commit 20fe987
Show file tree

Hide file tree

Showing 340 changed files with 9,079 additions and 3,516 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.61.0: {}
     release-2.60.0: {}
     release-2.59.0: {}
     release-2.58.1: {}

diff --git a/.github/REVIEWERS.yml b/.github/REVIEWERS.yml
@@ -61,6 +61,12 @@ labels:
     reviewers:
       - svetakvsundhar
     exclusionList: []
+  - name: kafka
+    reviewers:
+      - johnjcasey
+      - fozzie15
+      - Dippatel98
+      - sjvanrossum
   - name: Build
     reviewers:
       - damccorm

diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run",
-    "modification": 4
+    "modification": 3
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json b/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json
@@ -0,0 +1,4 @@
+{
+    "comment": "Modify this file in a trivial way to cause this test suite to run",
+    "modification": 1
+}
diff --git a/.github/trigger_files/beam_PostCommit_Java_DataflowV2.json b/.github/trigger_files/beam_PostCommit_Java_DataflowV2.json
@@ -1,3 +1,4 @@
 {
-    "comment": "Modify this file in a trivial way to cause this test suite to run"
+    "comment": "Modify this file in a trivial way to cause this test suite to run",
+    "modification": 1
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "modification": 1
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Batch.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Batch.json
@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 2
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Docker.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Docker.json
@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "modification": "1"
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json
@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 3
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json
@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 5
 }
diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
@@ -1,5 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 2
+  "modification": 4
 }
 
diff --git a/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json b/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json
@@ -1,5 +1,5 @@
 
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 2
 }
diff --git a/.github/trigger_files/beam_PostCommit_TransformService_Direct.json b/.github/trigger_files/beam_PostCommit_TransformService_Direct.json
@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "revision: "2"
 }
diff --git a/.github/trigger_files/beam_PostCommit_XVR_Direct.json b/.github/trigger_files/beam_PostCommit_XVR_Direct.json
@@ -0,0 +1,3 @@
+{
+  "https://github.com/apache/beam/pull/32648": "testing Flink 1.19 support"
+}
diff --git a/.github/trigger_files/beam_PostCommit_XVR_Flink.json b/.github/trigger_files/beam_PostCommit_XVR_Flink.json
@@ -1,3 +1,3 @@
 {
-  "https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
+  "https://github.com/apache/beam/pull/32648": "testing Flink 1.19 support"
 }
diff --git a/...orkflows/playground_backend_precommit.yml → ...b/workflows/beam_Playground_Precommit.yml b/...orkflows/playground_backend_precommit.yml → ...b/workflows/beam_Playground_Precommit.yml
@@ -17,28 +17,43 @@ name: Playground PreCommit
 
 on:
   workflow_dispatch:
-  pull_request:
+  pull_request_target:
     paths:
       - .github/workflows/playground_backend_precommit.yml
       - playground/backend/**
+  issue_comment:
+    types: [created]
 
 env:
   DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
   GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
 
 jobs:
-  precommit_check:
-    name: precommit-check
-    runs-on: ubuntu-latest
+  beam_Playground_PreCommit:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'pull_request_target' ||
+      github.event.comment.body == 'Run Playground PreCommit'
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: [beam_Playground_PreCommit]
+        job_phrase: [Run Playground PreCommit]
     env:
       DATASTORE_EMULATOR_VERSION: '423.0.0'
       PYTHON_VERSION: '3.9'
       JAVA_VERSION: '11'
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
@@ -58,7 +73,7 @@ jobs:
           sudo chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
           sudo apt-get update --yes
           sudo apt-get install sbt --yes
-          sudo wget https://codeload.github.com/spotify/scio.g8/zip/7c1ba7c1651dfd70976028842e721da4107c0d6d -O scio.g8.zip && unzip scio.g8.zip && mv scio.g8-7c1ba7c1651dfd70976028842e721da4107c0d6d /opt/scio.g8
+          sudo wget https://codeload.github.com/spotify/scio.g8/zip/7c1ba7c1651dfd70976028842e721da4107c0d6d -O scio.g8.zip && unzip scio.g8.zip && sudo mv scio.g8-7c1ba7c1651dfd70976028842e721da4107c0d6d /opt/scio.g8
       - name: Set up Cloud SDK and its components
         uses: google-github-actions/setup-gcloud@v2
         with:

diff --git a/.github/workflows/beam_PreCommit_Java.yml b/.github/workflows/beam_PreCommit_Java.yml
@@ -19,6 +19,7 @@ on:
     tags: ['v*']
     branches: ['master', 'release-*']
     paths:
+      - "buildSrc/**"
       - 'model/**'
       - 'sdks/java/**'
       - 'runners/**'

diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml
@@ -245,8 +245,7 @@ jobs:
 
   stage_docker:
     if:  ${{ fromJson(github.event.inputs.STAGE).docker_artifacts == 'yes'}}
-    # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step
-    runs-on: ubuntu-22.04
+    runs-on: [self-hosted, ubuntu-20.04, highmem]
     steps:
       - name: Checkout
         uses: actions/checkout@v4

diff --git a/.github/workflows/self-assign.yml b/.github/workflows/self-assign.yml
@@ -40,12 +40,16 @@ jobs:
                 repo: context.repo.repo,
                 assignees: [context.payload.comment.user.login]
               });
-              github.rest.issues.removeLabel({
-                issue_number: context.issue.number,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                name: 'awaiting triage'
-              });
+              try {
+                github.rest.issues.removeLabel({
+                  issue_number: context.issue.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: 'awaiting triage'
+                });
+              } catch (error) {
+                console.log(`Failed to remove awaiting triage label. It may not exist on this issue. Error ${error}`);
+              }
             } else if (bodyString == '.close-issue') {
               console.log('Closing issue');
               if (i + 1 < body.length && body[i+1].toLowerCase() == 'not_planned') {

diff --git a/CHANGES.md b/CHANGES.md
@@ -53,6 +53,40 @@
 * ([#X](https://github.com/apache/beam/issues/X)).
 -->
 
+# [2.62.0] - Unreleased
+
+## Highlights
+
+* New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)).
+* New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)).
+
+## I/Os
+
+* Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+
+## New Features / Improvements
+
+* X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+
+## Breaking Changes
+
+* X behavior was changed ([#X](https://github.com/apache/beam/issues/X)).
+
+## Deprecations
+
+* X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)).
+
+## Bugfixes
+
+* Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+
+## Security Fixes
+* Fixed (CVE-YYYY-NNNN)[https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN] (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)).
+
+## Known Issues
+
+* ([#X](https://github.com/apache/beam/issues/X)).
+
 # [2.61.0] - Unreleased
 
 ## Highlights
@@ -69,6 +103,9 @@
 * [Managed Iceberg] Now available in Python SDK ([#31495](https://github.com/apache/beam/pull/31495))
 * [Managed Iceberg] Add support for TIMESTAMP, TIME, and DATE types ([#32688](https://github.com/apache/beam/pull/32688))
 * BigQuery CDC writes are now available in Python SDK, only supported when using StorageWrite API at least once mode ([#32527](https://github.com/apache/beam/issues/32527))
+* [Managed Iceberg] Allow updating table partition specs during pipeline runtime ([#32879](https://github.com/apache/beam/pull/32879))
+* Added BigQueryIO as a Managed IO ([#31486](https://github.com/apache/beam/pull/31486))
+* Support for writing to [Solace messages queues](https://solace.com/) (`SolaceIO.Write`) added (Java) ([#31905](https://github.com/apache/beam/issues/31905)).
 
 ## New Features / Improvements
 
@@ -93,6 +130,7 @@
 * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
 * (Java) Fixed tearDown not invoked when DoFn throws on Portable Runners ([#18592](https://github.com/apache/beam/issues/18592), [#31381](https://github.com/apache/beam/issues/31381)).
 * (Java) Fixed protobuf error with MapState.remove() in Dataflow Streaming Java Legacy Runner without Streaming Engine ([#32892](https://github.com/apache/beam/issues/32892)).
+* Adding flag to support conditionally disabling auto-commit in JdbcIO ReadFn ([#31111](https://github.com/apache/beam/issues/31111))
 
 ## Security Fixes
 * Fixed (CVE-YYYY-NNNN)[https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN] (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)).

diff --git a/contributor-docs/code-change-guide.md b/contributor-docs/code-change-guide.md
@@ -145,7 +145,7 @@ in the Google Cloud documentation.
 
 Depending on the languages involved, your `PATH` file needs to have the following elements configured.
 
-* A Java environment that uses a supported Java version, preferably Java 8.
+* A Java environment that uses a supported Java version, preferably Java 11.
   * This environment is needed for all development, because Beam is a Gradle project that uses JVM.
   * Recommended: To manage Java versions, use [sdkman](https://sdkman.io/install).
 
@@ -624,6 +624,11 @@ Tips for using the Dataflow runner:
 
 ## Appendix
 
+### Common Issues
+
+* If you run into some strange errors such as `java.lang.NoClassDefFoundError`, run `./gradlew clean` first
+* To run one single Java test with gradle, use `--tests` to filter, for example, `./gradlew :it:google-cloud-platform:WordCountIntegrationTest --tests "org.apache.beam.it.gcp.WordCountIT.testWordCountDataflow"`
+
 ### Directories of snapshot builds
 
 * https://repository.apache.org/content/groups/snapshots/org/apache/beam/ Java SDK build (nightly)

diff --git a/contributor-docs/release-guide.md b/contributor-docs/release-guide.md
@@ -756,7 +756,7 @@ template; please adjust as you see fit.
 
     Reviewers are encouraged to test their own use cases with the release candidate, and vote +1 if
     no issues are found. Only PMC member votes will count towards the final vote, but votes from all
-    community members is encouraged and helpful for finding regressions; you can either test your own
+    community members are encouraged and helpful for finding regressions; you can either test your own
     use cases [13] or use cases from the validation sheet [10].
 
     The complete staging area is available for your review, which includes:
@@ -765,7 +765,7 @@ template; please adjust as you see fit.
     * all artifacts to be deployed to the Maven Central Repository [4],
     * source code tag "v1.2.3-RC3" [5],
     * website pull request listing the release [6], the blog post [6], and publishing the API reference manual [7].
-    * Python artifacts are deployed along with the source release to the dist.apache.org [2] and PyPI[8].
+    * Python artifacts are deployed along with the source release to dist.apache.org [2] and PyPI[8].
     * Go artifacts and documentation are available at pkg.go.dev [9]
     * Validation sheet with a tab for 1.2.3 release to help with validation [10].
     * Docker images published to Docker Hub [11].

diff --git a/examples/java/build.gradle b/examples/java/build.gradle
@@ -66,6 +66,8 @@ dependencies {
   implementation project(":sdks:java:extensions:python")
   implementation project(":sdks:java:io:google-cloud-platform")
   implementation project(":sdks:java:io:kafka")
+  runtimeOnly project(":sdks:java:io:iceberg")
+  implementation project(":sdks:java:managed")
   implementation project(":sdks:java:extensions:ml")
   implementation library.java.avro
   implementation library.java.bigdataoss_util
@@ -100,6 +102,8 @@ dependencies {
   implementation "org.apache.httpcomponents:httpcore:4.4.13"
   implementation "com.fasterxml.jackson.core:jackson-annotations:2.14.1"
   implementation "com.fasterxml.jackson.core:jackson-core:2.14.1"
+  runtimeOnly library.java.hadoop_client
+  runtimeOnly library.java.bigdataoss_gcs_connector
   testImplementation project(path: ":runners:direct-java", configuration: "shadow")
   testImplementation project(":sdks:java:io:google-cloud-platform")
   testImplementation project(":sdks:java:extensions:ml")