From fd809380e0531c5496555a30b5542d64b4535587 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 2 Mar 2023 10:21:39 -0500 Subject: [PATCH 1/7] Update Beam website to release 2.46.0 --- website/www/site/config.toml | 2 +- .../www/site/content/en/blog/beam-2.46.0.md | 229 ++++++++++++++++++ .../site/content/en/get-started/downloads.md | 14 +- 3 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 website/www/site/content/en/blog/beam-2.46.0.md diff --git a/website/www/site/config.toml b/website/www/site/config.toml index 4fbdbac21809..3f21c90e39df 100644 --- a/website/www/site/config.toml +++ b/website/www/site/config.toml @@ -104,7 +104,7 @@ github_project_repo = "https://github.com/apache/beam" [params] description = "Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes." -release_latest = "2.45.0" +release_latest = "2.46.0" # The repository and branch where the files live in Github or Colab. This is used # to serve and stage from your local branch, but publish to the master branch. # e.g. https://github.com/{{< param branch_repo >}}/path/to/notebook.ipynb diff --git a/website/www/site/content/en/blog/beam-2.46.0.md b/website/www/site/content/en/blog/beam-2.46.0.md new file mode 100644 index 000000000000..4b358b2c17ce --- /dev/null +++ b/website/www/site/content/en/blog/beam-2.46.0.md @@ -0,0 +1,229 @@ +--- +title: "Apache Beam 2.46.0" +date: YYYY-MM-DD H:MM:00 Z - TODO +categories: + - blog + - release +authors: + - damccorm +--- + + +We are happy to present the new 2.46.0 release of Beam. +This release includes both improvements and new functionality. +See the [download page](/get-started/downloads/#2460-2023-01-13) for this release. - TODO - update this anchor + + + +For more information on changes in 2.46.0, check out the [detailed release notes](https://github.com/apache/beam/milestone/9?closed=1). + +## Highlights + +* Java SDK containers migrated to [Eclipse Temurin](https://hub.docker.com/_/eclipse-temurin) + as a base. This change migrates away from the deprecated [OpenJDK](https://hub.docker.com/_/openjdk) + container. Eclipse Temurin is currently based upon Ubuntu 22.04 while the OpenJDK + container was based upon Debian 11. +* RunInference PTransform will accept model paths as SideInputs in Python SDK. ([#24042](https://github.com/apache/beam/issues/24042)) +* RunInference supports ONNX runtime in Python SDK ([#22972](https://github.com/apache/beam/issues/22972)) +* Tensorflow Model Handler for RunInference in Python SDK ([#25366](https://github.com/apache/beam/issues/25366)) +* Java SDK modules migrated to use `:sdks:java:extensions:avro` ([#24748](https://github.com/apache/beam/issues/24748)) + +## I/Os + +* Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). +* Added in JmsIO a retry policy for failed publications (Java) ([#24971](https://github.com/apache/beam/issues/24971)). +* Support for `LZMA` compression/decompression of text files added to the Python SDK ([#25316](https://github.com/apache/beam/issues/25316)) +* Added ReadFrom/WriteTo Csv/Json as top-level transforms to the Python SDK. + +## New Features / Improvements + +* Add UDF metrics support for Samza portable mode. +* Option for SparkRunner to avoid the need of SDF output to fit in memory ([#23852](https://github.com/apache/beam/issues/23852)). + This helps e.g. with ParquetIO reads. Turn the feature on by adding experiment `use_bounded_concurrent_output_for_sdf`. +* Add `WatchFilePattern` transform, which can be used as a side input to the RunInference PTransfrom to watch for model updates using a file pattern. ([#24042](https://github.com/apache/beam/issues/24042)) +* Add support for loading TorchScript models with `PytorchModelHandler`. The TorchScript model path can be + passed to PytorchModelHandler using `torch_script_model_path=`. ([#25321](https://github.com/apache/beam/pull/25321)) +* The Go SDK now requires Go 1.19 to build. ([#25545](https://github.com/apache/beam/pull/25545)) +* The Go SDK now has an initial native Go implementation of a portable Beam Runner called Prism. ([#24789](https://github.com/apache/beam/pull/24789)) + * For more details and current state see https://github.com/apache/beam/tree/master/sdks/go/pkg/beam/runners/prism. + +## Breaking Changes + +* The deprecated SparkRunner for Spark 2 (see [2.41.0](#2410---2022-08-23)) was removed ([#25263](https://github.com/apache/beam/pull/25263)). +* Python's BatchElements performs more aggressive batching in some cases, + capping at 10 second rather than 1 second batches by default and excluding + fixed cost in this computation to better handle cases where the fixed cost + is larger than a single second. To get the old behavior, one can pass + `target_batch_duration_secs_including_fixed_cost=1` to BatchElements. + +## Deprecations + +* Avro related classes are deprecated in module `beam-sdks-java-core` and will be eventually removed. Please, migrate to a new module `beam-sdks-java-extensions-avro` instead by importing the classes from `org.apache.beam.sdk.extensions.avro` package. + For the sake of migration simplicity, the relative package path and the whole class hierarchy of Avro related classes in new module is preserved the same as it was before. + For example, import `org.apache.beam.sdk.extensions.avro.coders.AvroCoder` class instead of`org.apache.beam.sdk.coders.AvroCoder`. ([#24749](https://github.com/apache/beam/issues/24749)). + +## List of Contributors + +According to git shortlog, the following people contributed to the 2.46.0 release. Thank you to all contributors! + +Ahmet Altay + +Alan Zhang + +Alexey Romanenko + +Amrane Ait Zeouay + +Anand Inguva + +Andrew Pilloud + +Brian Hulette + +Bruno Volpato + +Byron Ellis + +Chamikara Jayalath + +Damon + +Danny McCormick + +Darkhan Nausharipov + +David Katz + +Dmitry Repin + +Doug Judd + +Egbert van der Wal + +Elizaveta Lomteva + +Evan Galpin + +Herman Mak + +Jack McCluskey + +Jan Lukavský + +Johanna Öjeling + +John Casey + +Jozef Vilcek + +Junhao Liu + +Juta Staes + +Katie Liu + +Kiley Sok + +Liam Miller-Cushon + +Luke Cwik + +Moritz Mack + +Ning Kang + +Oleh Borysevych + +Pablo E + +Pablo Estrada + +Reuven Lax + +Ritesh Ghorse + +Robert Bradshaw + +Robert Burke + +Ruslan Altynnikov + +Ryan Zhang + +Sam Rohde + +Sam Whittle + +Sam sam + +Sergei Lilichenko + +Shivam + +Shubham Krishna + +Theodore Ni + +Timur Sultanov + +Tony Tang + +Vachan + +Veronica Wasson + +Vincent Devillers + +Vitaly Terentyev + +William Ross Morrow + +Xinyu Liu + +Yi Hu + +ZhengLin Li + +Ziqi Ma + +ahmedabu98 + +alexeyinkin + +aliftadvantage + +bullet03 + +dannikay + +darshan-sj + +dependabot[bot] + +johnjcasey + +kamrankoupayi + +kileys + +liferoad + +nancyxu123 + +nickuncaged1201 + +pablo rodriguez defino + +tvalentyn + +xqhu \ No newline at end of file diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index 3575ec06a30c..cd6e3fc3163d 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -96,10 +96,18 @@ versions denoted `0.x.y`. ## Releases +### 2.46.0 (TODO - replace with date) +Official [source code download](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip). +[SHA-512](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip.sha512). +[signature](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip.asc). + +[Release notes](https://github.com/apache/beam/releases/tag/v2.46.0) +[Blog post](/blog/beam-2.46.0). + ### 2.45.0 (2023-02-15) -Official [source code download](https://downloads.apache.org/beam/2.45.0/apache-beam-2.45.0-source-release.zip). -[SHA-512](https://downloads.apache.org/beam/2.45.0/apache-beam-2.45.0-source-release.zip.sha512). -[signature](https://downloads.apache.org/beam/2.45.0/apache-beam-2.45.0-source-release.zip.asc). +Official [source code download](https://archive.apache.org/dist/beam/2.45.0/apache-beam-2.45.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.45.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.45.0-source-release.zip.asc). [Release notes](https://github.com/apache/beam/releases/tag/v2.45.0) [Blog post](/blog/beam-2.45.0). From 4faeb3c0beed4aae7f9341205452194cfc04e8a9 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 8 Mar 2023 12:48:15 -0500 Subject: [PATCH 2/7] Update date TODOs --- website/www/site/content/en/blog/beam-2.46.0.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/www/site/content/en/blog/beam-2.46.0.md b/website/www/site/content/en/blog/beam-2.46.0.md index 4b358b2c17ce..c7b3bf986b05 100644 --- a/website/www/site/content/en/blog/beam-2.46.0.md +++ b/website/www/site/content/en/blog/beam-2.46.0.md @@ -1,6 +1,6 @@ --- title: "Apache Beam 2.46.0" -date: YYYY-MM-DD H:MM:00 Z - TODO +date: 2023-03-08 13:00:00 -0500 categories: - blog - release @@ -21,7 +21,7 @@ limitations under the License. We are happy to present the new 2.46.0 release of Beam. This release includes both improvements and new functionality. -See the [download page](/get-started/downloads/#2460-2023-01-13) for this release. - TODO - update this anchor +See the [download page](/get-started/downloads/#2460-2023-03-08) for this release. @@ -226,4 +226,4 @@ pablo rodriguez defino tvalentyn -xqhu \ No newline at end of file +xqhu From 83f24571c6d8eff3ab5d3b340fd935f358683651 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 8 Mar 2023 12:48:45 -0500 Subject: [PATCH 3/7] Update downloads.md --- website/www/site/content/en/get-started/downloads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index cd6e3fc3163d..d03c1c926503 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -96,7 +96,7 @@ versions denoted `0.x.y`. ## Releases -### 2.46.0 (TODO - replace with date) +### 2.46.0 (2023-03-08) Official [source code download](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip). [SHA-512](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip.sha512). [signature](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip.asc). From f72e191ff5265b563a5bc4d975e312ed38b4b5e7 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 8 Mar 2023 12:57:00 -0500 Subject: [PATCH 4/7] Update downloads.md Fix 2.45 version --- website/www/site/content/en/get-started/downloads.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index d03c1c926503..0712768a58cf 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -106,8 +106,8 @@ Official [source code download](https://downloads.apache.org/beam/2.46.0/apache- ### 2.45.0 (2023-02-15) Official [source code download](https://archive.apache.org/dist/beam/2.45.0/apache-beam-2.45.0-source-release.zip). -[SHA-512](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.45.0-source-release.zip.sha512). -[signature](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.45.0-source-release.zip.asc). +[SHA-512](https://archive.apache.org/dist/beam/2.45.0/apache-beam-2.45.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.45.0/apache-beam-2.45.0-source-release.zip.asc). [Release notes](https://github.com/apache/beam/releases/tag/v2.45.0) [Blog post](/blog/beam-2.45.0). From 70df2296ade8ad71ae2992abd875335b394f5d18 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 8 Mar 2023 13:05:10 -0500 Subject: [PATCH 5/7] Remove placeholder IO callout --- website/www/site/content/en/blog/beam-2.46.0.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/www/site/content/en/blog/beam-2.46.0.md b/website/www/site/content/en/blog/beam-2.46.0.md index c7b3bf986b05..c019afce59e3 100644 --- a/website/www/site/content/en/blog/beam-2.46.0.md +++ b/website/www/site/content/en/blog/beam-2.46.0.md @@ -40,7 +40,6 @@ For more information on changes in 2.46.0, check out the [detailed release notes ## I/Os -* Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). * Added in JmsIO a retry policy for failed publications (Java) ([#24971](https://github.com/apache/beam/issues/24971)). * Support for `LZMA` compression/decompression of text files added to the Python SDK ([#25316](https://github.com/apache/beam/issues/25316)) * Added ReadFrom/WriteTo Csv/Json as top-level transforms to the Python SDK. From f7e384a7bc25bc87006e4f5d446ff03ee50a2bba Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 10 Mar 2023 14:44:25 -0500 Subject: [PATCH 6/7] Update date --- website/www/site/content/en/blog/beam-2.46.0.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/www/site/content/en/blog/beam-2.46.0.md b/website/www/site/content/en/blog/beam-2.46.0.md index c019afce59e3..278ef807b365 100644 --- a/website/www/site/content/en/blog/beam-2.46.0.md +++ b/website/www/site/content/en/blog/beam-2.46.0.md @@ -1,6 +1,6 @@ --- title: "Apache Beam 2.46.0" -date: 2023-03-08 13:00:00 -0500 +date: 2023-03-10 13:00:00 -0500 categories: - blog - release @@ -21,7 +21,7 @@ limitations under the License. We are happy to present the new 2.46.0 release of Beam. This release includes both improvements and new functionality. -See the [download page](/get-started/downloads/#2460-2023-03-08) for this release. +See the [download page](/get-started/downloads/#2460-2023-03-10) for this release. From a809bccfbe42864eecff326906e24e6efed4c1b5 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 10 Mar 2023 14:44:58 -0500 Subject: [PATCH 7/7] Update date --- website/www/site/content/en/get-started/downloads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index 0712768a58cf..6cc83fac00da 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -96,7 +96,7 @@ versions denoted `0.x.y`. ## Releases -### 2.46.0 (2023-03-08) +### 2.46.0 (2023-03-10) Official [source code download](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip). [SHA-512](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip.sha512). [signature](https://downloads.apache.org/beam/2.46.0/apache-beam-2.46.0-source-release.zip.asc).