diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index a4ce41fce073..a1a4c3bf039a 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -67,7 +67,7 @@ io: ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/ "mqtt": ["sdks/java/io/mqtt/**/*"] "parquet": ["sdks/java/io/parquet/**/*"] "rabbitmq": ["sdks/java/io/rabbitmq/**/*"] -"redis": ["sdks/java/io/redis/**/*"] +"redis": ["sdks/java/io/redis/**/*"] "solr": ["sdks/java/io/solr/**/*"] "spanner": ["sdks/go/pkg/beam/io/spannerio/**/*", "sdks/python/apache_beam/io/gcp/spanner.py", "sdks/python/apache_beam/io/gcp/experimental/spannerio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/**/*"] "bigtable": ["sdks/go/pkg/beam/io/bigtableio/**/*", "sdks/go/pkg/beam/io/xlang/bigtableio/**/*", "sdks/python/apache_beam/io/gcp/bigtableio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/**/*"] @@ -76,9 +76,9 @@ io: ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/ "thrift": ["sdks/java/io/thrift/**/*"] "tika": ["sdks/java/io/tika/**/*"] "xml": ["sdks/java/io/xml/**/*"] - + # Runners -"runners": ["runners/**/*", "sdks/go/pkg/beam/runners/**/*", "sdks/python/runners/**/*", "sdks/typescript/src/apache_beam/runners/**/*"] +"runners": ["runners/**/*", "sdks/go/pkg/beam/runners/**/*", "sdks/python/apache_beam/runners/**/*", "sdks/typescript/src/apache_beam/runners/**/*"] "core": ["runners/core-construction-java/**/*", "runners/core-java/**/*"] "dataflow": ["runners/google-cloud-dataflow-java/**/*", "sdks/go/pkg/beam/runners/dataflow/**/*", "sdks/python/runners/dataflow/**/*"] "direct": ["runners/direct-java/**/*", "sdks/go/pkg/beam/runners/direct/**/*", "sdks/python/runners/direct/**/*"] @@ -88,6 +88,7 @@ io: ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/ "jet": ["runners/jet/**/*"] "local": ["runners/local-java/**/*"] "portability": ["runners/portability/**/*"] +"prism": ["runners/prism/**/*", "sdks/go/pkg/beam/runners/prism/**/*", "sdks/go/cmd/prism/**/*", "sdks/python/apache_beam/runners/portability/prism_runner.py","sdks/python/apache_beam/runners/portability/prism_runner_test.py"] "samza": ["runners/samza/**/*"] "spark": ["runners/spark/**/*", "sdks/go/pkg/beam/runners/spark/**/*"] "twister2": ["runners/twister2/**/*"] diff --git a/README.md b/README.md index 8a6db1622068..24ab4963701d 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Have ideas for new SDKs or DSLs? See the [sdk-ideas label](https://github.com/ap Beam supports executing programs on multiple distributed processing backends through PipelineRunners. Currently, the following PipelineRunners are available: - The `DirectRunner` runs the pipeline on your local machine. +- The `PrismRunner` runs the pipeline on your local machine using Beam Portability. - The `DataflowRunner` submits the pipeline to the [Google Cloud Dataflow](http://cloud.google.com/dataflow/). - The `FlinkRunner` runs the pipeline on an Apache Flink cluster. The code has been donated from [dataArtisans/flink-dataflow](https://github.com/dataArtisans/flink-dataflow) and is now part of Beam. - The `SparkRunner` runs the pipeline on an Apache Spark cluster. diff --git a/website/www/site/content/en/blog/beam-2.59.0.md b/website/www/site/content/en/blog/beam-2.59.0.md index 68b712ae8fe1..2720eaed738f 100644 --- a/website/www/site/content/en/blog/beam-2.59.0.md +++ b/website/www/site/content/en/blog/beam-2.59.0.md @@ -30,7 +30,7 @@ For more information on changes in 2.59.0, check out the [detailed release notes ## Highlights * Added support for setting a configureable timeout when loading a model and performing inference in the [RunInference](https://beam.apache.org/documentation/ml/inference-overview/) transform using [with_exception_handling](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.RunInference.with_exception_handling) ([#32137](https://github.com/apache/beam/issues/32137)) -* Initial experimental support for using Prism with the Java and Python SDKs +* Initial experimental support for using [Prism](/documentation/runners/prism/) with the Java and Python SDKs * Prism is presently targeting local testing usage, or other small scale execution. * For Java, use 'PrismRunner', or 'TestPrismRunner' as an argument to the `--runner` flag. * For Python, use 'PrismRunner' as an argument to the `--runner` flag. diff --git a/website/www/site/content/en/documentation/pipelines/test-your-pipeline.md b/website/www/site/content/en/documentation/pipelines/test-your-pipeline.md index 744b12aa1625..07557dc39ac0 100644 --- a/website/www/site/content/en/documentation/pipelines/test-your-pipeline.md +++ b/website/www/site/content/en/documentation/pipelines/test-your-pipeline.md @@ -22,9 +22,9 @@ Testing your pipeline is a particularly important step in developing an effectiv Before running your pipeline on the runner of your choice, unit testing your pipeline code locally is often the best way to identify and fix bugs in your pipeline code. Unit testing your pipeline locally also allows you to use your familiar/favorite local debugging tools. -You can use [DirectRunner](/documentation/runners/direct), a local runner helpful for testing and local development. +You can use [DirectRunner](/documentation/runners/direct), or [PrismRunner](/documentation/runners/prism). Both are local runners helpful for testing and local development. -After you test your pipeline using the `DirectRunner`, you can use the runner of your choice to test on a small scale. For example, use the Flink runner with a local or remote Flink cluster. +After you test your pipeline locally, you can use the runner of your choice to test on a small scale. For example, use the Flink runner with a local or remote Flink cluster. The Beam SDKs provide a number of ways to unit test your pipeline code, from the lowest to the highest levels. From the lowest to the highest level, these are: diff --git a/website/www/site/content/en/documentation/runners/direct.md b/website/www/site/content/en/documentation/runners/direct.md index e18d4a902c15..82e7127d19cb 100644 --- a/website/www/site/content/en/documentation/runners/direct.md +++ b/website/www/site/content/en/documentation/runners/direct.md @@ -32,6 +32,7 @@ Using the Direct Runner for testing and development helps ensure that pipelines Here are some resources with information about how to test your pipelines.