From ee8e53be408659fb67c00cac45c8f37cc70529e2 Mon Sep 17 00:00:00 2001 From: Robert Bradshaw Date: Fri, 8 Sep 2023 10:38:59 -0700 Subject: [PATCH] Add standard {ReadFrom,WriteTo}{Csv,Json} yaml endpoints. (#28268) --- sdks/python/apache_beam/yaml/standard_io.yaml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/sdks/python/apache_beam/yaml/standard_io.yaml b/sdks/python/apache_beam/yaml/standard_io.yaml index ea21446c2fb9..e60f0026fd25 100644 --- a/sdks/python/apache_beam/yaml/standard_io.yaml +++ b/sdks/python/apache_beam/yaml/standard_io.yaml @@ -51,3 +51,32 @@ 'ReadFromBigQuery': 'apache_beam.yaml.yaml_io.read_from_bigquery' # Disable until https://github.com/apache/beam/issues/28162 is resolved. # 'WriteToBigQuery': 'apache_beam.yaml.yaml_io.write_to_bigquery' + +# Declared as a renaming transform to avoid exposing all +# (implementation-specific) pandas arguments and aligning with possible Java +# implementation. +# Invoking these directly as a PyTransform is still an option for anyone wanting +# to use these power-features in a language-dependent manner. +- type: renaming + transforms: + 'ReadFromCsv': 'ReadFromCsv' + 'WriteToCsv': 'WriteToCsv' + 'ReadFromJson': 'ReadFromJson' + 'WriteToJson': 'WriteToJson' + config: + mappings: + 'ReadFromCsv': + path: 'path' + 'WriteToCsv': + path: 'path' + 'ReadFromJson': + path: 'path' + 'WriteToJson': + path: 'path' + underlying_provider: + type: python + transforms: + 'ReadFromCsv': 'apache_beam.io.ReadFromCsv' + 'WriteToCsv': 'apache_beam.io.WriteToCsv' + 'ReadFromJson': 'apache_beam.io.ReadFromJson' + 'WriteToJson': 'apache_beam.io.WriteToJson'