Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YAML] Better docs for Filter and MapToFields. #33274

Merged
merged 4 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/yaml/generate_yaml_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ def main():
if options.markdown_file or options.html_file:
if '-' in transforms[0]:
extra_docs = 'Supported languages: ' + ', '.join(
t.split('-')[-1] for t in sorted(transforms))
t.split('-')[-1] for t in sorted(transforms)) + '.'
else:
extra_docs = ''
markdown_out.write(
Expand Down
30 changes: 28 additions & 2 deletions sdks/python/apache_beam/yaml/yaml_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from typing import Callable
from typing import Collection
from typing import Dict
from typing import Iterable
from typing import List
from typing import Mapping
from typing import Optional
Expand Down Expand Up @@ -616,6 +617,13 @@ def _PyJsFilter(

See more complete documentation on
[YAML Filtering](https://beam.apache.org/documentation/sdks/yaml-udf/#filtering).

Args:
keep: An expression evaluating to true for those records that should be kept.
language: The language of the above expression.
Defaults to generic.
error_handling: Whether and where to output records that throw errors when
the above expressions are evaluated.
""" # pylint: disable=line-too-long
keep_fn = _as_callable_for_pcoll(pcoll, keep, "keep", language or 'generic')
return pcoll | beam.Filter(keep_fn)
Expand Down Expand Up @@ -661,14 +669,32 @@ def normalize_fields(pcoll, fields, drop=(), append=False, language='generic'):

@beam.ptransform.ptransform_fn
@maybe_with_exception_handling_transform_fn
def _PyJsMapToFields(pcoll, language='generic', **mapping_args):
def _PyJsMapToFields(
pcoll,
fields: Mapping[str, Union[str, Mapping[str, str]]],
append: Optional[bool] = False,
drop: Optional[Iterable[str]] = None,
language: Optional[str] = None):
"""Creates records with new fields defined in terms of the input fields.

See more complete documentation on
[YAML Mapping Functions](https://beam.apache.org/documentation/sdks/yaml-udf/#mapping-functions).

Args:
fields: The output fields to compute, each mapping to the expression or
callable that creates them.
append: Whether to append the created fields to the set of
fields already present, outputting a union of both the new fields and
the original fields for each record. Defaults to False.
drop: If `append` is true, enumerates a subset of fields from the
original record that should not be kept
language: The language used to define (and execute) the
expressions and/or callables in `fields`. Defaults to generic.
error_handling: Whether and where to output records that throw errors when
the above expressions are evaluated.
""" # pylint: disable=line-too-long
input_schema, fields = normalize_fields(
pcoll, language=language, **mapping_args)
pcoll, fields, drop or (), append, language=language or 'generic')
if language == 'javascript':
options.YamlOptions.check_enabled(pcoll.pipeline, 'javascript')

Expand Down
Loading