Skip to content

Commit

Permalink
Add docs for YAML AssertThat. (apache#31448)
Browse files Browse the repository at this point in the history
This is the first transform in the (alphabetical) list, so it'd
be good to not have it empty.

Also produce slightly nicer examples for repeated arguments.
  • Loading branch information
robertwb authored May 29, 2024
1 parent 19630e5 commit 80d85aa
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 3 deletions.
24 changes: 22 additions & 2 deletions sdks/python/apache_beam/yaml/generate_yaml_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,18 @@
from apache_beam.yaml import yaml_provider


def _singular(name):
# Simply removing an 's' (or 'es', or 'ies', ...) may result in surprising
# manglings. Better to play it safe and leave a correctly-spelled plural
# than a botched singular in our examples configs.
return {
'args': 'arg',
'attributes': 'attribute',
'elements': 'element',
'fields': 'field',
}.get(name, name)


def _fake_value(name, beam_type):
type_info = beam_type.WhichOneof("type_info")
if type_info == "atomic_type":
Expand All @@ -38,9 +50,17 @@ def _fake_value(name, beam_type):
else:
return name
elif type_info == "array_type":
return [_fake_value(name, beam_type.array_type.element_type), '...']
return [
_fake_value(_singular(name), beam_type.array_type.element_type),
_fake_value(_singular(name), beam_type.array_type.element_type),
'...'
]
elif type_info == "iterable_type":
return [_fake_value(name, beam_type.iterable_type.element_type), '...']
return [
_fake_value(_singular(name), beam_type.iterable_type.element_type),
_fake_value(_singular(name), beam_type.iterable_type.element_type),
'...'
]
elif type_info == "map_type":
if beam_type.map_type.key_type.atomic_type == schema_pb2.STRING:
return {
Expand Down
25 changes: 24 additions & 1 deletion sdks/python/apache_beam/yaml/yaml_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,30 @@ def dicts_to_rows(o):

class YamlProviders:
class AssertEqual(beam.PTransform):
def __init__(self, elements):
"""Asserts that the input contains exactly the elements provided.
This is primarily used for testing; it will cause the entire pipeline to
fail if the input to this transform is not exactly the set of `elements`
given in the config parameter.
As with Create, YAML/JSON-style mappings are interpreted as Beam rows,
e.g.::
type: AssertEqual
input: SomeTransform
config:
elements:
- {a: 0, b: "foo"}
- {a: 1, b: "bar"}
would ensure that `SomeTransform` produced exactly two elements with values
`(a=0, b="foo")` and `(a=1, b="bar")` respectively.
Args:
elements: The set of elements that should belong to the PCollection.
YAML/JSON-style mappings will be interpreted as Beam rows.
"""
def __init__(self, elements: Iterable[Any]):
self._elements = elements

def expand(self, pcoll):
Expand Down

0 comments on commit 80d85aa

Please sign in to comment.