[yaml] Fix yaml provider schema validation and merging (#31974)

Signed-off-by: Jeffrey Kinard <jeff@thekinards.com>
apache · Jul 25, 2024 · 6a1d917 · 6a1d917
1 parent dc1e134
commit 6a1d917
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/sdks/python/apache_beam/yaml/pipeline.schema.yaml b/sdks/python/apache_beam/yaml/pipeline.schema.yaml
@@ -168,8 +168,10 @@ $defs:
 
   providerOrProviderInclude:
     if:
-      properties:
-        include {}
+      allOf: [
+        { properties: { include: { type: string }}},
+        { required: [ "include" ] }
+      ]
     then:
       $ref: '#/$defs/providerInclude'
     else:

diff --git a/sdks/python/apache_beam/yaml/yaml_transform.py b/sdks/python/apache_beam/yaml/yaml_transform.py
@@ -1064,5 +1064,5 @@ def expand_pipeline(
   return YamlTransform(
       pipeline_as_composite(pipeline_spec['pipeline']),
       yaml_provider.merge_providers(
-          pipeline_spec.get('providers', []), providers or
-          {})).expand(beam.pvalue.PBegin(pipeline))
+          yaml_provider.parse_providers(pipeline_spec.get('providers', [])),
+          providers or {})).expand(beam.pvalue.PBegin(pipeline))