Support data type map in Glue import. (#340)

* Support data type map in Glue import. Resolves #334 * Support data type map in Glue import. Resolves #334
datacontract · Jul 21, 2024 · 56f4dbe · 56f4dbe
1 parent 3b3419a
commit 56f4dbe
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Support data type map in Glue import. (#340)
+
 ### Fixed
 
 - Fix an issue where the quality and definition `$ref` are not always resolved.

diff --git a/datacontract/imports/glue_importer.py b/datacontract/imports/glue_importer.py
@@ -186,7 +186,7 @@ def create_typed_field(dtype: str) -> Field:
     """
     field = Field()
     dtype = dtype.strip().lower().replace(" ", "")
-    if dtype.startswith(("array", "struct")):
+    if dtype.startswith(("array", "struct", "map")):
         orig_dtype: str = dtype
         if dtype.startswith("array"):
             field.type = "array"
@@ -195,6 +195,12 @@ def create_typed_field(dtype: str) -> Field:
             field.type = "struct"
             for f in split_struct(orig_dtype[7:-1]):
                 field.fields[f.split(":", 1)[0].strip()] = create_typed_field(f.split(":", 1)[1])
+        elif dtype.startswith("map"):
+            field.type = "map"
+            key_type = orig_dtype[4:-1].split(",", 1)[0]
+            value_type = orig_dtype[4:-1].split(",", 1)[1]
+            field.keys = create_typed_field(key_type)
+            field.values = create_typed_field(value_type)
     else:
         field.type = map_type_from_sql(dtype)
     return field

diff --git a/tests/fixtures/glue/datacontract.yaml b/tests/fixtures/glue/datacontract.yaml
@@ -43,6 +43,12 @@ models:
                             type: string
                         sub_field_four:
                             type: integer
+            field_eight:
+                type: map
+                keys:
+                    type: string
+                values:
+                    type: integer
             part_one:
                 description: Partition Key
                 required: True

diff --git a/tests/test_import_glue.py b/tests/test_import_glue.py
@@ -65,6 +65,10 @@ def setup_mock_glue(aws_credentials):
                             "Name": "field_seven",
                             "Type": "array<struct<sub_field_three:string, sub_field_four:int>>",
                         },
+                        {
+                            "Name": "field_eight",
+                            "Type": "map<string,int>",
+                        },
                     ]
                 },
                 "PartitionKeys": [