diff --git a/CHANGELOG.md b/CHANGELOG.md index a9b526b6..338d97ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Support data type map in Glue import. (#340) + ### Fixed - Fix an issue where the quality and definition `$ref` are not always resolved. diff --git a/datacontract/imports/glue_importer.py b/datacontract/imports/glue_importer.py index 29a96807..b65fff07 100644 --- a/datacontract/imports/glue_importer.py +++ b/datacontract/imports/glue_importer.py @@ -186,7 +186,7 @@ def create_typed_field(dtype: str) -> Field: """ field = Field() dtype = dtype.strip().lower().replace(" ", "") - if dtype.startswith(("array", "struct")): + if dtype.startswith(("array", "struct", "map")): orig_dtype: str = dtype if dtype.startswith("array"): field.type = "array" @@ -195,6 +195,12 @@ def create_typed_field(dtype: str) -> Field: field.type = "struct" for f in split_struct(orig_dtype[7:-1]): field.fields[f.split(":", 1)[0].strip()] = create_typed_field(f.split(":", 1)[1]) + elif dtype.startswith("map"): + field.type = "map" + key_type = orig_dtype[4:-1].split(",", 1)[0] + value_type = orig_dtype[4:-1].split(",", 1)[1] + field.keys = create_typed_field(key_type) + field.values = create_typed_field(value_type) else: field.type = map_type_from_sql(dtype) return field diff --git a/tests/fixtures/glue/datacontract.yaml b/tests/fixtures/glue/datacontract.yaml index 5475b55f..b12a6dba 100644 --- a/tests/fixtures/glue/datacontract.yaml +++ b/tests/fixtures/glue/datacontract.yaml @@ -43,6 +43,12 @@ models: type: string sub_field_four: type: integer + field_eight: + type: map + keys: + type: string + values: + type: integer part_one: description: Partition Key required: True diff --git a/tests/test_import_glue.py b/tests/test_import_glue.py index a66e3ab4..65a035e4 100644 --- a/tests/test_import_glue.py +++ b/tests/test_import_glue.py @@ -65,6 +65,10 @@ def setup_mock_glue(aws_credentials): "Name": "field_seven", "Type": "array>", }, + { + "Name": "field_eight", + "Type": "map", + }, ] }, "PartitionKeys": [