From 44f79e4f7898197bd1b3302d573484fa7c75a20c Mon Sep 17 00:00:00 2001 From: Petra Rebernjak Date: Tue, 30 Aug 2022 18:45:01 +0200 Subject: [PATCH 1/2] Make sure data types are strings Signed-off-by: Petra Rebernjak --- .../contrib/spark_offline_store/spark_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index 5b9f562181..cc815c3793 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -160,8 +160,8 @@ def get_table_column_names_and_types( ) df = spark_session.sql(f"SELECT * FROM {self.get_table_query_string()}") return ( - (fields["name"], fields["type"]) - for fields in df.schema.jsonValue()["fields"] + (field.name, field.dataType.simpleString()) + for field in df.schema ) def get_table_query_string(self) -> str: From b1164b23a36ac28933ac6a1cf20d50ee8d326b78 Mon Sep 17 00:00:00 2001 From: Petra Rebernjak Date: Tue, 30 Aug 2022 20:09:44 +0200 Subject: [PATCH 2/2] Lint Signed-off-by: Petra Rebernjak --- .../contrib/spark_offline_store/spark_source.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index cc815c3793..a27065fb5e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -159,10 +159,7 @@ def get_table_column_names_and_types( store_config=config.offline_store ) df = spark_session.sql(f"SELECT * FROM {self.get_table_query_string()}") - return ( - (field.name, field.dataType.simpleString()) - for field in df.schema - ) + return ((field.name, field.dataType.simpleString()) for field in df.schema) def get_table_query_string(self) -> str: """Returns a string that can directly be used to reference this table in SQL"""