return single output path, remove dependency on region

Signed-off-by: niklasvm <niklasvm@gmail.com>
feast-dev · Sep 3, 2022 · 8bd0b95 · 8bd0b95
1 parent 844bb83
commit 8bd0b95
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 19 deletions.
diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py
@@ -52,9 +52,6 @@ class SparkOfflineStoreConfig(FeastConfigBaseModel):
     staging_location: Optional[StrictStr] = None
     """ Remote path for batch materialization jobs"""
 
-    region: Optional[StrictStr] = None
-    """ AWS Region if applicable for s3-based staging locations"""
-
 
 class SparkOfflineStore(OfflineStore):
     @staticmethod
@@ -379,7 +376,8 @@ def to_remote_storage(self) -> List[str]:
                 )
                 sdf.write.parquet(output_uri)
 
-                return _list_files_in_folder(output_uri)
+                return [output_uri]
+
             elif self._config.offline_store.staging_location.startswith("s3://"):
 
                 spark_compatible_s3_staging_location = (
@@ -394,9 +392,7 @@ def to_remote_storage(self) -> List[str]:
                 )
                 sdf.write.parquet(output_uri)
 
-                return aws_utils.list_s3_files(
-                    self._config.offline_store.region, output_uri
-                )
+                return [output_uri]
 
             else:
                 raise NotImplementedError(
@@ -508,17 +504,6 @@ def _format_datetime(t: datetime) -> str:
     return dt
 
 
-def _list_files_in_folder(folder):
-    """List full filenames in a folder"""
-    files = []
-    for file in os.listdir(folder):
-        filename = os.path.join(folder, file)
-        if os.path.isfile(filename):
-            files.append(filename)
-
-    return files
-
-
 def _cast_data_frame(
     df_new: pyspark.sql.DataFrame, df_existing: pyspark.sql.DataFrame
 ) -> pyspark.sql.DataFrame:

diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py
@@ -61,7 +61,6 @@ def create_offline_store_config(self):
         self.spark_offline_store_config.staging_location = "file://" + str(
             tempfile.TemporaryDirectory()
         )
-        self.spark_offline_store_config.region = "eu-west-1"
         return self.spark_offline_store_config
 
     def create_data_source(