fix: Spark kafka processor sorting (#3479)

Fix spark_kafka_processor sort Signed-off-by: shaurya.rawat <shaurya.rawat@new-work.se>
feast-dev · Feb 17, 2023 · f2cbf43 · f2cbf43
1 parent 4575ef0
commit f2cbf43
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/sdk/python/feast/infra/contrib/spark_kafka_processor.py b/sdk/python/feast/infra/contrib/spark_kafka_processor.py
@@ -131,7 +131,7 @@ def batch_write(row: DataFrame, batch_id: int):
             # Also add a 'created' column.
             rows = (
                 rows.sort_values(
-                    by=self.join_keys + [self.sfv.timestamp_field], ascending=True
+                    by=[*self.join_keys, self.sfv.timestamp_field], ascending=False
                 )
                 .groupby(self.join_keys)
                 .nth(0)