feat: Make bytewax job write as mini-batches (#3777)

Signed-off-by: Hai Nguyen <quanghai.ng1512@gmail.com> Signed-off-by: James Crabtree <james.crabtree@sailpoint.com>
feast-dev · Oct 23, 2023 · fabd417 · fabd417
1 parent c35b0c1
commit fabd417
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py
@@ -91,6 +91,9 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel):
     print_pod_logs_on_failure: bool = True
     """(optional) Print pod logs on job failure.  Only applies to synchronous materialization"""
 
+    mini_batch_size: int = 1000
+    """ (optional) Number of rows to process per write operation (default 1000)"""
+
 
 class BytewaxMaterializationEngine(BatchMaterializationEngine):
     def __init__(