add resumable replay verify if a partition failed

fEst1ck · Oct 15, 2023 · eae267c · eae267c
1 parent 341087a
commit eae267c
Showing 1 changed file with 15 additions and 1 deletion.
diff --git a/testsuite/replay_verify.py b/testsuite/replay_verify.py
@@ -70,6 +70,20 @@
 ]
 
 
+def replay_with_retry(*args):
+    MAX_RETRIES = 3
+    attempt = 0
+    while attempt < MAX_RETRIES:
+        (n, return_code) = replay_verify_partition(*args)
+        if return_code == 0:
+            return (n, return_code)
+        elif attempt < MAX_RETRIES - 1:
+            print(f"Attempt {attempt + 1} failed for arguments {args}. Retrying.")
+            attempt += 1
+        else:
+            return (n, return_code)
+
+
 def replay_verify_partition(
     n: int,
     N: int,
@@ -201,7 +215,7 @@ def main():
 
     with Pool(N) as p:
         all_partitions = p.starmap(
-            replay_verify_partition,
+            replay_with_retry,
             [
                 (
                     n,