feat: add proving retries

yarn-project/prover-client/package.json

@@ -27,7 +27,7 @@
     "formatting": "run -T prettier --check ./src && run -T eslint ./src",
     "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
     "bb": "node --no-warnings ./dest/bb/index.js",
-    "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=600000 --forceExit"
+    "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=750000 --forceExit"
   },
   "jest": {
     "moduleNameMapper": {

yarn-project/prover-client/package.local.json

@@ -1,5 +1,5 @@
 {
   "scripts": {
-    "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=600000 --forceExit"
+    "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=750000 --forceExit"
   }
 }

yarn-project/prover-client/src/prover-pool/memory-proving-queue.test.ts

@@ -45,14 +45,27 @@ describe('MemoryProvingQueue', () => {
     await expect(promise).resolves.toEqual(new RootParityInput(proof, vk, publicInputs));
   });
 
-  it('notifies of errors', async () => {
+  it('retries failed jobs', async () => {
     const inputs = makeBaseParityInputs();
-    const promise = queue.getBaseParityProof(inputs);
+    void queue.getBaseParityProof(inputs);
+
     const job = await queue.getProvingJob();
     expect(job?.request.inputs).toEqual(inputs);
 
     const error = new Error('test error');
+
     await queue.rejectProvingJob(job!.id, error);
+    await expect(queue.getProvingJob()).resolves.toEqual(job);
+  });
+
+  it('notifies errors', async () => {
+    const promise = queue.getBaseParityProof(makeBaseParityInputs());
+
+    const error = new Error('test error');
+    await queue.rejectProvingJob((await queue.getProvingJob())!.id, error);
+    await queue.rejectProvingJob((await queue.getProvingJob())!.id, error);
+    await queue.rejectProvingJob((await queue.getProvingJob())!.id, error);
+
     await expect(promise).rejects.toEqual(error);
   });
 });

yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts

@@ -33,8 +33,11 @@ type ProvingJobWithResolvers<T extends ProvingRequest = ProvingRequest> = {
   id: string;
   request: T;
   signal?: AbortSignal;
+  attempts: number;
 } & PromiseWithResolvers<ProvingRequestResult<T['type']>>;
 
+const MAX_RETRIES = 3;
+
 export class MemoryProvingQueue implements CircuitProver, ProvingJobSource {
   private jobId = 0;
   private log = createDebugLogger('aztec:prover-client:prover-pool:queue');
@@ -95,7 +98,18 @@ export class MemoryProvingQueue implements CircuitProver, ProvingJobSource {
       return Promise.resolve();
     }
 
-    job.reject(err);
+    if (job.attempts < MAX_RETRIES) {
+      job.attempts++;
+      this.log.warn(
+        `Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}. Retry ${
+          job.attempts
+        }/${MAX_RETRIES}`,
+      );
+      this.queue.put(job);
+    } else {
+      this.log.error(`Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}`);
+      job.reject(err);
+    }
     return Promise.resolve();
   }
 
@@ -111,13 +125,16 @@ export class MemoryProvingQueue implements CircuitProver, ProvingJobSource {
       promise,
       resolve,
       reject,
+      attempts: 1,
     };
 
     if (signal) {
       signal.addEventListener('abort', () => reject(new AbortedError('Operation has been aborted')));
     }
 
-    this.log.info(`Adding ${ProvingRequestType[request.type]} proving job to queue`);
+    this.log.info(
+      `Adding id=${item.id} type=${ProvingRequestType[request.type]} proving job to queue depth=${this.queue.length()}`,
+    );
     // TODO (alexg) remove the `any`
     if (!this.queue.put(item as any)) {
       throw new Error();