diff --git a/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts b/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts index 88afa9859ed..1a379a2fffb 100644 --- a/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts +++ b/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts @@ -33,8 +33,11 @@ type ProvingJobWithResolvers = { id: string; request: T; signal?: AbortSignal; + retries: number; } & PromiseWithResolvers>; +const MAX_RETRIES = 3; + export class MemoryProvingQueue implements CircuitProver, ProvingJobSource { private jobId = 0; private log = createDebugLogger('aztec:prover-client:prover-pool:queue'); @@ -95,7 +98,18 @@ export class MemoryProvingQueue implements CircuitProver, ProvingJobSource { return Promise.resolve(); } - job.reject(err); + if (job.retries < MAX_RETRIES) { + job.retries++; + this.log.warn( + `Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}. Retry ${ + job.retries + }/${MAX_RETRIES}`, + ); + this.queue.put(job); + } else { + this.log.error(`Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}`); + job.reject(err); + } return Promise.resolve(); } @@ -111,13 +125,16 @@ export class MemoryProvingQueue implements CircuitProver, ProvingJobSource { promise, resolve, reject, + retries: 0, }; if (signal) { signal.addEventListener('abort', () => reject(new AbortedError('Operation has been aborted'))); } - this.log.info(`Adding ${ProvingRequestType[request.type]} proving job to queue`); + this.log.info( + `Adding id=${item.id} type=${ProvingRequestType[request.type]} proving job to queue depth=${this.queue.length()}`, + ); // TODO (alexg) remove the `any` if (!this.queue.put(item as any)) { throw new Error();