From 262ee2fd65d490b6922d41c1ef44a2330802ad20 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 2 May 2024 06:07:17 +0000 Subject: [PATCH 1/3] feat: add proving retries --- yarn-project/prover-client/package.json | 2 +- yarn-project/prover-client/package.local.json | 2 +- .../prover-pool/memory-proving-queue.test.ts | 17 +++++++++++++-- .../src/prover-pool/memory-proving-queue.ts | 21 +++++++++++++++++-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 3bdf4749602..fff69ca9878 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -27,7 +27,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "bb": "node --no-warnings ./dest/bb/index.js", - "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=600000 --forceExit" + "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=750000 --forceExit" }, "jest": { "moduleNameMapper": { diff --git a/yarn-project/prover-client/package.local.json b/yarn-project/prover-client/package.local.json index 5eeed2af06a..04b562bf213 100644 --- a/yarn-project/prover-client/package.local.json +++ b/yarn-project/prover-client/package.local.json @@ -1,5 +1,5 @@ { "scripts": { - "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=600000 --forceExit" + "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=750000 --forceExit" } } diff --git a/yarn-project/prover-client/src/prover-pool/memory-proving-queue.test.ts b/yarn-project/prover-client/src/prover-pool/memory-proving-queue.test.ts index b9520f333a5..dc3d77ab7dc 100644 --- a/yarn-project/prover-client/src/prover-pool/memory-proving-queue.test.ts +++ b/yarn-project/prover-client/src/prover-pool/memory-proving-queue.test.ts @@ -45,14 +45,27 @@ describe('MemoryProvingQueue', () => { await expect(promise).resolves.toEqual(new RootParityInput(proof, vk, publicInputs)); }); - it('notifies of errors', async () => { + it('retries failed jobs', async () => { const inputs = makeBaseParityInputs(); - const promise = queue.getBaseParityProof(inputs); + void queue.getBaseParityProof(inputs); + const job = await queue.getProvingJob(); expect(job?.request.inputs).toEqual(inputs); const error = new Error('test error'); + await queue.rejectProvingJob(job!.id, error); + await expect(queue.getProvingJob()).resolves.toEqual(job); + }); + + it('notifies errors', async () => { + const promise = queue.getBaseParityProof(makeBaseParityInputs()); + + const error = new Error('test error'); + await queue.rejectProvingJob((await queue.getProvingJob())!.id, error); + await queue.rejectProvingJob((await queue.getProvingJob())!.id, error); + await queue.rejectProvingJob((await queue.getProvingJob())!.id, error); + await expect(promise).rejects.toEqual(error); }); }); diff --git a/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts b/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts index 88afa9859ed..c3e5194f027 100644 --- a/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts +++ b/yarn-project/prover-client/src/prover-pool/memory-proving-queue.ts @@ -33,8 +33,11 @@ type ProvingJobWithResolvers = { id: string; request: T; signal?: AbortSignal; + attempts: number; } & PromiseWithResolvers>; +const MAX_RETRIES = 3; + export class MemoryProvingQueue implements CircuitProver, ProvingJobSource { private jobId = 0; private log = createDebugLogger('aztec:prover-client:prover-pool:queue'); @@ -95,7 +98,18 @@ export class MemoryProvingQueue implements CircuitProver, ProvingJobSource { return Promise.resolve(); } - job.reject(err); + if (job.attempts < MAX_RETRIES) { + job.attempts++; + this.log.warn( + `Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}. Retry ${ + job.attempts + }/${MAX_RETRIES}`, + ); + this.queue.put(job); + } else { + this.log.error(`Job id=${job.id} type=${ProvingRequestType[job.request.type]} failed with error: ${err}`); + job.reject(err); + } return Promise.resolve(); } @@ -111,13 +125,16 @@ export class MemoryProvingQueue implements CircuitProver, ProvingJobSource { promise, resolve, reject, + attempts: 1, }; if (signal) { signal.addEventListener('abort', () => reject(new AbortedError('Operation has been aborted'))); } - this.log.info(`Adding ${ProvingRequestType[request.type]} proving job to queue`); + this.log.info( + `Adding id=${item.id} type=${ProvingRequestType[request.type]} proving job to queue depth=${this.queue.length()}`, + ); // TODO (alexg) remove the `any` if (!this.queue.put(item as any)) { throw new Error(); From 37edb1d3c94b79caf118ff3911053ff3f65af849 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 2 May 2024 09:15:33 +0000 Subject: [PATCH 2/3] fix: increase default number of proving agents --- yarn-project/prover-client/src/config.ts | 2 +- yarn-project/prover-client/src/prover-pool/prover-pool.ts | 3 +++ yarn-project/prover-client/src/tx-prover/tx-prover.ts | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/yarn-project/prover-client/src/config.ts b/yarn-project/prover-client/src/config.ts index 0b8f7cce6ea..07911a9790c 100644 --- a/yarn-project/prover-client/src/config.ts +++ b/yarn-project/prover-client/src/config.ts @@ -29,7 +29,7 @@ export function getProverEnvVars(): ProverConfig { ACVM_BINARY_PATH = '', BB_WORKING_DIRECTORY = tmpdir(), BB_BINARY_PATH = '', - PROVER_AGENTS = '1', + PROVER_AGENTS = '10', PROVER_REAL_PROOFS = '', } = process.env; diff --git a/yarn-project/prover-client/src/prover-pool/prover-pool.ts b/yarn-project/prover-client/src/prover-pool/prover-pool.ts index 0750b41520e..4916e3e98a4 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-pool.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-pool.ts @@ -1,4 +1,5 @@ import { type ProvingJobSource } from '@aztec/circuit-types'; +import { sleep } from '@aztec/foundation/sleep'; import { type SimulationProvider } from '@aztec/simulator'; import { mkdtemp } from 'fs/promises'; @@ -33,6 +34,8 @@ export class ProverPool { for (const agent of this.agents) { agent.start(source); + // stagger that start of each agent to avoid contention + await sleep(10); } } diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index e5b08a74560..88f987e8124 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -68,9 +68,9 @@ export class TxProver implements ProverClient { throw new Error(); } - pool = ProverPool.nativePool(config, config.proverAgents, 10); + pool = ProverPool.nativePool(config, config.proverAgents, 50); } else { - pool = ProverPool.testPool(simulationProvider, config.proverAgents, 10); + pool = ProverPool.testPool(simulationProvider, config.proverAgents, 50); } const prover = new TxProver(worldStateSynchronizer, getVerificationKeys(), pool); From 4038202e9a06aadd2dfa642148408c40bb5b14a6 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 2 May 2024 09:15:33 +0000 Subject: [PATCH 3/3] fix: increase default number of proving agents --- yarn-project/end-to-end/Earthfile | 2 ++ yarn-project/prover-client/src/config.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 3a9c18ff04b..49dbde955ce 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -6,9 +6,11 @@ E2E_COMPOSE_TEST: ARG compose_file=./scripts/docker-compose.yml ARG debug="aztec:*" ARG EARTHLY_TARGET_NAME + ARG prover_agents=10 LOCALLY ENV TEST=$test ENV DEBUG=$debug + ENV PROVER_AGENTS=$prover_agents LET project_name=$(echo $test | sed 's/\./_/g') IF docker compose > /dev/null 2>&1 LET CMD="docker compose" diff --git a/yarn-project/prover-client/src/config.ts b/yarn-project/prover-client/src/config.ts index 07911a9790c..0b8f7cce6ea 100644 --- a/yarn-project/prover-client/src/config.ts +++ b/yarn-project/prover-client/src/config.ts @@ -29,7 +29,7 @@ export function getProverEnvVars(): ProverConfig { ACVM_BINARY_PATH = '', BB_WORKING_DIRECTORY = tmpdir(), BB_BINARY_PATH = '', - PROVER_AGENTS = '10', + PROVER_AGENTS = '1', PROVER_REAL_PROOFS = '', } = process.env;