From a7d0591248cb2fcb2c94ad1d5cb79e9e6a1e1acd Mon Sep 17 00:00:00 2001 From: ilitteri Date: Thu, 25 Apr 2024 18:40:19 -0300 Subject: [PATCH 01/23] Support delete cmd --- prover/prover_cli/src/cli.rs | 4 +++- prover/prover_cli/src/commands/mod.rs | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/prover/prover_cli/src/cli.rs b/prover/prover_cli/src/cli.rs index 844387f983e..5a1512fb8e4 100644 --- a/prover/prover_cli/src/cli.rs +++ b/prover/prover_cli/src/cli.rs @@ -1,6 +1,6 @@ use clap::{command, Parser, Subcommand}; -use crate::commands::get_file_info; +use crate::commands::{delete, get_file_info}; pub const VERSION_STRING: &str = env!("CARGO_PKG_VERSION"); @@ -14,12 +14,14 @@ struct ProverCLI { #[derive(Subcommand)] enum ProverCommand { FileInfo(get_file_info::Args), + Delete(delete::Args), } pub async fn start() -> anyhow::Result<()> { let ProverCLI { command } = ProverCLI::parse(); match command { ProverCommand::FileInfo(args) => get_file_info::run(args).await?, + ProverCommand::Delete(args) => delete::run(args).await?, }; Ok(()) diff --git a/prover/prover_cli/src/commands/mod.rs b/prover/prover_cli/src/commands/mod.rs index 3e9a45cb72a..6c2b220a38c 100644 --- a/prover/prover_cli/src/commands/mod.rs +++ b/prover/prover_cli/src/commands/mod.rs @@ -1 +1,2 @@ +pub(crate) mod delete; pub(crate) mod get_file_info; From e1ca28a4a5c073a735f9817cd88a1c89ae4d9ca6 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Thu, 25 Apr 2024 18:40:37 -0300 Subject: [PATCH 02/23] Add delete cmd implementation --- prover/prover_cli/src/commands/delete.rs | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 prover/prover_cli/src/commands/delete.rs diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs new file mode 100644 index 00000000000..8780d3d46e5 --- /dev/null +++ b/prover/prover_cli/src/commands/delete.rs @@ -0,0 +1,56 @@ +use anyhow::Context; +use clap::Args as ClapArgs; +use prover_dal::{Connection, ConnectionPool, Prover, ProverDal}; +use zksync_config::PostgresConfig; +use zksync_env_config::FromEnv; +use zksync_types::L1BatchNumber; + +#[derive(ClapArgs)] +pub(crate) struct Args { + #[clap(short, long, conflicts_with = "all")] + batch: L1BatchNumber, + #[clap(short, long, conflicts_with = "batch", default_value_t = false)] + all: bool, + #[clap(short, long, default_value_t = false)] + failed: bool, +} + +pub(crate) async fn run(args: Args) -> anyhow::Result<()> { + let config = PostgresConfig::from_env()?; + let prover_connection_pool = + ConnectionPool::::builder(config.prover_url()?, config.max_connections()?) + .build() + .await + .context("failed to build a prover_connection_pool")?; + let conn = prover_connection_pool.connection().await.unwrap(); + + if args.all { + delete_prover_db(conn).await?; + } else { + delete_batch_data(conn, args.batch, args.failed).await?; + } + + Ok(()) +} + +// This is a placeholder function to delete the batch data from the prover database. +async fn delete_prover_db(_conn: Connection<'_, Prover>) -> anyhow::Result<()> { + Ok(()) +} + +async fn delete_batch_data( + mut conn: Connection<'_, Prover>, + block_number: L1BatchNumber, + _failed: bool, +) -> anyhow::Result<()> { + conn.fri_proof_compressor_dal() + .delete_batch_data(block_number) + .await; + conn.fri_prover_jobs_dal() + .delete_batch_data(block_number) + .await; + conn.fri_witness_generator_dal() + .delete_batch_data(block_number) + .await; + Ok(()) +} From d026ca2ee7d2618f6a3a97f16abe24896a0d2a67 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Thu, 25 Apr 2024 18:41:09 -0300 Subject: [PATCH 03/23] Implement queries for deleting batch data --- ...7249ec09c0daf4368021788207370213a6d94.json | 14 ++++ ...60846e2e2e9c131c5857f977500e012f7141b.json | 14 ++++ ...7b9f7559f338f81190e03a026f64fe5db7cbe.json | 14 ++++ ...ea52faf4df7023b541447a489cf0db5d192de.json | 14 ++++ ...356f9e77694131bd093163feb4f4a513de9d0.json | 14 ++++ ...3a5e612d786b8e236752870cf870109effe03.json | 14 ++++ ...03cf3909e060d81fbe9d3f12afd90c4ae9118.json | 14 ++++ .../src/fri_proof_compressor_dal.rs | 14 ++++ prover/prover_dal/src/fri_prover_dal.rs | 42 ++++++++++++ .../src/fri_witness_generator_dal.rs | 67 +++++++++++++++++++ 10 files changed, 221 insertions(+) create mode 100644 prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json create mode 100644 prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json create mode 100644 prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json create mode 100644 prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json create mode 100644 prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json create mode 100644 prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json create mode 100644 prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json diff --git a/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json b/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json new file mode 100644 index 00000000000..967f02586d7 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM proof_compression_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94" +} diff --git a/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json b/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json new file mode 100644 index 00000000000..fbc8db58e91 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n prover_jobs_fri\n WHERE\n l1_batch_number = $1;\n \n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b" +} diff --git a/prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json b/prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json new file mode 100644 index 00000000000..bd350a79cbe --- /dev/null +++ b/prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n witness_inputs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe" +} diff --git a/prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json b/prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json new file mode 100644 index 00000000000..6ebc4fdc65b --- /dev/null +++ b/prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n leaf_aggregation_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de" +} diff --git a/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json b/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json new file mode 100644 index 00000000000..0ea0b74e3f9 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n prover_jobs_fri_archive\n WHERE\n l1_batch_number = $1;\n \n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0" +} diff --git a/prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json b/prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json new file mode 100644 index 00000000000..eac31abb40e --- /dev/null +++ b/prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n node_aggregation_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03" +} diff --git a/prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json b/prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json new file mode 100644 index 00000000000..b8d9e7afa3c --- /dev/null +++ b/prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n scheduler_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118" +} diff --git a/prover/prover_dal/src/fri_proof_compressor_dal.rs b/prover/prover_dal/src/fri_proof_compressor_dal.rs index 01231d33b00..8091799ee0b 100644 --- a/prover/prover_dal/src/fri_proof_compressor_dal.rs +++ b/prover/prover_dal/src/fri_proof_compressor_dal.rs @@ -328,4 +328,18 @@ impl FriProofCompressorDal<'_, '_> { .collect() } } + + pub async fn delete_batch_data(&mut self, block_number: L1BatchNumber) { + sqlx::query!( + r#" + DELETE FROM proof_compression_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(block_number.0) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } } diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index 2da9f69d311..bb36a67214b 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -594,4 +594,46 @@ impl FriProverDal<'_, '_> { .unwrap() .unwrap_or(0) as usize } + + pub async fn delete_prover_jobs_fri_batch_data(&mut self, l1_batch_number: L1BatchNumber) { + sqlx::query!( + r#" + DELETE FROM + prover_jobs_fri + WHERE + l1_batch_number = $1; + + "#, + i64::from(l1_batch_number.0) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_prover_jobs_fri_archive_batch_data( + &mut self, + l1_batch_number: L1BatchNumber, + ) { + sqlx::query!( + r#" + DELETE FROM + prover_jobs_fri_archive + WHERE + l1_batch_number = $1; + + "#, + i64::from(l1_batch_number.0) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_batch_data(&mut self, l1_batch_number: L1BatchNumber) { + self.delete_prover_jobs_fri_batch_data(l1_batch_number) + .await; + self.delete_prover_jobs_fri_archive_batch_data(l1_batch_number) + .await; + } } diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index d2b58f5f75d..efc62eee411 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1151,4 +1151,71 @@ impl FriWitnessGeneratorDal<'_, '_> { .map(|id| ProtocolVersionId::try_from(id as u16).unwrap()) .unwrap() } + + pub async fn delete_witness_inputs_batch_data(&mut self, block_number: L1BatchNumber) { + sqlx::query!( + r#" + DELETE FROM + witness_inputs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(block_number.0) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_leaf_aggregation_batch_data(&mut self, block_number: L1BatchNumber) { + sqlx::query!( + r#" + DELETE FROM + leaf_aggregation_witness_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(block_number.0) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_node_aggregation_batch_data(&mut self, block_number: L1BatchNumber) { + sqlx::query!( + r#" + DELETE FROM + node_aggregation_witness_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(block_number.0) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_scheduler_batch_data(&mut self, block_number: L1BatchNumber) { + sqlx::query!( + r#" + DELETE FROM + scheduler_witness_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(block_number.0) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_batch_data(&mut self, block_number: L1BatchNumber) { + self.delete_witness_inputs_batch_data(block_number).await; + self.delete_leaf_aggregation_batch_data(block_number).await; + self.delete_node_aggregation_batch_data(block_number).await; + self.delete_scheduler_batch_data(block_number).await; + } } From 914e13d59093ac7608442dd45ba268faaaedda75 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 29 Apr 2024 17:49:36 -0300 Subject: [PATCH 04/23] Add queries for deleting the data --- ...d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json | 12 ++++++++++++ ...f59ce7119b39f44f231ca4e4f055a6491355070ff724.json | 12 ++++++++++++ ...571961b9e1502048d058e61099cc5990d7b72a537a07.json | 12 ++++++++++++ ...0b7be9763bb1c30ebeeec30089476d11ac957209398c.json | 12 ++++++++++++ ...4b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json | 12 ++++++++++++ ...d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json | 12 ++++++++++++ prover/prover_dal/src/fri_gpu_prover_queue_dal.rs | 7 +++++++ prover/prover_dal/src/fri_proof_compressor_dal.rs | 7 +++++++ prover/prover_dal/src/fri_protocol_versions_dal.rs | 7 +++++++ prover/prover_dal/src/fri_prover_dal.rs | 7 +++++++ .../src/fri_scheduler_dependency_tracker_dal.rs | 7 +++++++ prover/prover_dal/src/fri_witness_generator_dal.rs | 7 +++++++ 12 files changed, 114 insertions(+) create mode 100644 prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json create mode 100644 prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json create mode 100644 prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json create mode 100644 prover/prover_dal/.sqlx/query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json create mode 100644 prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json create mode 100644 prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json diff --git a/prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json b/prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json new file mode 100644 index 00000000000..95ee1eced8f --- /dev/null +++ b/prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "TRUNCATE TABLE prover_fri_protocol_versions", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6" +} diff --git a/prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json b/prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json new file mode 100644 index 00000000000..0f1a668a6ee --- /dev/null +++ b/prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "TRUNCATE TABLE gpu_prover_queue_fri, gpu_prover_queue_fri_archive", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724" +} diff --git a/prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json b/prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json new file mode 100644 index 00000000000..31f4e34def3 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "TRUNCATE TABLE scheduler_dependency_tracker_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07" +} diff --git a/prover/prover_dal/.sqlx/query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json b/prover/prover_dal/.sqlx/query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json new file mode 100644 index 00000000000..b9191435bac --- /dev/null +++ b/prover/prover_dal/.sqlx/query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "TRUNCATE TABLE proof_compression_jobs_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c" +} diff --git a/prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json b/prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json new file mode 100644 index 00000000000..6398b3e0822 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "TRUNCATE TABLE prover_jobs_fri, prover_jobs_fri_archive", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09" +} diff --git a/prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json b/prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json new file mode 100644 index 00000000000..9e4ac4d8f94 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "TRUNCATE TABLE witness_inputs_fri, leaf_aggregation_witness_jobs_fri, node_aggregation_witness_jobs_fri, scheduler_witness_jobs_fri, scheduler_dependency_tracker_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9" +} diff --git a/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs b/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs index 731d74df2ca..ddc7a9aefdb 100644 --- a/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs +++ b/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs @@ -209,4 +209,11 @@ impl FriGpuProverQueueDal<'_, '_> { .unwrap() .unwrap_or(0) as usize } + + pub async fn delete_all(&mut self) { + sqlx::query!("TRUNCATE TABLE gpu_prover_queue_fri, gpu_prover_queue_fri_archive") + .execute(self.storage.conn()) + .await + .unwrap(); + } } diff --git a/prover/prover_dal/src/fri_proof_compressor_dal.rs b/prover/prover_dal/src/fri_proof_compressor_dal.rs index 8091799ee0b..1c6ee3a411b 100644 --- a/prover/prover_dal/src/fri_proof_compressor_dal.rs +++ b/prover/prover_dal/src/fri_proof_compressor_dal.rs @@ -342,4 +342,11 @@ impl FriProofCompressorDal<'_, '_> { .await .unwrap(); } + + pub async fn delete_all(&mut self) { + sqlx::query!("TRUNCATE TABLE proof_compression_jobs_fri") + .execute(self.storage.conn()) + .await + .unwrap(); + } } diff --git a/prover/prover_dal/src/fri_protocol_versions_dal.rs b/prover/prover_dal/src/fri_protocol_versions_dal.rs index 3d9b0b783ee..7a96623305a 100644 --- a/prover/prover_dal/src/fri_protocol_versions_dal.rs +++ b/prover/prover_dal/src/fri_protocol_versions_dal.rs @@ -90,4 +90,11 @@ impl FriProtocolVersionsDal<'_, '_> { .map(|row| ProtocolVersionId::try_from(row.id as u16).unwrap()) .collect() } + + pub async fn delete_all(&mut self) { + sqlx::query!("TRUNCATE TABLE prover_fri_protocol_versions") + .execute(self.storage.conn()) + .await + .unwrap(); + } } diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index bb36a67214b..ed53aacd67a 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -636,4 +636,11 @@ impl FriProverDal<'_, '_> { self.delete_prover_jobs_fri_archive_batch_data(l1_batch_number) .await; } + + pub async fn delete_all(&mut self) { + sqlx::query!("TRUNCATE TABLE prover_jobs_fri, prover_jobs_fri_archive") + .execute(self.storage.conn()) + .await + .unwrap(); + } } diff --git a/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs b/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs index 5e47fb1ec06..5b8ef40ea0d 100644 --- a/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs +++ b/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs @@ -149,4 +149,11 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> { }) .unwrap() } + + pub async fn delete_all(&mut self) { + sqlx::query!("TRUNCATE TABLE scheduler_dependency_tracker_fri") + .execute(self.storage.conn()) + .await + .unwrap(); + } } diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index efc62eee411..968002d63c1 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1218,4 +1218,11 @@ impl FriWitnessGeneratorDal<'_, '_> { self.delete_node_aggregation_batch_data(block_number).await; self.delete_scheduler_batch_data(block_number).await; } + + pub async fn delete_all(&mut self) { + sqlx::query!("TRUNCATE TABLE witness_inputs_fri, leaf_aggregation_witness_jobs_fri, node_aggregation_witness_jobs_fri, scheduler_witness_jobs_fri, scheduler_dependency_tracker_fri") + .execute(self.storage.conn()) + .await + .unwrap(); + } } From 68fa9d1b05f902dbd49a336895033012eb2d023c Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 29 Apr 2024 17:49:59 -0300 Subject: [PATCH 05/23] Implement `delete_prover_db` --- prover/prover_cli/src/commands/delete.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs index 8780d3d46e5..df3341d2b28 100644 --- a/prover/prover_cli/src/commands/delete.rs +++ b/prover/prover_cli/src/commands/delete.rs @@ -33,8 +33,15 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { Ok(()) } -// This is a placeholder function to delete the batch data from the prover database. -async fn delete_prover_db(_conn: Connection<'_, Prover>) -> anyhow::Result<()> { +async fn delete_prover_db(mut conn: Connection<'_, Prover>) -> anyhow::Result<()> { + conn.fri_gpu_prover_queue_dal().delete_all().await; + conn.fri_prover_jobs_dal().delete_all().await; + conn.fri_scheduler_dependency_tracker_dal() + .delete_all() + .await; + conn.fri_protocol_versions_dal().delete_all().await; + conn.fri_proof_compressor_dal().delete_all().await; + conn.fri_witness_generator_dal().delete_all().await; Ok(()) } From 8d48303ce25329d98e54f03ebca7fd09c3c0f9eb Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 29 Apr 2024 17:50:20 -0300 Subject: [PATCH 06/23] Add prompt for user to confirm the action --- prover/Cargo.lock | 39 ++++++++++++++++++++++++ prover/prover_cli/Cargo.toml | 2 +- prover/prover_cli/src/commands/delete.rs | 12 ++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 268dab9e526..27b432104b1 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -1082,6 +1082,19 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bed69047ed42e52c7e38d6421eeb8ceefb4f2a2b52eed59137f7bad7908f6800" +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "const-decoder" version = "0.3.0" @@ -1597,6 +1610,19 @@ dependencies = [ "unicode-xid 0.2.4", ] +[[package]] +name = "dialoguer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" +dependencies = [ + "console", + "shell-words", + "tempfile", + "thiserror", + "zeroize", +] + [[package]] name = "digest" version = "0.9.0" @@ -1739,6 +1765,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -4359,6 +4391,7 @@ dependencies = [ "bincode", "clap 4.4.6", "colored", + "dialoguer", "env_logger 0.10.2", "hex", "log", @@ -5384,6 +5417,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "shivini" version = "0.2.0" diff --git a/prover/prover_cli/Cargo.toml b/prover/prover_cli/Cargo.toml index 994a8f93a84..b006fe002fc 100644 --- a/prover/prover_cli/Cargo.toml +++ b/prover/prover_cli/Cargo.toml @@ -13,6 +13,7 @@ categories.workspace = true tokio = { version = "1", features = ["rt-multi-thread", "macros"] } env_logger = "0.10" log = "0.4" +dialoguer = "0.11" clap = { workspace = true, features = ["derive"] } tracing.workspace = true @@ -29,4 +30,3 @@ zksync_types.workspace = true zksync_prover_fri_types.workspace = true zksync_prover_interface.workspace = true prover_dal.workspace = true - diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs index df3341d2b28..8de81d32a4d 100644 --- a/prover/prover_cli/src/commands/delete.rs +++ b/prover/prover_cli/src/commands/delete.rs @@ -1,5 +1,6 @@ use anyhow::Context; use clap::Args as ClapArgs; +use dialoguer::{theme::ColorfulTheme, Input}; use prover_dal::{Connection, ConnectionPool, Prover, ProverDal}; use zksync_config::PostgresConfig; use zksync_env_config::FromEnv; @@ -16,6 +17,17 @@ pub(crate) struct Args { } pub(crate) async fn run(args: Args) -> anyhow::Result<()> { + let confirmation = Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Are you sure you want to delete the data?") + .default("yes".to_owned()) + .interact_text() + .unwrap(); + + if confirmation != "yes" { + println!("Aborted"); + return Ok(()); + } + let config = PostgresConfig::from_env()?; let prover_connection_pool = ConnectionPool::::builder(config.prover_url()?, config.max_connections()?) From d849eb90a25e6f646bf64e70eb6cc5c04eb69f2e Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 29 Apr 2024 18:29:47 -0300 Subject: [PATCH 07/23] Fix queries --- ...5129da7699cac6abe80533b6a8c848eaf2673.json | 12 +++++++ ...7fd298947384bcb1f4c9e479ea11fe21c3dc.json} | 4 +-- ...a750f999f2371e437b2b5042dfa9d11a28aa6.json | 12 ------- ...14cc001ab8f927536c54ca8624fcf53c1d630.json | 12 +++++++ ...3fb71f93ba8a66f17a5bce1994345d78994ef.json | 12 +++++++ ...69c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json | 12 +++++++ ...19b39f44f231ca4e4f055a6491355070ff724.json | 12 ------- ...155a63bf176555bf3da67a984eeb36def1e24.json | 12 +++++++ ...d63d68859bc719313a30fbd399f1009a07fa3.json | 12 +++++++ ...0ab4f3a9e187e78b734a7de4daff996fdd8af.json | 12 +++++++ ...49fa5bc3370cc3ee76c37aab325132f55e5b9.json | 12 +++++++ ...d6c3b3abeee23bd6099111901547cffccacdc.json | 12 +++++++ ...9e1502048d058e61099cc5990d7b72a537a07.json | 12 ------- ...b5004ff11de1aae6d9639366dde9b7b42d800.json | 12 +++++++ ...5cbfdb1f9f0259996948e02040e9175af939a.json | 12 +++++++ ...c9b2bb64b67d0273355c0847e2d99b1cb7d09.json | 12 ------- ...22e807dcfd1ff4cda8b59ddedb5c44ee34df6.json | 12 +++++++ ...a3c0d59703299479d3626b22ab648d675ce40.json | 12 +++++++ ...5cc9feb543e2ab4d91f27559d844280337ea9.json | 12 ------- ...c30ee2cd1c87236ac2cdf1602bf5f479e2fae.json | 12 +++++++ ...d28dc33bbed5f85720d0125b0a6ce65526d67.json | 12 +++++++ ...d375fd4baf93f109270cbab3ee4e61cfb2c67.json | 12 +++++++ .../src/fri_gpu_prover_queue_dal.rs | 16 ++++++++-- .../src/fri_proof_compressor_dal.rs | 4 +-- .../src/fri_protocol_versions_dal.rs | 4 +-- prover/prover_dal/src/fri_prover_dal.rs | 16 ++++++++-- .../fri_scheduler_dependency_tracker_dal.rs | 4 +-- .../src/fri_witness_generator_dal.rs | 32 +++++++++++++++++-- 28 files changed, 258 insertions(+), 74 deletions(-) create mode 100644 prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json rename prover/prover_dal/.sqlx/{query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json => query-0aad3107c7bc18eba553dc9d2aa87fd298947384bcb1f4c9e479ea11fe21c3dc.json} (50%) delete mode 100644 prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json create mode 100644 prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json create mode 100644 prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json create mode 100644 prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json delete mode 100644 prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json create mode 100644 prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json create mode 100644 prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json create mode 100644 prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json create mode 100644 prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json create mode 100644 prover/prover_dal/.sqlx/query-7ac4fe1dcf659f448828e9352cfd6c3b3abeee23bd6099111901547cffccacdc.json delete mode 100644 prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json create mode 100644 prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json create mode 100644 prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json delete mode 100644 prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json create mode 100644 prover/prover_dal/.sqlx/query-a91c0489a8830a3dbd628c75aca22e807dcfd1ff4cda8b59ddedb5c44ee34df6.json create mode 100644 prover/prover_dal/.sqlx/query-aff36fccb6408f736085c64eefda3c0d59703299479d3626b22ab648d675ce40.json delete mode 100644 prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json create mode 100644 prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json create mode 100644 prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json create mode 100644 prover/prover_dal/.sqlx/query-fc15423c4eef939d0b47f1cf068d375fd4baf93f109270cbab3ee4e61cfb2c67.json diff --git a/prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json b/prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json new file mode 100644 index 00000000000..1eecb02f2bd --- /dev/null +++ b/prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM leaf_aggregation_witness_jobs_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673" +} diff --git a/prover/prover_dal/.sqlx/query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json b/prover/prover_dal/.sqlx/query-0aad3107c7bc18eba553dc9d2aa87fd298947384bcb1f4c9e479ea11fe21c3dc.json similarity index 50% rename from prover/prover_dal/.sqlx/query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json rename to prover/prover_dal/.sqlx/query-0aad3107c7bc18eba553dc9d2aa87fd298947384bcb1f4c9e479ea11fe21c3dc.json index b9191435bac..c40cd0343af 100644 --- a/prover/prover_dal/.sqlx/query-947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c.json +++ b/prover/prover_dal/.sqlx/query-0aad3107c7bc18eba553dc9d2aa87fd298947384bcb1f4c9e479ea11fe21c3dc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "TRUNCATE TABLE proof_compression_jobs_fri", + "query": "DELETE FROM gpu_prover_queue_fri", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "947aa12dde06dbc303d90b7be9763bb1c30ebeeec30089476d11ac957209398c" + "hash": "0aad3107c7bc18eba553dc9d2aa87fd298947384bcb1f4c9e479ea11fe21c3dc" } diff --git a/prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json b/prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json deleted file mode 100644 index 95ee1eced8f..00000000000 --- a/prover/prover_dal/.sqlx/query-0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "TRUNCATE TABLE prover_fri_protocol_versions", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "0b67b0551b13a8d61814d9921d7a750f999f2371e437b2b5042dfa9d11a28aa6" -} diff --git a/prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json b/prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json new file mode 100644 index 00000000000..365a03f7793 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM prover_jobs_fri_archive_id_seq", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630" +} diff --git a/prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json b/prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json new file mode 100644 index 00000000000..94d26c79d73 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM node_aggregation_witness_jobs_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef" +} diff --git a/prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json b/prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json new file mode 100644 index 00000000000..01bab2bcb17 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM leaf_aggregation_witness_jobs_fri_id_seq", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5" +} diff --git a/prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json b/prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json deleted file mode 100644 index 0f1a668a6ee..00000000000 --- a/prover/prover_dal/.sqlx/query-462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "TRUNCATE TABLE gpu_prover_queue_fri, gpu_prover_queue_fri_archive", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "462e0028e7c5554676a7f59ce7119b39f44f231ca4e4f055a6491355070ff724" -} diff --git a/prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json b/prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json new file mode 100644 index 00000000000..88f2f8c563a --- /dev/null +++ b/prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM witness_inputs_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24" +} diff --git a/prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json b/prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json new file mode 100644 index 00000000000..a12f66f5b08 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM gpu_prover_queue_fri_id_seq", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3" +} diff --git a/prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json b/prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json new file mode 100644 index 00000000000..786e7af0d10 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM prover_jobs_fri_id_seq", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af" +} diff --git a/prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json b/prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json new file mode 100644 index 00000000000..0581cd54c63 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM node_aggregation_witness_jobs_fri_id_seq", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9" +} diff --git a/prover/prover_dal/.sqlx/query-7ac4fe1dcf659f448828e9352cfd6c3b3abeee23bd6099111901547cffccacdc.json b/prover/prover_dal/.sqlx/query-7ac4fe1dcf659f448828e9352cfd6c3b3abeee23bd6099111901547cffccacdc.json new file mode 100644 index 00000000000..975f24fddc8 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-7ac4fe1dcf659f448828e9352cfd6c3b3abeee23bd6099111901547cffccacdc.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM prover_jobs_fri_archive", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "7ac4fe1dcf659f448828e9352cfd6c3b3abeee23bd6099111901547cffccacdc" +} diff --git a/prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json b/prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json deleted file mode 100644 index 31f4e34def3..00000000000 --- a/prover/prover_dal/.sqlx/query-8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "TRUNCATE TABLE scheduler_dependency_tracker_fri", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "8b27384b4f89c9d4d735571961b9e1502048d058e61099cc5990d7b72a537a07" -} diff --git a/prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json b/prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json new file mode 100644 index 00000000000..1efcca428cf --- /dev/null +++ b/prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM scheduler_witness_jobs_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800" +} diff --git a/prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json b/prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json new file mode 100644 index 00000000000..fae2e71cf92 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM scheduler_dependency_tracker_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a" +} diff --git a/prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json b/prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json deleted file mode 100644 index 6398b3e0822..00000000000 --- a/prover/prover_dal/.sqlx/query-a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "TRUNCATE TABLE prover_jobs_fri, prover_jobs_fri_archive", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "a751f5e9f3677e1d61054b48dd1c9b2bb64b67d0273355c0847e2d99b1cb7d09" -} diff --git a/prover/prover_dal/.sqlx/query-a91c0489a8830a3dbd628c75aca22e807dcfd1ff4cda8b59ddedb5c44ee34df6.json b/prover/prover_dal/.sqlx/query-a91c0489a8830a3dbd628c75aca22e807dcfd1ff4cda8b59ddedb5c44ee34df6.json new file mode 100644 index 00000000000..000ee666c7f --- /dev/null +++ b/prover/prover_dal/.sqlx/query-a91c0489a8830a3dbd628c75aca22e807dcfd1ff4cda8b59ddedb5c44ee34df6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM proof_compression_jobs_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a91c0489a8830a3dbd628c75aca22e807dcfd1ff4cda8b59ddedb5c44ee34df6" +} diff --git a/prover/prover_dal/.sqlx/query-aff36fccb6408f736085c64eefda3c0d59703299479d3626b22ab648d675ce40.json b/prover/prover_dal/.sqlx/query-aff36fccb6408f736085c64eefda3c0d59703299479d3626b22ab648d675ce40.json new file mode 100644 index 00000000000..dd39cf64b86 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-aff36fccb6408f736085c64eefda3c0d59703299479d3626b22ab648d675ce40.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM prover_jobs_fri", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "aff36fccb6408f736085c64eefda3c0d59703299479d3626b22ab648d675ce40" +} diff --git a/prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json b/prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json deleted file mode 100644 index 9e4ac4d8f94..00000000000 --- a/prover/prover_dal/.sqlx/query-b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "TRUNCATE TABLE witness_inputs_fri, leaf_aggregation_witness_jobs_fri, node_aggregation_witness_jobs_fri, scheduler_witness_jobs_fri, scheduler_dependency_tracker_fri", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "b97bbe7fb3ef219d7408d30897c5cc9feb543e2ab4d91f27559d844280337ea9" -} diff --git a/prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json b/prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json new file mode 100644 index 00000000000..d0c71cb48d2 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM gpu_prover_queue_fri_archive", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae" +} diff --git a/prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json b/prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json new file mode 100644 index 00000000000..d8953e107fe --- /dev/null +++ b/prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM gpu_prover_queue_fri_archive_id_seq", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67" +} diff --git a/prover/prover_dal/.sqlx/query-fc15423c4eef939d0b47f1cf068d375fd4baf93f109270cbab3ee4e61cfb2c67.json b/prover/prover_dal/.sqlx/query-fc15423c4eef939d0b47f1cf068d375fd4baf93f109270cbab3ee4e61cfb2c67.json new file mode 100644 index 00000000000..b10e1c0681e --- /dev/null +++ b/prover/prover_dal/.sqlx/query-fc15423c4eef939d0b47f1cf068d375fd4baf93f109270cbab3ee4e61cfb2c67.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM prover_fri_protocol_versions", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "fc15423c4eef939d0b47f1cf068d375fd4baf93f109270cbab3ee4e61cfb2c67" +} diff --git a/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs b/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs index ddc7a9aefdb..485237d7b25 100644 --- a/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs +++ b/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs @@ -210,10 +210,22 @@ impl FriGpuProverQueueDal<'_, '_> { .unwrap_or(0) as usize } - pub async fn delete_all(&mut self) { - sqlx::query!("TRUNCATE TABLE gpu_prover_queue_fri, gpu_prover_queue_fri_archive") + pub async fn delete_gpu_prover_queue_fri(&mut self) { + sqlx::query!("DELETE FROM gpu_prover_queue_fri") .execute(self.storage.conn()) .await .unwrap(); } + + pub async fn delete_gpu_prover_queue_fri_archive(&mut self) { + sqlx::query!("DELETE FROM gpu_prover_queue_fri") + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete(&mut self) { + self.delete_gpu_prover_queue_fri().await; + self.delete_gpu_prover_queue_fri_archive().await; + } } diff --git a/prover/prover_dal/src/fri_proof_compressor_dal.rs b/prover/prover_dal/src/fri_proof_compressor_dal.rs index 1c6ee3a411b..6e37bd7ba62 100644 --- a/prover/prover_dal/src/fri_proof_compressor_dal.rs +++ b/prover/prover_dal/src/fri_proof_compressor_dal.rs @@ -343,8 +343,8 @@ impl FriProofCompressorDal<'_, '_> { .unwrap(); } - pub async fn delete_all(&mut self) { - sqlx::query!("TRUNCATE TABLE proof_compression_jobs_fri") + pub async fn delete(&mut self) { + sqlx::query!("DELETE FROM proof_compression_jobs_fri") .execute(self.storage.conn()) .await .unwrap(); diff --git a/prover/prover_dal/src/fri_protocol_versions_dal.rs b/prover/prover_dal/src/fri_protocol_versions_dal.rs index 7a96623305a..93fedaed284 100644 --- a/prover/prover_dal/src/fri_protocol_versions_dal.rs +++ b/prover/prover_dal/src/fri_protocol_versions_dal.rs @@ -91,8 +91,8 @@ impl FriProtocolVersionsDal<'_, '_> { .collect() } - pub async fn delete_all(&mut self) { - sqlx::query!("TRUNCATE TABLE prover_fri_protocol_versions") + pub async fn delete(&mut self) { + sqlx::query!("DELETE FROM prover_fri_protocol_versions") .execute(self.storage.conn()) .await .unwrap(); diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index ed53aacd67a..30824a1e98a 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -637,10 +637,22 @@ impl FriProverDal<'_, '_> { .await; } - pub async fn delete_all(&mut self) { - sqlx::query!("TRUNCATE TABLE prover_jobs_fri, prover_jobs_fri_archive") + pub async fn delete_prover_jobs_fri(&mut self) { + sqlx::query!("DELETE FROM prover_jobs_fri") .execute(self.storage.conn()) .await .unwrap(); } + + pub async fn delete_prover_jobs_fri_archive(&mut self) { + sqlx::query!("DELETE FROM prover_jobs_fri_archive") + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete(&mut self) { + self.delete_prover_jobs_fri().await; + self.delete_prover_jobs_fri_archive().await; + } } diff --git a/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs b/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs index 5b8ef40ea0d..34a9021530d 100644 --- a/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs +++ b/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs @@ -150,8 +150,8 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> { .unwrap() } - pub async fn delete_all(&mut self) { - sqlx::query!("TRUNCATE TABLE scheduler_dependency_tracker_fri") + pub async fn delete(&mut self) { + sqlx::query!("DELETE FROM scheduler_dependency_tracker_fri") .execute(self.storage.conn()) .await .unwrap(); diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 968002d63c1..b2934e70f7d 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1219,10 +1219,38 @@ impl FriWitnessGeneratorDal<'_, '_> { self.delete_scheduler_batch_data(block_number).await; } - pub async fn delete_all(&mut self) { - sqlx::query!("TRUNCATE TABLE witness_inputs_fri, leaf_aggregation_witness_jobs_fri, node_aggregation_witness_jobs_fri, scheduler_witness_jobs_fri, scheduler_dependency_tracker_fri") + pub async fn delete_basic_witness_generator_data(&mut self) { + sqlx::query!("DELETE FROM witness_inputs_fri") .execute(self.storage.conn()) .await .unwrap(); } + + pub async fn delete_leaf_aggregation_generator_data(&mut self) { + sqlx::query!("DELETE FROM leaf_aggregation_witness_jobs_fri") + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_node_aggregation_generator_data(&mut self) { + sqlx::query!("DELETE FROM node_aggregation_witness_jobs_fri") + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete_scheduler_generator_data(&mut self) { + sqlx::query!("DELETE FROM scheduler_witness_jobs_fri") + .execute(self.storage.conn()) + .await + .unwrap(); + } + + pub async fn delete(&mut self) { + self.delete_basic_witness_generator_data().await; + self.delete_leaf_aggregation_generator_data().await; + self.delete_node_aggregation_generator_data().await; + self.delete_scheduler_generator_data().await; + } } From 2be245c3d5508db4b0ae6d33dff8df1acd1c125a Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 29 Apr 2024 18:30:05 -0300 Subject: [PATCH 08/23] Fix command --- prover/prover_cli/src/commands/delete.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs index 8de81d32a4d..ed1965fc616 100644 --- a/prover/prover_cli/src/commands/delete.rs +++ b/prover/prover_cli/src/commands/delete.rs @@ -1,4 +1,4 @@ -use anyhow::Context; +use anyhow::{bail, Context}; use clap::Args as ClapArgs; use dialoguer::{theme::ColorfulTheme, Input}; use prover_dal::{Connection, ConnectionPool, Prover, ProverDal}; @@ -9,7 +9,7 @@ use zksync_types::L1BatchNumber; #[derive(ClapArgs)] pub(crate) struct Args { #[clap(short, long, conflicts_with = "all")] - batch: L1BatchNumber, + batch: Option, #[clap(short, long, conflicts_with = "batch", default_value_t = false)] all: bool, #[clap(short, long, default_value_t = false)] @@ -17,6 +17,10 @@ pub(crate) struct Args { } pub(crate) async fn run(args: Args) -> anyhow::Result<()> { + if !args.all && args.batch.is_none() { + bail!("Either --all or --batch should be provided"); + } + let confirmation = Input::::with_theme(&ColorfulTheme::default()) .with_prompt("Are you sure you want to delete the data?") .default("yes".to_owned()) @@ -39,21 +43,19 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { if args.all { delete_prover_db(conn).await?; } else { - delete_batch_data(conn, args.batch, args.failed).await?; + delete_batch_data(conn, args.batch.unwrap(), args.failed).await?; } Ok(()) } async fn delete_prover_db(mut conn: Connection<'_, Prover>) -> anyhow::Result<()> { - conn.fri_gpu_prover_queue_dal().delete_all().await; - conn.fri_prover_jobs_dal().delete_all().await; - conn.fri_scheduler_dependency_tracker_dal() - .delete_all() - .await; - conn.fri_protocol_versions_dal().delete_all().await; - conn.fri_proof_compressor_dal().delete_all().await; - conn.fri_witness_generator_dal().delete_all().await; + conn.fri_gpu_prover_queue_dal().delete().await; + conn.fri_prover_jobs_dal().delete().await; + conn.fri_scheduler_dependency_tracker_dal().delete().await; + conn.fri_protocol_versions_dal().delete().await; + conn.fri_proof_compressor_dal().delete().await; + conn.fri_witness_generator_dal().delete().await; Ok(()) } From aae15a144ed414dd537eb9876cc7b44d17052d37 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Tue, 30 Apr 2024 12:10:47 -0300 Subject: [PATCH 09/23] Refactor queries --- ...5129da7699cac6abe80533b6a8c848eaf2673.json | 12 --- ...14cc001ab8f927536c54ca8624fcf53c1d630.json | 12 --- ...3fb71f93ba8a66f17a5bce1994345d78994ef.json | 12 --- ...69c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json | 12 --- ...155a63bf176555bf3da67a984eeb36def1e24.json | 12 --- ...d63d68859bc719313a30fbd399f1009a07fa3.json | 12 --- ...0ab4f3a9e187e78b734a7de4daff996fdd8af.json | 12 --- ...49fa5bc3370cc3ee76c37aab325132f55e5b9.json | 12 --- ...b5004ff11de1aae6d9639366dde9b7b42d800.json | 12 --- ...c30ee2cd1c87236ac2cdf1602bf5f479e2fae.json | 12 --- ...d28dc33bbed5f85720d0125b0a6ce65526d67.json | 12 --- .../src/fri_witness_generator_dal.rs | 92 ++++++++++++------- 12 files changed, 58 insertions(+), 166 deletions(-) delete mode 100644 prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json delete mode 100644 prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json delete mode 100644 prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json delete mode 100644 prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json delete mode 100644 prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json delete mode 100644 prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json delete mode 100644 prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json delete mode 100644 prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json delete mode 100644 prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json delete mode 100644 prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json delete mode 100644 prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json diff --git a/prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json b/prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json deleted file mode 100644 index 1eecb02f2bd..00000000000 --- a/prover/prover_dal/.sqlx/query-05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM leaf_aggregation_witness_jobs_fri", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "05256ba89dead8256bdbaa631f65129da7699cac6abe80533b6a8c848eaf2673" -} diff --git a/prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json b/prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json deleted file mode 100644 index 365a03f7793..00000000000 --- a/prover/prover_dal/.sqlx/query-2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM prover_jobs_fri_archive_id_seq", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "2c571bc7a9bf3812530332f335814cc001ab8f927536c54ca8624fcf53c1d630" -} diff --git a/prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json b/prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json deleted file mode 100644 index 94d26c79d73..00000000000 --- a/prover/prover_dal/.sqlx/query-39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM node_aggregation_witness_jobs_fri", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "39c37e86ce5ebcebf80ae0224b43fb71f93ba8a66f17a5bce1994345d78994ef" -} diff --git a/prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json b/prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json deleted file mode 100644 index 01bab2bcb17..00000000000 --- a/prover/prover_dal/.sqlx/query-44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM leaf_aggregation_witness_jobs_fri_id_seq", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "44a23121b20fa840253be115dd169c8d3722f18d3d01a658a6dc4f0ae4f1dbb5" -} diff --git a/prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json b/prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json deleted file mode 100644 index 88f2f8c563a..00000000000 --- a/prover/prover_dal/.sqlx/query-4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM witness_inputs_fri", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "4d60b156ec6c7c55989f89ed92f155a63bf176555bf3da67a984eeb36def1e24" -} diff --git a/prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json b/prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json deleted file mode 100644 index a12f66f5b08..00000000000 --- a/prover/prover_dal/.sqlx/query-5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM gpu_prover_queue_fri_id_seq", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "5a69f8c1aff9f7d36ca7f67db1dd63d68859bc719313a30fbd399f1009a07fa3" -} diff --git a/prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json b/prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json deleted file mode 100644 index 786e7af0d10..00000000000 --- a/prover/prover_dal/.sqlx/query-5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM prover_jobs_fri_id_seq", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "5bad22fbd2454672db89075046a0ab4f3a9e187e78b734a7de4daff996fdd8af" -} diff --git a/prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json b/prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json deleted file mode 100644 index 0581cd54c63..00000000000 --- a/prover/prover_dal/.sqlx/query-62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM node_aggregation_witness_jobs_fri_id_seq", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "62a4486b00390450b7471e9eecd49fa5bc3370cc3ee76c37aab325132f55e5b9" -} diff --git a/prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json b/prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json deleted file mode 100644 index 1efcca428cf..00000000000 --- a/prover/prover_dal/.sqlx/query-98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM scheduler_witness_jobs_fri", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "98dca4e4565b8fe3b8fbeb19fceb5004ff11de1aae6d9639366dde9b7b42d800" -} diff --git a/prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json b/prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json deleted file mode 100644 index d0c71cb48d2..00000000000 --- a/prover/prover_dal/.sqlx/query-dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM gpu_prover_queue_fri_archive", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "dd7938f1c11d92e1743cd07e7eac30ee2cd1c87236ac2cdf1602bf5f479e2fae" -} diff --git a/prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json b/prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json deleted file mode 100644 index d8953e107fe..00000000000 --- a/prover/prover_dal/.sqlx/query-eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM gpu_prover_queue_fri_archive_id_seq", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "eaacbe5ce3d77abaa2b29b563ffd28dc33bbed5f85720d0125b0a6ce65526d67" -} diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index b2934e70f7d..0fc8f9aa2be 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1211,46 +1211,70 @@ impl FriWitnessGeneratorDal<'_, '_> { .await .unwrap(); } - - pub async fn delete_batch_data(&mut self, block_number: L1BatchNumber) { - self.delete_witness_inputs_batch_data(block_number).await; - self.delete_leaf_aggregation_batch_data(block_number).await; - self.delete_node_aggregation_batch_data(block_number).await; - self.delete_scheduler_batch_data(block_number).await; - } - - pub async fn delete_basic_witness_generator_data(&mut self) { - sqlx::query!("DELETE FROM witness_inputs_fri") - .execute(self.storage.conn()) - .await - .unwrap(); - } - - pub async fn delete_leaf_aggregation_generator_data(&mut self) { - sqlx::query!("DELETE FROM leaf_aggregation_witness_jobs_fri") - .execute(self.storage.conn()) - .await - .unwrap(); + pub async fn delete_witness_generator_data_for_batch( + &mut self, + block_number: L1BatchNumber, + aggregation_round: AggregationRound, + ) { + sqlx::query( + format!( + r#" + DELETE FROM + {} + WHERE + l1_batch_number = {} + "#, + Self::input_table_name_for(aggregation_round), + i64::from(block_number.0), + ) + .as_str(), + ) + .execute(self.storage.conn()) + .await + .unwrap(); } - pub async fn delete_node_aggregation_generator_data(&mut self) { - sqlx::query!("DELETE FROM node_aggregation_witness_jobs_fri") - .execute(self.storage.conn()) - .await - .unwrap(); + pub async fn delete_batch_data(&mut self, block_number: L1BatchNumber) { + self.delete_witness_generator_data_for_batch(block_number, AggregationRound::BasicCircuits) + .await; + self.delete_witness_generator_data_for_batch( + block_number, + AggregationRound::LeafAggregation, + ) + .await; + self.delete_witness_generator_data_for_batch( + block_number, + AggregationRound::NodeAggregation, + ) + .await; + self.delete_witness_generator_data_for_batch(block_number, AggregationRound::Scheduler) + .await; } - pub async fn delete_scheduler_generator_data(&mut self) { - sqlx::query!("DELETE FROM scheduler_witness_jobs_fri") - .execute(self.storage.conn()) - .await - .unwrap(); + pub async fn delete_witness_generator_data(&mut self, aggregation_round: AggregationRound) { + sqlx::query( + format!( + r#" + DELETE FROM + {} + "#, + Self::input_table_name_for(aggregation_round) + ) + .as_str(), + ) + .execute(self.storage.conn()) + .await + .unwrap(); } pub async fn delete(&mut self) { - self.delete_basic_witness_generator_data().await; - self.delete_leaf_aggregation_generator_data().await; - self.delete_node_aggregation_generator_data().await; - self.delete_scheduler_generator_data().await; + self.delete_witness_generator_data(AggregationRound::BasicCircuits) + .await; + self.delete_witness_generator_data(AggregationRound::LeafAggregation) + .await; + self.delete_witness_generator_data(AggregationRound::NodeAggregation) + .await; + self.delete_witness_generator_data(AggregationRound::Scheduler) + .await; } } From e891a0fa59d2be2d63b3b1ac36848b1c40461b91 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Tue, 30 Apr 2024 15:26:18 -0300 Subject: [PATCH 10/23] Implement functionallity for --failed flag --- prover/prover_cli/src/commands/delete.rs | 67 ++++++-- ...df921b9575247fd6fc0dfdbcd6bc55c9b06f3.json | 14 ++ ...7249ec09c0daf4368021788207370213a6d94.json | 14 -- ...60846e2e2e9c131c5857f977500e012f7141b.json | 14 -- ...045c759d9536b6858ca832120f2daf9434e66.json | 14 ++ ...356f9e77694131bd093163feb4f4a513de9d0.json | 14 -- ...5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json | 14 ++ .../src/fri_gpu_prover_queue_dal.rs | 16 +- .../src/fri_proof_compressor_dal.rs | 47 +++-- .../src/fri_protocol_versions_dal.rs | 3 +- prover/prover_dal/src/fri_prover_dal.rs | 125 +++++++++----- .../fri_scheduler_dependency_tracker_dal.rs | 3 +- .../src/fri_witness_generator_dal.rs | 161 ++++++++---------- 13 files changed, 286 insertions(+), 220 deletions(-) create mode 100644 prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json delete mode 100644 prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json delete mode 100644 prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json create mode 100644 prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json delete mode 100644 prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json create mode 100644 prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs index ed1965fc616..770ac67733e 100644 --- a/prover/prover_cli/src/commands/delete.rs +++ b/prover/prover_cli/src/commands/delete.rs @@ -1,18 +1,24 @@ use anyhow::{bail, Context}; use clap::Args as ClapArgs; use dialoguer::{theme::ColorfulTheme, Input}; -use prover_dal::{Connection, ConnectionPool, Prover, ProverDal}; +use prover_dal::{ + fri_proof_compressor_dal::ProofCompressionJobStatus, + fri_witness_generator_dal::FriWitnessJobStatus, Connection, ConnectionPool, Prover, ProverDal, +}; use zksync_config::PostgresConfig; use zksync_env_config::FromEnv; -use zksync_types::L1BatchNumber; +use zksync_types::{prover_dal::ProverJobStatus, L1BatchNumber}; #[derive(ClapArgs)] pub(crate) struct Args { + /// Batch number to delete #[clap(short, long, conflicts_with = "all")] batch: Option, + /// Delete data from all batches #[clap(short, long, conflicts_with = "batch", default_value_t = false)] all: bool, - #[clap(short, long, default_value_t = false)] + /// Delete only failed data, used only with --batch + #[clap(short, long, default_value_t = false, conflicts_with = "all")] failed: bool, } @@ -50,28 +56,53 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { } async fn delete_prover_db(mut conn: Connection<'_, Prover>) -> anyhow::Result<()> { - conn.fri_gpu_prover_queue_dal().delete().await; - conn.fri_prover_jobs_dal().delete().await; - conn.fri_scheduler_dependency_tracker_dal().delete().await; - conn.fri_protocol_versions_dal().delete().await; - conn.fri_proof_compressor_dal().delete().await; - conn.fri_witness_generator_dal().delete().await; - Ok(()) + conn.fri_gpu_prover_queue_dal() + .delete() + .await + .context("failed to delete gpu prover queue")?; + conn.fri_prover_jobs_dal() + .delete() + .await + .context("failed to delete prover jobs")?; + conn.fri_scheduler_dependency_tracker_dal() + .delete() + .await + .context("failed to delete scheduler dependency tracker")?; + conn.fri_protocol_versions_dal() + .delete() + .await + .context("failed to delete protocol versions")?; + conn.fri_proof_compressor_dal() + .delete() + .await + .context("failed to delete proof compressor")?; + conn.fri_witness_generator_dal() + .delete() + .await + .context("failed to delete witness generator")? } async fn delete_batch_data( mut conn: Connection<'_, Prover>, block_number: L1BatchNumber, - _failed: bool, + failed: bool, ) -> anyhow::Result<()> { conn.fri_proof_compressor_dal() - .delete_batch_data(block_number) - .await; + .delete_batch_data_with_status( + block_number, + failed.then_some(ProofCompressionJobStatus::Failed), + ) + .await + .context("failed to delete proof compressor data")?; conn.fri_prover_jobs_dal() - .delete_batch_data(block_number) - .await; + .delete_batch_data_with_status( + block_number, + failed.then_some(ProverJobStatus::Failed(Default::default())), + ) + .await + .context("failed to delete prover jobs data")?; conn.fri_witness_generator_dal() - .delete_batch_data(block_number) - .await; - Ok(()) + .delete_batch_data_with_status(block_number, failed.then_some(FriWitnessJobStatus::Failed)) + .await + .context("failed to delete witness generator data")? } diff --git a/prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json b/prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json new file mode 100644 index 00000000000..8950c9cc33a --- /dev/null +++ b/prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM proof_compression_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3" +} diff --git a/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json b/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json deleted file mode 100644 index 967f02586d7..00000000000 --- a/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM proof_compression_jobs_fri\n WHERE\n l1_batch_number = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94" -} diff --git a/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json b/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json deleted file mode 100644 index fbc8db58e91..00000000000 --- a/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n prover_jobs_fri\n WHERE\n l1_batch_number = $1;\n \n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b" -} diff --git a/prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json b/prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json new file mode 100644 index 00000000000..a07c8b92c2e --- /dev/null +++ b/prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n prover_jobs_fri\n WHERE\n l1_batch_number = $1;\n \n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66" +} diff --git a/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json b/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json deleted file mode 100644 index 0ea0b74e3f9..00000000000 --- a/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n prover_jobs_fri_archive\n WHERE\n l1_batch_number = $1;\n \n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0" -} diff --git a/prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json b/prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json new file mode 100644 index 00000000000..7d5f8b48d3d --- /dev/null +++ b/prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n prover_jobs_fri_archive\n WHERE\n l1_batch_number = $1;\n \n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3" +} diff --git a/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs b/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs index 485237d7b25..d8f0f656a14 100644 --- a/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs +++ b/prover/prover_dal/src/fri_gpu_prover_queue_dal.rs @@ -210,22 +210,24 @@ impl FriGpuProverQueueDal<'_, '_> { .unwrap_or(0) as usize } - pub async fn delete_gpu_prover_queue_fri(&mut self) { + pub async fn delete_gpu_prover_queue_fri( + &mut self, + ) -> sqlx::Result { sqlx::query!("DELETE FROM gpu_prover_queue_fri") .execute(self.storage.conn()) .await - .unwrap(); } - pub async fn delete_gpu_prover_queue_fri_archive(&mut self) { + pub async fn delete_gpu_prover_queue_fri_archive( + &mut self, + ) -> sqlx::Result { sqlx::query!("DELETE FROM gpu_prover_queue_fri") .execute(self.storage.conn()) .await - .unwrap(); } - pub async fn delete(&mut self) { - self.delete_gpu_prover_queue_fri().await; - self.delete_gpu_prover_queue_fri_archive().await; + pub async fn delete(&mut self) -> sqlx::Result { + self.delete_gpu_prover_queue_fri().await?; + self.delete_gpu_prover_queue_fri_archive().await } } diff --git a/prover/prover_dal/src/fri_proof_compressor_dal.rs b/prover/prover_dal/src/fri_proof_compressor_dal.rs index 6e37bd7ba62..50fd330b36b 100644 --- a/prover/prover_dal/src/fri_proof_compressor_dal.rs +++ b/prover/prover_dal/src/fri_proof_compressor_dal.rs @@ -329,24 +329,43 @@ impl FriProofCompressorDal<'_, '_> { } } - pub async fn delete_batch_data(&mut self, block_number: L1BatchNumber) { - sqlx::query!( - r#" - DELETE FROM proof_compression_jobs_fri - WHERE - l1_batch_number = $1 - "#, - i64::from(block_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); + pub async fn delete_batch_data_with_status( + &mut self, + block_number: L1BatchNumber, + status: Option, + ) -> sqlx::Result { + if let Some(status) = status { + sqlx::query( + format!( + " + DELETE FROM proof_compression_jobs_fri + WHERE + l1_batch_number = {l1_batch_number} + AND status = '{status}' + ", + l1_batch_number = i64::from(block_number.0), + ) + .as_str(), + ) + .execute(self.storage.conn()) + .await + } else { + sqlx::query!( + r#" + DELETE FROM proof_compression_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(block_number.0) + ) + .execute(self.storage.conn()) + .await + } } - pub async fn delete(&mut self) { + pub async fn delete(&mut self) -> sqlx::Result { sqlx::query!("DELETE FROM proof_compression_jobs_fri") .execute(self.storage.conn()) .await - .unwrap(); } } diff --git a/prover/prover_dal/src/fri_protocol_versions_dal.rs b/prover/prover_dal/src/fri_protocol_versions_dal.rs index 93fedaed284..2d67761f2d7 100644 --- a/prover/prover_dal/src/fri_protocol_versions_dal.rs +++ b/prover/prover_dal/src/fri_protocol_versions_dal.rs @@ -91,10 +91,9 @@ impl FriProtocolVersionsDal<'_, '_> { .collect() } - pub async fn delete(&mut self) { + pub async fn delete(&mut self) -> sqlx::Result { sqlx::query!("DELETE FROM prover_fri_protocol_versions") .execute(self.storage.conn()) .await - .unwrap(); } } diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index 30824a1e98a..a00e271ca28 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -4,7 +4,9 @@ use std::{collections::HashMap, convert::TryFrom, time::Duration}; use zksync_basic_types::{ basic_fri_types::{AggregationRound, CircuitIdRoundTuple}, protocol_version::ProtocolVersionId, - prover_dal::{FriProverJobMetadata, JobCountStatistics, StuckJobs, EIP_4844_CIRCUIT_ID}, + prover_dal::{ + FriProverJobMetadata, JobCountStatistics, ProverJobStatus, StuckJobs, EIP_4844_CIRCUIT_ID, + }, L1BatchNumber, }; use zksync_db_connection::{ @@ -595,64 +597,103 @@ impl FriProverDal<'_, '_> { .unwrap_or(0) as usize } - pub async fn delete_prover_jobs_fri_batch_data(&mut self, l1_batch_number: L1BatchNumber) { - sqlx::query!( - r#" - DELETE FROM - prover_jobs_fri - WHERE - l1_batch_number = $1; - - "#, - i64::from(l1_batch_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); + pub async fn delete_prover_jobs_fri_batch_data_with_status( + &mut self, + l1_batch_number: L1BatchNumber, + status: &Option, + ) -> sqlx::Result { + if let Some(status) = status { + sqlx::query(&format!( + " + DELETE FROM + prover_jobs_fri + WHERE + l1_batch_number = {l1_batch_number} + AND status = '{status}' + + ", + l1_batch_number = i64::from(l1_batch_number.0), + )) + .execute(self.storage.conn()) + .await + } else { + sqlx::query!( + r#" + DELETE FROM + prover_jobs_fri + WHERE + l1_batch_number = $1; + + "#, + i64::from(l1_batch_number.0) + ) + .execute(self.storage.conn()) + .await + } } - pub async fn delete_prover_jobs_fri_archive_batch_data( + pub async fn delete_prover_jobs_fri_archive_batch_data_with_status( &mut self, l1_batch_number: L1BatchNumber, - ) { - sqlx::query!( - r#" - DELETE FROM - prover_jobs_fri_archive - WHERE - l1_batch_number = $1; - - "#, - i64::from(l1_batch_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); + status: &Option, + ) -> sqlx::Result { + if let Some(status) = status { + sqlx::query(&format!( + " + DELETE FROM + prover_jobs_fri_archive + WHERE + l1_batch_number = {l1_batch_number} + AND status = '{status}' + + ", + l1_batch_number = i64::from(l1_batch_number.0), + )) + .execute(self.storage.conn()) + .await + } else { + sqlx::query!( + r#" + DELETE FROM + prover_jobs_fri_archive + WHERE + l1_batch_number = $1; + + "#, + i64::from(l1_batch_number.0) + ) + .execute(self.storage.conn()) + .await + } } - pub async fn delete_batch_data(&mut self, l1_batch_number: L1BatchNumber) { - self.delete_prover_jobs_fri_batch_data(l1_batch_number) - .await; - self.delete_prover_jobs_fri_archive_batch_data(l1_batch_number) - .await; + pub async fn delete_batch_data_with_status( + &mut self, + l1_batch_number: L1BatchNumber, + status: Option, + ) -> sqlx::Result { + self.delete_prover_jobs_fri_batch_data_with_status(l1_batch_number, &status) + .await?; + self.delete_prover_jobs_fri_archive_batch_data_with_status(l1_batch_number, &status) + .await } - pub async fn delete_prover_jobs_fri(&mut self) { + pub async fn delete_prover_jobs_fri(&mut self) -> sqlx::Result { sqlx::query!("DELETE FROM prover_jobs_fri") .execute(self.storage.conn()) .await - .unwrap(); } - pub async fn delete_prover_jobs_fri_archive(&mut self) { + pub async fn delete_prover_jobs_fri_archive( + &mut self, + ) -> sqlx::Result { sqlx::query!("DELETE FROM prover_jobs_fri_archive") .execute(self.storage.conn()) .await - .unwrap(); } - pub async fn delete(&mut self) { - self.delete_prover_jobs_fri().await; - self.delete_prover_jobs_fri_archive().await; + pub async fn delete(&mut self) -> sqlx::Result { + self.delete_prover_jobs_fri().await?; + self.delete_prover_jobs_fri_archive().await } } diff --git a/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs b/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs index 34a9021530d..4061b9e6034 100644 --- a/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs +++ b/prover/prover_dal/src/fri_scheduler_dependency_tracker_dal.rs @@ -150,10 +150,9 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> { .unwrap() } - pub async fn delete(&mut self) { + pub async fn delete(&mut self) -> sqlx::Result { sqlx::query!("DELETE FROM scheduler_dependency_tracker_fri") .execute(self.storage.conn()) .await - .unwrap(); } } diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 0fc8f9aa2be..022f32d8c0d 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1151,107 +1151,83 @@ impl FriWitnessGeneratorDal<'_, '_> { .map(|id| ProtocolVersionId::try_from(id as u16).unwrap()) .unwrap() } - - pub async fn delete_witness_inputs_batch_data(&mut self, block_number: L1BatchNumber) { - sqlx::query!( - r#" - DELETE FROM - witness_inputs_fri - WHERE - l1_batch_number = $1 - "#, - i64::from(block_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); - } - - pub async fn delete_leaf_aggregation_batch_data(&mut self, block_number: L1BatchNumber) { - sqlx::query!( - r#" - DELETE FROM - leaf_aggregation_witness_jobs_fri - WHERE - l1_batch_number = $1 - "#, - i64::from(block_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); - } - - pub async fn delete_node_aggregation_batch_data(&mut self, block_number: L1BatchNumber) { - sqlx::query!( - r#" - DELETE FROM - node_aggregation_witness_jobs_fri - WHERE - l1_batch_number = $1 - "#, - i64::from(block_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); - } - - pub async fn delete_scheduler_batch_data(&mut self, block_number: L1BatchNumber) { - sqlx::query!( - r#" - DELETE FROM - scheduler_witness_jobs_fri - WHERE - l1_batch_number = $1 - "#, - i64::from(block_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); - } - pub async fn delete_witness_generator_data_for_batch( + pub async fn delete_witness_generator_data_for_batch_with_status( &mut self, block_number: L1BatchNumber, aggregation_round: AggregationRound, - ) { - sqlx::query( - format!( - r#" - DELETE FROM - {} - WHERE - l1_batch_number = {} - "#, - Self::input_table_name_for(aggregation_round), - i64::from(block_number.0), + status: &Option, + ) -> sqlx::Result { + if let Some(status) = status { + sqlx::query( + format!( + r#" + DELETE FROM + {table} + WHERE + l1_batch_number = {l1_batch_number} + AND status = '{status}' + "#, + table = Self::input_table_name_for(aggregation_round), + l1_batch_number = i64::from(block_number.0), + ) + .as_str(), ) - .as_str(), - ) - .execute(self.storage.conn()) - .await - .unwrap(); + .execute(self.storage.conn()) + .await + } else { + sqlx::query( + format!( + r#" + DELETE FROM + {} + WHERE + l1_batch_number = {} + "#, + Self::input_table_name_for(aggregation_round), + i64::from(block_number.0), + ) + .as_str(), + ) + .execute(self.storage.conn()) + .await + } } - pub async fn delete_batch_data(&mut self, block_number: L1BatchNumber) { - self.delete_witness_generator_data_for_batch(block_number, AggregationRound::BasicCircuits) - .await; - self.delete_witness_generator_data_for_batch( + pub async fn delete_batch_data_with_status( + &mut self, + block_number: L1BatchNumber, + status: Option, + ) -> sqlx::Result { + self.delete_witness_generator_data_for_batch_with_status( + block_number, + AggregationRound::BasicCircuits, + &status, + ) + .await?; + self.delete_witness_generator_data_for_batch_with_status( block_number, AggregationRound::LeafAggregation, + &status, ) - .await; - self.delete_witness_generator_data_for_batch( + .await?; + self.delete_witness_generator_data_for_batch_with_status( block_number, AggregationRound::NodeAggregation, + &status, ) - .await; - self.delete_witness_generator_data_for_batch(block_number, AggregationRound::Scheduler) - .await; + .await?; + self.delete_witness_generator_data_for_batch_with_status( + block_number, + AggregationRound::Scheduler, + &status, + ) + .await } - pub async fn delete_witness_generator_data(&mut self, aggregation_round: AggregationRound) { + pub async fn delete_witness_generator_data( + &mut self, + aggregation_round: AggregationRound, + ) -> sqlx::Result { sqlx::query( format!( r#" @@ -1264,17 +1240,16 @@ impl FriWitnessGeneratorDal<'_, '_> { ) .execute(self.storage.conn()) .await - .unwrap(); } - pub async fn delete(&mut self) { + pub async fn delete(&mut self) -> sqlx::Result { self.delete_witness_generator_data(AggregationRound::BasicCircuits) - .await; + .await?; self.delete_witness_generator_data(AggregationRound::LeafAggregation) - .await; + .await?; self.delete_witness_generator_data(AggregationRound::NodeAggregation) - .await; + .await?; self.delete_witness_generator_data(AggregationRound::Scheduler) - .await; + .await } } From 9f28e4c349041b6489abdd94d6c59796d74413f8 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Thu, 2 May 2024 09:37:16 -0300 Subject: [PATCH 11/23] zk lint rust --- prover/prover_dal/src/fri_prover_dal.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index 673a8767f82..ee1067c1cea 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -4,9 +4,7 @@ use std::{collections::HashMap, convert::TryFrom, time::Duration}; use zksync_basic_types::{ basic_fri_types::{AggregationRound, CircuitIdRoundTuple}, protocol_version::ProtocolVersionId, - prover_dal::{ - FriProverJobMetadata, JobCountStatistics, ProverJobStatus, StuckJobs, EIP_4844_CIRCUIT_ID, - }, + prover_dal::{FriProverJobMetadata, JobCountStatistics, ProverJobStatus, StuckJobs}, L1BatchNumber, }; use zksync_db_connection::{ From 5b593be7ccecea47f1f44a063a6b75e8c4d0c6bd Mon Sep 17 00:00:00 2001 From: ilitteri Date: Fri, 3 May 2024 09:10:59 -0300 Subject: [PATCH 12/23] Remove --failed functionality --- prover/prover_cli/src/commands/delete.rs | 30 ++---- ...df921b9575247fd6fc0dfdbcd6bc55c9b06f3.json | 14 --- ...7249ec09c0daf4368021788207370213a6d94.json | 14 +++ ...60846e2e2e9c131c5857f977500e012f7141b.json | 14 +++ ...7b9f7559f338f81190e03a026f64fe5db7cbe.json | 14 --- ...ea52faf4df7023b541447a489cf0db5d192de.json | 14 --- ...045c759d9536b6858ca832120f2daf9434e66.json | 14 --- ...356f9e77694131bd093163feb4f4a513de9d0.json | 14 +++ ...5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json | 14 --- ...5cbfdb1f9f0259996948e02040e9175af939a.json | 12 --- ...3a5e612d786b8e236752870cf870109effe03.json | 14 --- ...03cf3909e060d81fbe9d3f12afd90c4ae9118.json | 14 --- .../src/fri_proof_compressor_dal.rs | 40 +++----- prover/prover_dal/src/fri_prover_dal.rs | 93 ++++++------------- .../src/fri_witness_generator_dal.rs | 77 +++++---------- 15 files changed, 114 insertions(+), 278 deletions(-) delete mode 100644 prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json create mode 100644 prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json create mode 100644 prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json delete mode 100644 prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json delete mode 100644 prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json delete mode 100644 prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json create mode 100644 prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json delete mode 100644 prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json delete mode 100644 prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json delete mode 100644 prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json delete mode 100644 prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs index 61080e00ee2..67001643eaa 100644 --- a/prover/prover_cli/src/commands/delete.rs +++ b/prover/prover_cli/src/commands/delete.rs @@ -1,13 +1,10 @@ use anyhow::{bail, Context}; use clap::Args as ClapArgs; use dialoguer::{theme::ColorfulTheme, Input}; -use prover_dal::{ - fri_proof_compressor_dal::ProofCompressionJobStatus, - fri_witness_generator_dal::FriWitnessJobStatus, Connection, ConnectionPool, Prover, ProverDal, -}; +use prover_dal::{Connection, ConnectionPool, Prover, ProverDal}; use zksync_config::PostgresConfig; use zksync_env_config::FromEnv; -use zksync_types::{prover_dal::ProverJobStatus, L1BatchNumber}; +use zksync_types::L1BatchNumber; #[derive(ClapArgs)] pub(crate) struct Args { @@ -17,9 +14,6 @@ pub(crate) struct Args { /// Delete data from all batches #[clap(short, long, conflicts_with = "batch", default_value_t = false)] all: bool, - /// Delete only failed data, used only with --batch - #[clap(short, long, default_value_t = false, conflicts_with = "all")] - failed: bool, } pub(crate) async fn run(args: Args) -> anyhow::Result<()> { @@ -29,9 +23,8 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { let confirmation = Input::::with_theme(&ColorfulTheme::default()) .with_prompt("Are you sure you want to delete the data?") - .default("yes".to_owned()) - .interact_text() - .unwrap(); + .default("no".to_owned()) + .interact_text()?; if confirmation != "yes" { println!("Aborted"); @@ -49,7 +42,7 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { if args.all { delete_prover_db(conn).await?; } else { - delete_batch_data(conn, args.batch.unwrap(), args.failed).await?; + delete_batch_data(conn, args.batch.unwrap()).await?; } Ok(()) @@ -82,24 +75,17 @@ async fn delete_prover_db(mut conn: Connection<'_, Prover>) -> anyhow::Result<() async fn delete_batch_data( mut conn: Connection<'_, Prover>, block_number: L1BatchNumber, - failed: bool, ) -> anyhow::Result<()> { conn.fri_proof_compressor_dal() - .delete_batch_data_with_status( - block_number, - failed.then_some(ProofCompressionJobStatus::Failed), - ) + .delete_batch_data(block_number) .await .context("failed to delete proof compressor data")?; conn.fri_prover_jobs_dal() - .delete_batch_data_with_status( - block_number, - failed.then_some(ProverJobStatus::Failed(Default::default())), - ) + .delete_batch_data(block_number) .await .context("failed to delete prover jobs data")?; conn.fri_witness_generator_dal() - .delete_batch_data_with_status(block_number, failed.then_some(FriWitnessJobStatus::Failed)) + .delete_batch_data(block_number) .await .context("failed to delete witness generator data")?; Ok(()) diff --git a/prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json b/prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json deleted file mode 100644 index 8950c9cc33a..00000000000 --- a/prover/prover_dal/.sqlx/query-2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM proof_compression_jobs_fri\n WHERE\n l1_batch_number = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "2a3f0f83d3f4926f6155fa9e30ddf921b9575247fd6fc0dfdbcd6bc55c9b06f3" -} diff --git a/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json b/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json new file mode 100644 index 00000000000..967f02586d7 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM proof_compression_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "2df88abaae97b6f916b104375bd7249ec09c0daf4368021788207370213a6d94" +} diff --git a/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json b/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json new file mode 100644 index 00000000000..fbc8db58e91 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n prover_jobs_fri\n WHERE\n l1_batch_number = $1;\n \n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3ae5d20066035f93cf8e421a55060846e2e2e9c131c5857f977500e012f7141b" +} diff --git a/prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json b/prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json deleted file mode 100644 index bd350a79cbe..00000000000 --- a/prover/prover_dal/.sqlx/query-5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n witness_inputs_fri\n WHERE\n l1_batch_number = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "5c2e007df433dc7f3dc73ddcd557b9f7559f338f81190e03a026f64fe5db7cbe" -} diff --git a/prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json b/prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json deleted file mode 100644 index 6ebc4fdc65b..00000000000 --- a/prover/prover_dal/.sqlx/query-7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n leaf_aggregation_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "7bb749117d2696219341f293facea52faf4df7023b541447a489cf0db5d192de" -} diff --git a/prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json b/prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json deleted file mode 100644 index a07c8b92c2e..00000000000 --- a/prover/prover_dal/.sqlx/query-7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n prover_jobs_fri\n WHERE\n l1_batch_number = $1;\n \n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "7d1ef508419e4c18ec442df8ba5045c759d9536b6858ca832120f2daf9434e66" -} diff --git a/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json b/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json new file mode 100644 index 00000000000..0ea0b74e3f9 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n prover_jobs_fri_archive\n WHERE\n l1_batch_number = $1;\n \n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9375645f2c854f98c2fd628bd27356f9e77694131bd093163feb4f4a513de9d0" +} diff --git a/prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json b/prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json deleted file mode 100644 index 7d5f8b48d3d..00000000000 --- a/prover/prover_dal/.sqlx/query-95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n prover_jobs_fri_archive\n WHERE\n l1_batch_number = $1;\n \n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "95ec3849dac1df8fe177994767c5ce9cdc7dbb878cbdcbe7004fcbd1e5faeda3" -} diff --git a/prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json b/prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json deleted file mode 100644 index fae2e71cf92..00000000000 --- a/prover/prover_dal/.sqlx/query-9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM scheduler_dependency_tracker_fri", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "9ecc8358c93eba6a3e85eb795cb5cbfdb1f9f0259996948e02040e9175af939a" -} diff --git a/prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json b/prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json deleted file mode 100644 index eac31abb40e..00000000000 --- a/prover/prover_dal/.sqlx/query-b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n node_aggregation_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "b5dc3ee25cb61ca38a33d6a4d973a5e612d786b8e236752870cf870109effe03" -} diff --git a/prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json b/prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json deleted file mode 100644 index b8d9e7afa3c..00000000000 --- a/prover/prover_dal/.sqlx/query-d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM\n scheduler_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "d2a4c3af39d4b8f065962b48e2d03cf3909e060d81fbe9d3f12afd90c4ae9118" -} diff --git a/prover/prover_dal/src/fri_proof_compressor_dal.rs b/prover/prover_dal/src/fri_proof_compressor_dal.rs index 50fd330b36b..1355ff01c83 100644 --- a/prover/prover_dal/src/fri_proof_compressor_dal.rs +++ b/prover/prover_dal/src/fri_proof_compressor_dal.rs @@ -329,38 +329,20 @@ impl FriProofCompressorDal<'_, '_> { } } - pub async fn delete_batch_data_with_status( + pub async fn delete_batch_data( &mut self, block_number: L1BatchNumber, - status: Option, ) -> sqlx::Result { - if let Some(status) = status { - sqlx::query( - format!( - " - DELETE FROM proof_compression_jobs_fri - WHERE - l1_batch_number = {l1_batch_number} - AND status = '{status}' - ", - l1_batch_number = i64::from(block_number.0), - ) - .as_str(), - ) - .execute(self.storage.conn()) - .await - } else { - sqlx::query!( - r#" - DELETE FROM proof_compression_jobs_fri - WHERE - l1_batch_number = $1 - "#, - i64::from(block_number.0) - ) - .execute(self.storage.conn()) - .await - } + sqlx::query!( + r#" + DELETE FROM proof_compression_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(block_number.0) + ) + .execute(self.storage.conn()) + .await } pub async fn delete(&mut self) -> sqlx::Result { diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index ee1067c1cea..3259e883cd9 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -643,84 +643,49 @@ impl FriProverDal<'_, '_> { .collect() } - pub async fn delete_prover_jobs_fri_batch_data_with_status( + pub async fn delete_prover_jobs_fri_batch_data( &mut self, l1_batch_number: L1BatchNumber, - status: &Option, ) -> sqlx::Result { - if let Some(status) = status { - sqlx::query(&format!( - " - DELETE FROM - prover_jobs_fri - WHERE - l1_batch_number = {l1_batch_number} - AND status = '{status}' - - ", - l1_batch_number = i64::from(l1_batch_number.0), - )) - .execute(self.storage.conn()) - .await - } else { - sqlx::query!( - r#" - DELETE FROM - prover_jobs_fri - WHERE - l1_batch_number = $1; - - "#, - i64::from(l1_batch_number.0) - ) - .execute(self.storage.conn()) - .await - } + sqlx::query!( + r#" + DELETE FROM + prover_jobs_fri + WHERE + l1_batch_number = $1; + + "#, + i64::from(l1_batch_number.0) + ) + .execute(self.storage.conn()) + .await } - pub async fn delete_prover_jobs_fri_archive_batch_data_with_status( + pub async fn delete_prover_jobs_fri_archive_batch_data( &mut self, l1_batch_number: L1BatchNumber, - status: &Option, ) -> sqlx::Result { - if let Some(status) = status { - sqlx::query(&format!( - " - DELETE FROM - prover_jobs_fri_archive - WHERE - l1_batch_number = {l1_batch_number} - AND status = '{status}' - - ", - l1_batch_number = i64::from(l1_batch_number.0), - )) - .execute(self.storage.conn()) - .await - } else { - sqlx::query!( - r#" - DELETE FROM - prover_jobs_fri_archive - WHERE - l1_batch_number = $1; - - "#, - i64::from(l1_batch_number.0) - ) - .execute(self.storage.conn()) - .await - } + sqlx::query!( + r#" + DELETE FROM + prover_jobs_fri_archive + WHERE + l1_batch_number = $1; + + "#, + i64::from(l1_batch_number.0) + ) + .execute(self.storage.conn()) + .await } - pub async fn delete_batch_data_with_status( + pub async fn delete_batch_data( &mut self, l1_batch_number: L1BatchNumber, - status: Option, ) -> sqlx::Result { - self.delete_prover_jobs_fri_batch_data_with_status(l1_batch_number, &status) + self.delete_prover_jobs_fri_batch_data(l1_batch_number) .await?; - self.delete_prover_jobs_fri_archive_batch_data_with_status(l1_batch_number, &status) + self.delete_prover_jobs_fri_archive_batch_data(l1_batch_number) .await } diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index faf34ffdfea..f74858cb916 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1395,77 +1395,46 @@ impl FriWitnessGeneratorDal<'_, '_> { .map(|id| ProtocolVersionId::try_from(id as u16).unwrap()) .unwrap() } - pub async fn delete_witness_generator_data_for_batch_with_status( + pub async fn delete_witness_generator_data_for_batch( &mut self, block_number: L1BatchNumber, aggregation_round: AggregationRound, - status: &Option, ) -> sqlx::Result { - if let Some(status) = status { - sqlx::query( - format!( - r#" - DELETE FROM - {table} - WHERE - l1_batch_number = {l1_batch_number} - AND status = '{status}' - "#, - table = Self::input_table_name_for(aggregation_round), - l1_batch_number = i64::from(block_number.0), - ) - .as_str(), - ) - .execute(self.storage.conn()) - .await - } else { - sqlx::query( - format!( - r#" - DELETE FROM - {} - WHERE - l1_batch_number = {} - "#, - Self::input_table_name_for(aggregation_round), - i64::from(block_number.0), - ) - .as_str(), + sqlx::query( + format!( + r#" + DELETE FROM + {table} + WHERE + l1_batch_number = {l1_batch_number} + "#, + table = Self::input_table_name_for(aggregation_round), + l1_batch_number = i64::from(block_number.0), ) - .execute(self.storage.conn()) - .await - } + .as_str(), + ) + .execute(self.storage.conn()) + .await } - pub async fn delete_batch_data_with_status( + pub async fn delete_batch_data( &mut self, block_number: L1BatchNumber, - status: Option, ) -> sqlx::Result { - self.delete_witness_generator_data_for_batch_with_status( - block_number, - AggregationRound::BasicCircuits, - &status, - ) - .await?; - self.delete_witness_generator_data_for_batch_with_status( + self.delete_witness_generator_data_for_batch(block_number, AggregationRound::BasicCircuits) + .await?; + self.delete_witness_generator_data_for_batch( block_number, AggregationRound::LeafAggregation, - &status, ) .await?; - self.delete_witness_generator_data_for_batch_with_status( + self.delete_witness_generator_data_for_batch( block_number, AggregationRound::NodeAggregation, - &status, ) .await?; - self.delete_witness_generator_data_for_batch_with_status( - block_number, - AggregationRound::Scheduler, - &status, - ) - .await + self.delete_witness_generator_data_for_batch(block_number, AggregationRound::Scheduler) + .await } pub async fn delete_witness_generator_data( @@ -1493,6 +1462,8 @@ impl FriWitnessGeneratorDal<'_, '_> { .await?; self.delete_witness_generator_data(AggregationRound::NodeAggregation) .await?; + self.delete_witness_generator_data(AggregationRound::RecursionTip) + .await?; self.delete_witness_generator_data(AggregationRound::Scheduler) .await } From b463fcd86a2abf91abc40ec10214e3b30766f5db Mon Sep 17 00:00:00 2001 From: ilitteri Date: Fri, 3 May 2024 09:13:23 -0300 Subject: [PATCH 13/23] zk lint rust --- prover/prover_dal/src/fri_prover_dal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index 3259e883cd9..49c82eaee93 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -4,7 +4,7 @@ use std::{collections::HashMap, convert::TryFrom, time::Duration}; use zksync_basic_types::{ basic_fri_types::{AggregationRound, CircuitIdRoundTuple}, protocol_version::ProtocolVersionId, - prover_dal::{FriProverJobMetadata, JobCountStatistics, ProverJobStatus, StuckJobs}, + prover_dal::{FriProverJobMetadata, JobCountStatistics, StuckJobs}, L1BatchNumber, }; use zksync_db_connection::{ From 0dc428d408bb020dcf9bd8515fef71a1cac0e581 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Fri, 3 May 2024 10:23:49 -0300 Subject: [PATCH 14/23] Delete recursion tip data for batch --- prover/prover_dal/src/fri_witness_generator_dal.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 39b1584628e..32d03f9e626 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1589,6 +1589,8 @@ impl FriWitnessGeneratorDal<'_, '_> { AggregationRound::NodeAggregation, ) .await?; + self.delete_witness_generator_data(AggregationRound::RecursionTip) + .await?; self.delete_witness_generator_data_for_batch(block_number, AggregationRound::Scheduler) .await } From 0fdddae88c7f745575cc415360e1a074f10d2769 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Fri, 3 May 2024 10:36:35 -0300 Subject: [PATCH 15/23] Update deps --- prover/Cargo.lock | 17 ----------------- prover/prover_cli/Cargo.toml | 6 ++---- prover/prover_cli/src/main.rs | 8 +------- 3 files changed, 3 insertions(+), 28 deletions(-) diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 68e6c49c6c4..55aab41fbaf 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -1809,11 +1809,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ - "humantime", - "is-terminal", "log", - "regex", - "termcolor", ] [[package]] @@ -2805,17 +2801,6 @@ dependencies = [ "serde", ] -[[package]] -name = "is-terminal" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" -dependencies = [ - "hermit-abi 0.3.6", - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "itertools" version = "0.10.5" @@ -4392,9 +4377,7 @@ dependencies = [ "clap 4.4.6", "colored", "dialoguer", - "env_logger 0.10.2", "hex", - "log", "prover_dal", "strum", "tokio", diff --git a/prover/prover_cli/Cargo.toml b/prover/prover_cli/Cargo.toml index 841a2e02619..71374214082 100644 --- a/prover/prover_cli/Cargo.toml +++ b/prover/prover_cli/Cargo.toml @@ -10,12 +10,10 @@ keywords.workspace = true categories.workspace = true [dependencies] -tokio = { version = "1", features = ["rt-multi-thread", "macros"] } -env_logger = "0.10" -log = "0.4" dialoguer = "0.11" -colored = "2.1.0" +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +colored.workspace = true clap = { workspace = true, features = ["derive"] } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/prover/prover_cli/src/main.rs b/prover/prover_cli/src/main.rs index b979a36ed50..f0a7909514e 100644 --- a/prover/prover_cli/src/main.rs +++ b/prover/prover_cli/src/main.rs @@ -2,16 +2,10 @@ use prover_cli::cli; #[tokio::main] async fn main() { - env_logger::builder() - .filter_module("zksync_db_connection", log::LevelFilter::Off) - .filter_module("sqlx", log::LevelFilter::Off) - .filter_level(log::LevelFilter::Debug) - .init(); - match cli::start().await { Ok(_) => {} Err(err) => { - log::error!("{err:?}"); + tracing::error!("{err:?}"); std::process::exit(1); } } From 2e39b455b783ea5fc3271d06185f3618fd0b7a08 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 6 May 2024 13:40:07 -0300 Subject: [PATCH 16/23] Fix cmd --- prover/prover_cli/src/commands/delete.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs index 67001643eaa..b0d7ccee72a 100644 --- a/prover/prover_cli/src/commands/delete.rs +++ b/prover/prover_cli/src/commands/delete.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Context}; +use anyhow::Context; use clap::Args as ClapArgs; use dialoguer::{theme::ColorfulTheme, Input}; use prover_dal::{Connection, ConnectionPool, Prover, ProverDal}; @@ -8,19 +8,21 @@ use zksync_types::L1BatchNumber; #[derive(ClapArgs)] pub(crate) struct Args { - /// Batch number to delete - #[clap(short, long, conflicts_with = "all")] - batch: Option, /// Delete data from all batches - #[clap(short, long, conflicts_with = "batch", default_value_t = false)] + #[clap( + short, + long, + required_unless_present = "batch", + conflicts_with = "batch", + default_value_t = false + )] all: bool, + /// Batch number to delete + #[clap(short, long, required_unless_present = "all", conflicts_with = "all", default_value_t = L1BatchNumber(0))] + batch: L1BatchNumber, } pub(crate) async fn run(args: Args) -> anyhow::Result<()> { - if !args.all && args.batch.is_none() { - bail!("Either --all or --batch should be provided"); - } - let confirmation = Input::::with_theme(&ColorfulTheme::default()) .with_prompt("Are you sure you want to delete the data?") .default("no".to_owned()) @@ -42,7 +44,7 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { if args.all { delete_prover_db(conn).await?; } else { - delete_batch_data(conn, args.batch.unwrap()).await?; + delete_batch_data(conn, args.batch).await?; } Ok(()) From 77c8f2a6912ed7d58404de9ad159bc5c00afad6b Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 6 May 2024 13:42:32 -0300 Subject: [PATCH 17/23] Move dep to workspace --- prover/Cargo.toml | 1 + prover/prover_cli/Cargo.toml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 57808b5e253..ef7418d2a7e 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -37,6 +37,7 @@ clap = "4.4.6" colored = "2.0" const-decoder = "0.3.0" ctrlc = "3.1" +dialoguer = "0.11" futures = "0.3" hex = "0.4" itertools = "0.10.5" diff --git a/prover/prover_cli/Cargo.toml b/prover/prover_cli/Cargo.toml index 71374214082..91bcbf456ba 100644 --- a/prover/prover_cli/Cargo.toml +++ b/prover/prover_cli/Cargo.toml @@ -10,8 +10,7 @@ keywords.workspace = true categories.workspace = true [dependencies] -dialoguer = "0.11" - +dialoguer.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } colored.workspace = true clap = { workspace = true, features = ["derive"] } From 6c11115fdebc8272801bd31a649f2821f6b71604 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Tue, 7 May 2024 09:12:54 -0300 Subject: [PATCH 18/23] Implement restart cmd --- prover/prover_cli/src/commands/restart.rs | 55 +++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 prover/prover_cli/src/commands/restart.rs diff --git a/prover/prover_cli/src/commands/restart.rs b/prover/prover_cli/src/commands/restart.rs new file mode 100644 index 00000000000..eea4cb01411 --- /dev/null +++ b/prover/prover_cli/src/commands/restart.rs @@ -0,0 +1,55 @@ +use anyhow::Context; +use clap::Args as ClapArgs; +use prover_dal::{ + fri_witness_generator_dal::FriWitnessJobStatus, ConnectionPool, Prover, ProverDal, +}; +use zksync_config::PostgresConfig; +use zksync_env_config::FromEnv; +use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber}; + +#[derive(ClapArgs)] +pub(crate) struct Args { + /// Batch number to restart + #[clap(short, long)] + batch: L1BatchNumber, +} + +pub(crate) async fn run(args: Args) -> anyhow::Result<()> { + let config = PostgresConfig::from_env()?; + let prover_connection_pool = + ConnectionPool::::builder(config.prover_url()?, config.max_connections()?) + .build() + .await + .context("failed to build a prover_connection_pool")?; + let mut conn = prover_connection_pool.connection().await.unwrap(); + + conn.fri_proof_compressor_dal() + .delete_batch_data(args.batch) + .await + .context("failed to delete proof compression job for batch")?; + conn.fri_prover_jobs_dal() + .delete_batch_data(args.batch) + .await + .context("failed to delete prover jobs for batch")?; + conn.fri_witness_generator_dal() + .delete_witness_generator_data_for_batch(args.batch, AggregationRound::LeafAggregation) + .await + .context("failed to restart batch: fri_witness_generator_dal()")?; + conn.fri_witness_generator_dal() + .delete_witness_generator_data_for_batch(args.batch, AggregationRound::NodeAggregation) + .await + .context("failed to restart batch: fri_witness_generator_dal()")?; + conn.fri_witness_generator_dal() + .delete_witness_generator_data_for_batch(args.batch, AggregationRound::RecursionTip) + .await + .context("failed to restart batch: fri_witness_generator_dal()")?; + conn.fri_witness_generator_dal() + .delete_witness_generator_data_for_batch(args.batch, AggregationRound::Scheduler) + .await + .context("failed to restart batch: fri_witness_generator_dal()")?; + conn.fri_witness_generator_dal() + .mark_witness_job(FriWitnessJobStatus::Queued, args.batch) + .await; + + Ok(()) +} From 39a113b606b8156a26dae5e3be07e911c8be5173 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Tue, 7 May 2024 09:13:04 -0300 Subject: [PATCH 19/23] Handle restart cmd --- prover/prover_cli/src/cli.rs | 4 +++- prover/prover_cli/src/commands/mod.rs | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/prover/prover_cli/src/cli.rs b/prover/prover_cli/src/cli.rs index eee2f8190cf..65642d24bb5 100644 --- a/prover/prover_cli/src/cli.rs +++ b/prover/prover_cli/src/cli.rs @@ -1,6 +1,6 @@ use clap::{command, Parser, Subcommand}; -use crate::commands::{self, delete, get_file_info}; +use crate::commands::{self, delete, get_file_info, restart}; pub const VERSION_STRING: &str = env!("CARGO_PKG_VERSION"); @@ -17,6 +17,7 @@ enum ProverCommand { Delete(delete::Args), #[command(subcommand)] Status(commands::StatusCommand), + Restart(restart::Args), } pub async fn start() -> anyhow::Result<()> { @@ -25,6 +26,7 @@ pub async fn start() -> anyhow::Result<()> { ProverCommand::FileInfo(args) => get_file_info::run(args).await?, ProverCommand::Delete(args) => delete::run(args).await?, ProverCommand::Status(cmd) => cmd.run().await?, + ProverCommand::Restart(args) => restart::run(args).await?, }; Ok(()) diff --git a/prover/prover_cli/src/commands/mod.rs b/prover/prover_cli/src/commands/mod.rs index da6441a0747..70822e06dd3 100644 --- a/prover/prover_cli/src/commands/mod.rs +++ b/prover/prover_cli/src/commands/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod delete; pub(crate) mod get_file_info; +pub(crate) mod restart; pub(crate) mod status; pub(crate) use status::StatusCommand; From 6fce48007e3e935c92221d65a1203ed3e60e585e Mon Sep 17 00:00:00 2001 From: ilitteri Date: Tue, 7 May 2024 09:25:34 -0300 Subject: [PATCH 20/23] Implement restart for prover jobs --- prover/prover_cli/src/commands/restart.rs | 46 +++++++++++++++++------ 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/prover/prover_cli/src/commands/restart.rs b/prover/prover_cli/src/commands/restart.rs index eea4cb01411..7b259bd87c3 100644 --- a/prover/prover_cli/src/commands/restart.rs +++ b/prover/prover_cli/src/commands/restart.rs @@ -1,7 +1,7 @@ use anyhow::Context; use clap::Args as ClapArgs; use prover_dal::{ - fri_witness_generator_dal::FriWitnessJobStatus, ConnectionPool, Prover, ProverDal, + fri_witness_generator_dal::FriWitnessJobStatus, Connection, ConnectionPool, Prover, ProverDal, }; use zksync_config::PostgresConfig; use zksync_env_config::FromEnv; @@ -10,8 +10,16 @@ use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber}; #[derive(ClapArgs)] pub(crate) struct Args { /// Batch number to restart - #[clap(short, long)] - batch: L1BatchNumber, + #[clap( + short, + long, + required_unless_present = "prover_job", + conflicts_with = "prover_job" + )] + batch: Option, + /// Prover job to restart + #[clap(short, long, required_unless_present = "batch")] + prover_job: Option, } pub(crate) async fn run(args: Args) -> anyhow::Result<()> { @@ -23,33 +31,49 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { .context("failed to build a prover_connection_pool")?; let mut conn = prover_connection_pool.connection().await.unwrap(); + if let Some(batch_number) = args.batch { + restart_batch(batch_number, &mut conn).await?; + } else if let Some(id) = args.prover_job { + restart_prover_job(id, &mut conn).await; + } + + Ok(()) +} + +async fn restart_batch( + batch_number: L1BatchNumber, + conn: &mut Connection<'_, Prover>, +) -> anyhow::Result<()> { conn.fri_proof_compressor_dal() - .delete_batch_data(args.batch) + .delete_batch_data(batch_number) .await .context("failed to delete proof compression job for batch")?; conn.fri_prover_jobs_dal() - .delete_batch_data(args.batch) + .delete_batch_data(batch_number) .await .context("failed to delete prover jobs for batch")?; conn.fri_witness_generator_dal() - .delete_witness_generator_data_for_batch(args.batch, AggregationRound::LeafAggregation) + .delete_witness_generator_data_for_batch(batch_number, AggregationRound::LeafAggregation) .await .context("failed to restart batch: fri_witness_generator_dal()")?; conn.fri_witness_generator_dal() - .delete_witness_generator_data_for_batch(args.batch, AggregationRound::NodeAggregation) + .delete_witness_generator_data_for_batch(batch_number, AggregationRound::NodeAggregation) .await .context("failed to restart batch: fri_witness_generator_dal()")?; conn.fri_witness_generator_dal() - .delete_witness_generator_data_for_batch(args.batch, AggregationRound::RecursionTip) + .delete_witness_generator_data_for_batch(batch_number, AggregationRound::RecursionTip) .await .context("failed to restart batch: fri_witness_generator_dal()")?; conn.fri_witness_generator_dal() - .delete_witness_generator_data_for_batch(args.batch, AggregationRound::Scheduler) + .delete_witness_generator_data_for_batch(batch_number, AggregationRound::Scheduler) .await .context("failed to restart batch: fri_witness_generator_dal()")?; conn.fri_witness_generator_dal() - .mark_witness_job(FriWitnessJobStatus::Queued, args.batch) + .mark_witness_job(FriWitnessJobStatus::Queued, batch_number) .await; - Ok(()) } + +async fn restart_prover_job(id: u32, conn: &mut Connection<'_, Prover>) { + conn.fri_prover_jobs_dal().update_status(id, "queued").await; +} From d42de0034dc6f464e980ed4fef037a9e9097abec Mon Sep 17 00:00:00 2001 From: ilitteri Date: Wed, 8 May 2024 09:11:15 -0300 Subject: [PATCH 21/23] Use singletoon connection pool --- prover/prover_cli/src/commands/delete.rs | 9 ++++----- prover/prover_cli/src/commands/restart.rs | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/prover/prover_cli/src/commands/delete.rs b/prover/prover_cli/src/commands/delete.rs index b0d7ccee72a..93ddf482faa 100644 --- a/prover/prover_cli/src/commands/delete.rs +++ b/prover/prover_cli/src/commands/delete.rs @@ -34,11 +34,10 @@ pub(crate) async fn run(args: Args) -> anyhow::Result<()> { } let config = PostgresConfig::from_env()?; - let prover_connection_pool = - ConnectionPool::::builder(config.prover_url()?, config.max_connections()?) - .build() - .await - .context("failed to build a prover_connection_pool")?; + let prover_connection_pool = ConnectionPool::::singleton(config.prover_url()?) + .build() + .await + .context("failed to build a prover_connection_pool")?; let conn = prover_connection_pool.connection().await.unwrap(); if args.all { diff --git a/prover/prover_cli/src/commands/restart.rs b/prover/prover_cli/src/commands/restart.rs index 7b259bd87c3..f4d10d64b62 100644 --- a/prover/prover_cli/src/commands/restart.rs +++ b/prover/prover_cli/src/commands/restart.rs @@ -24,11 +24,10 @@ pub(crate) struct Args { pub(crate) async fn run(args: Args) -> anyhow::Result<()> { let config = PostgresConfig::from_env()?; - let prover_connection_pool = - ConnectionPool::::builder(config.prover_url()?, config.max_connections()?) - .build() - .await - .context("failed to build a prover_connection_pool")?; + let prover_connection_pool = ConnectionPool::::singleton(config.prover_url()?) + .build() + .await + .context("failed to build a prover_connection_pool")?; let mut conn = prover_connection_pool.connection().await.unwrap(); if let Some(batch_number) = args.batch { From de4aec6b6c7c92cf86a82e325dd2f18b0b626f62 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Wed, 8 May 2024 16:25:02 -0300 Subject: [PATCH 22/23] Fix merge --- prover/prover_cli/src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/prover_cli/src/cli.rs b/prover/prover_cli/src/cli.rs index b43bf83a3c4..7ded422012d 100644 --- a/prover/prover_cli/src/cli.rs +++ b/prover/prover_cli/src/cli.rs @@ -40,7 +40,7 @@ pub async fn start() -> anyhow::Result<()> { match command { ProverCommand::FileInfo(args) => get_file_info::run(args).await?, ProverCommand::Delete(args) => delete::run(args).await?, - ProverCommand::Status(cmd) => cmd.run().await?, + ProverCommand::Status(cmd) => cmd.run(config).await?, ProverCommand::Restart(args) => restart::run(args).await?, }; From 4ce5a2ffc42fdfcc56837ce6bd0a20094faff561 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Mon, 13 May 2024 09:48:11 -0300 Subject: [PATCH 23/23] Update Cargo.lock --- prover/Cargo.lock | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/prover/Cargo.lock b/prover/Cargo.lock index d36d27459e0..baa10545dd6 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -1111,6 +1111,19 @@ dependencies = [ "crossbeam-utils 0.8.19", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "const-decoder" version = "0.3.0"