From 0f5a5c018472c6b5baa67c3a8d369f44e5c81ec7 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 12 Sep 2022 23:57:04 +0200 Subject: [PATCH 1/8] This is a draft of how to store and count the total number of outputs that are traversed when building the index. --- src/index.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/index.rs b/src/index.rs index 7444ff7695..85c8f466f8 100644 --- a/src/index.rs +++ b/src/index.rs @@ -13,6 +13,8 @@ const OUTPOINT_TO_ORDINAL_RANGES: TableDefinition<[u8; 36], [u8]> = TableDefinition::new("OUTPOINT_TO_ORDINAL_RANGES"); const OUTPOINT_TO_TXID: TableDefinition<[u8; 36], [u8; 32]> = TableDefinition::new("OUTPOINT_TO_TXID"); +// Don't know if we need a whole table, cause I just want to store one value persistently. +const OUTPUTS_TRAVERSED: TableDefinition = TableDefinition::new("TOTAL_OUTPUTS_TRAVERSED"); pub(crate) struct Index { client: Client, @@ -56,6 +58,7 @@ impl Index { tx.open_table(HEIGHT_TO_HASH)?; tx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; tx.open_table(OUTPOINT_TO_TXID)?; + tx.open_table(OUTPUTS_TRAVERSED)?; // set default value if empty? This looks really wrong tx.commit()?; @@ -88,11 +91,13 @@ impl Index { .unwrap_or(0); let outputs_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; - + // Double unwrap, let's go !! + let outputs_traversed = wtx.open_table(OUTPUTS_TRAVERSED)?.get(&0).unwrap().unwrap(); let stats = wtx.stats()?; println!("blocks indexed: {}", blocks_indexed); println!("outputs indexed: {}", outputs_indexed); + println!("outputs traversed: {}", outputs_traversed); println!("tree height: {}", stats.tree_height()); println!("free pages: {}", stats.free_pages()); println!("stored: {}", Bytes(stats.stored_bytes())); @@ -156,9 +161,11 @@ impl Index { let mut height_to_hash = wtx.open_table(HEIGHT_TO_HASH)?; let mut outpoint_to_ordinal_ranges = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; let mut outpoint_to_txid = wtx.open_table(OUTPOINT_TO_TXID)?; + let mut outputs_traversed = wtx.open_table(OUTPUTS_TRAVERSED)?; let start = Instant::now(); let mut ordinal_ranges_written = 0; + let mut outputs_in_block = 0; let height = height_to_hash .range(0..)? @@ -248,6 +255,7 @@ impl Index { &mut outpoint_to_txid, &mut input_ordinal_ranges, &mut ordinal_ranges_written, + &mut outputs_in_block, )?; coinbase_inputs.extend(input_ordinal_ranges); @@ -261,11 +269,20 @@ impl Index { &mut outpoint_to_txid, &mut coinbase_inputs, &mut ordinal_ranges_written, + &mut outputs_in_block, )?; } height_to_hash.insert(&height, &block.block_hash())?; + // TODO: error handling + if outputs_traversed.is_empty().unwrap() { + outputs_traversed.insert(&0, &0)?; + } + // TODO: double unwrap + let sum = outputs_in_block + outputs_traversed.get(&0).unwrap().unwrap(); + outputs_traversed.insert(&0, &sum)?; + log::info!( "Wrote {ordinal_ranges_written} ordinal ranges in {}ms", (Instant::now() - start).as_millis(), @@ -310,6 +327,7 @@ impl Index { #[allow(unused)] outpoint_to_txid: &mut Table<[u8; 36], [u8; 32]>, input_ordinal_ranges: &mut VecDeque<(u64, u64)>, ordinal_ranges_written: &mut u64, + outputs_traversed: &mut u64, ) -> Result { for (vout, output) in tx.output.iter().enumerate() { let outpoint = OutPoint { @@ -346,6 +364,8 @@ impl Index { *ordinal_ranges_written += 1; } + *outputs_traversed += 1; + outpoint_to_ordinal_ranges.insert(&serialize(&outpoint).try_into().unwrap(), &ordinals)?; } From 209aa9c13bb87aa87668cb865bf0cb4b1fa2bccb Mon Sep 17 00:00:00 2001 From: raphjaph Date: Tue, 13 Sep 2022 14:16:21 +0200 Subject: [PATCH 2/8] disambiguate between an output and a utxo --- src/index.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.rs b/src/index.rs index 85c8f466f8..f5b2996f1e 100644 --- a/src/index.rs +++ b/src/index.rs @@ -90,13 +90,13 @@ impl Index { .map(|(height, _hash)| height + 1) .unwrap_or(0); - let outputs_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; + let utxos_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; // Double unwrap, let's go !! let outputs_traversed = wtx.open_table(OUTPUTS_TRAVERSED)?.get(&0).unwrap().unwrap(); let stats = wtx.stats()?; println!("blocks indexed: {}", blocks_indexed); - println!("outputs indexed: {}", outputs_indexed); + println!("utxos indexed: {}", utxos_indexed); println!("outputs traversed: {}", outputs_traversed); println!("tree height: {}", stats.tree_height()); println!("free pages: {}", stats.free_pages()); From c155f5e8fb043618456757dea496e19331b273e9 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Tue, 13 Sep 2022 14:23:06 +0200 Subject: [PATCH 3/8] update test --- tests/info.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/info.rs b/tests/info.rs index bc0321de8c..704a7b62ec 100644 --- a/tests/info.rs +++ b/tests/info.rs @@ -9,7 +9,8 @@ fn basic() { .stdout_regex( r" blocks indexed: 1 - outputs indexed: 1 + utxos indexed: 1 + outputs traversed: 1 tree height: \d+ free pages: \d+ stored: .* From 3f44dd01eea15fa9c23a0b3cc9386334303c3253 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Tue, 13 Sep 2022 17:45:04 +0200 Subject: [PATCH 4/8] changed key of table to str --- src/index.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/index.rs b/src/index.rs index f5b2996f1e..c3ca4e111d 100644 --- a/src/index.rs +++ b/src/index.rs @@ -13,8 +13,7 @@ const OUTPOINT_TO_ORDINAL_RANGES: TableDefinition<[u8; 36], [u8]> = TableDefinition::new("OUTPOINT_TO_ORDINAL_RANGES"); const OUTPOINT_TO_TXID: TableDefinition<[u8; 36], [u8; 32]> = TableDefinition::new("OUTPOINT_TO_TXID"); -// Don't know if we need a whole table, cause I just want to store one value persistently. -const OUTPUTS_TRAVERSED: TableDefinition = TableDefinition::new("TOTAL_OUTPUTS_TRAVERSED"); +const OUTPUTS_TRAVERSED: TableDefinition = TableDefinition::new("OUTPUTS_TRAVERSED"); pub(crate) struct Index { client: Client, @@ -58,7 +57,7 @@ impl Index { tx.open_table(HEIGHT_TO_HASH)?; tx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; tx.open_table(OUTPOINT_TO_TXID)?; - tx.open_table(OUTPUTS_TRAVERSED)?; // set default value if empty? This looks really wrong + tx.open_table(OUTPUTS_TRAVERSED)?; tx.commit()?; @@ -91,8 +90,12 @@ impl Index { .unwrap_or(0); let utxos_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; - // Double unwrap, let's go !! - let outputs_traversed = wtx.open_table(OUTPUTS_TRAVERSED)?.get(&0).unwrap().unwrap(); + + let outputs_traversed = wtx + .open_table(OUTPUTS_TRAVERSED)? + .get("outputs_traversed")? + .unwrap_or(0); + let stats = wtx.stats()?; println!("blocks indexed: {}", blocks_indexed); @@ -275,13 +278,13 @@ impl Index { height_to_hash.insert(&height, &block.block_hash())?; - // TODO: error handling - if outputs_traversed.is_empty().unwrap() { - outputs_traversed.insert(&0, &0)?; + // TODO: how to set default differently? + if outputs_traversed.is_empty()? { + outputs_traversed.insert("outputs_traversed", &0)?; } - // TODO: double unwrap - let sum = outputs_in_block + outputs_traversed.get(&0).unwrap().unwrap(); - outputs_traversed.insert(&0, &sum)?; + // TODO: error handling + let sum = outputs_in_block + outputs_traversed.get("outputs_traversed")?.unwrap(); + outputs_traversed.insert("outputs_traversed", &sum)?; log::info!( "Wrote {ordinal_ranges_written} ordinal ranges in {}ms", From 70075d992c2ded9bb8e9ad665ba6ac215c3dbee0 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Thu, 15 Sep 2022 01:26:43 +0200 Subject: [PATCH 5/8] Apply suggestions from code review Co-authored-by: Casey Rodarmor --- src/index.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/index.rs b/src/index.rs index c3ca4e111d..9d77880871 100644 --- a/src/index.rs +++ b/src/index.rs @@ -13,7 +13,7 @@ const OUTPOINT_TO_ORDINAL_RANGES: TableDefinition<[u8; 36], [u8]> = TableDefinition::new("OUTPOINT_TO_ORDINAL_RANGES"); const OUTPOINT_TO_TXID: TableDefinition<[u8; 36], [u8; 32]> = TableDefinition::new("OUTPOINT_TO_TXID"); -const OUTPUTS_TRAVERSED: TableDefinition = TableDefinition::new("OUTPUTS_TRAVERSED"); +const STATISTICS: TableDefinition = TableDefinition::new("STATISTICS"); pub(crate) struct Index { client: Client, @@ -57,7 +57,7 @@ impl Index { tx.open_table(HEIGHT_TO_HASH)?; tx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; tx.open_table(OUTPOINT_TO_TXID)?; - tx.open_table(OUTPUTS_TRAVERSED)?; + tx.open_table(STATISTICS)?; tx.commit()?; @@ -92,8 +92,8 @@ impl Index { let utxos_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; let outputs_traversed = wtx - .open_table(OUTPUTS_TRAVERSED)? - .get("outputs_traversed")? + .open_table(STATISTICS)? + .get(Statistics::OutputsTraversed.into())? .unwrap_or(0); let stats = wtx.stats()?; @@ -164,7 +164,7 @@ impl Index { let mut height_to_hash = wtx.open_table(HEIGHT_TO_HASH)?; let mut outpoint_to_ordinal_ranges = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; let mut outpoint_to_txid = wtx.open_table(OUTPOINT_TO_TXID)?; - let mut outputs_traversed = wtx.open_table(OUTPUTS_TRAVERSED)?; + let mut statistics = wtx.open_table(STATISTICS)?; let start = Instant::now(); let mut ordinal_ranges_written = 0; @@ -278,10 +278,6 @@ impl Index { height_to_hash.insert(&height, &block.block_hash())?; - // TODO: how to set default differently? - if outputs_traversed.is_empty()? { - outputs_traversed.insert("outputs_traversed", &0)?; - } // TODO: error handling let sum = outputs_in_block + outputs_traversed.get("outputs_traversed")?.unwrap(); outputs_traversed.insert("outputs_traversed", &sum)?; From 59661be1a80bbee3b7a852ca3f2eb77cf6023793 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Thu, 15 Sep 2022 02:18:38 +0200 Subject: [PATCH 6/8] added statistics table --- src/index.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/index.rs b/src/index.rs index 9d77880871..f655b17450 100644 --- a/src/index.rs +++ b/src/index.rs @@ -27,6 +27,11 @@ pub(crate) enum List { Unspent(Vec<(u64, u64)>), } +#[repr(u64)] +enum Statistics { + OutputsTraversed = 0, +} + impl Index { pub(crate) fn open(options: &Options) -> Result { let rpc_url = options.rpc_url(); @@ -91,10 +96,8 @@ impl Index { let utxos_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; - let outputs_traversed = wtx - .open_table(STATISTICS)? - .get(Statistics::OutputsTraversed.into())? - .unwrap_or(0); + let key = Statistics::OutputsTraversed as u64; + let outputs_traversed = wtx.open_table(STATISTICS)?.get(&key)?.unwrap_or(0); let stats = wtx.stats()?; @@ -278,9 +281,13 @@ impl Index { height_to_hash.insert(&height, &block.block_hash())?; - // TODO: error handling - let sum = outputs_in_block + outputs_traversed.get("outputs_traversed")?.unwrap(); - outputs_traversed.insert("outputs_traversed", &sum)?; + statistics.insert( + &(Statistics::OutputsTraversed as u64), + &(statistics + .get(&(Statistics::OutputsTraversed as u64))? + .unwrap_or(0) + + outputs_in_block), + )?; log::info!( "Wrote {ordinal_ranges_written} ordinal ranges in {}ms", From e1c22ea36288de8d7f277b94cf75d7b0ee5e6f95 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Thu, 15 Sep 2022 13:19:33 +0200 Subject: [PATCH 7/8] implemented From trait for for u64 --- src/index.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/index.rs b/src/index.rs index f655b17450..75f9280705 100644 --- a/src/index.rs +++ b/src/index.rs @@ -28,10 +28,16 @@ pub(crate) enum List { } #[repr(u64)] -enum Statistics { +enum Statistic { OutputsTraversed = 0, } +impl From for u64 { + fn from(statistic: Statistic) -> Self { + statistic as u64 + } +} + impl Index { pub(crate) fn open(options: &Options) -> Result { let rpc_url = options.rpc_url(); @@ -96,8 +102,10 @@ impl Index { let utxos_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; - let key = Statistics::OutputsTraversed as u64; - let outputs_traversed = wtx.open_table(STATISTICS)?.get(&key)?.unwrap_or(0); + let outputs_traversed = wtx + .open_table(STATISTICS)? + .get(&Statistic::OutputsTraversed.into())? + .unwrap_or(0); let stats = wtx.stats()?; @@ -282,9 +290,9 @@ impl Index { height_to_hash.insert(&height, &block.block_hash())?; statistics.insert( - &(Statistics::OutputsTraversed as u64), + &Statistic::OutputsTraversed.into(), &(statistics - .get(&(Statistics::OutputsTraversed as u64))? + .get(&(Statistic::OutputsTraversed.into()))? .unwrap_or(0) + outputs_in_block), )?; From 27d5e4b66d33dae0bcf159843b1cad82e43526aa Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 19 Sep 2022 19:51:33 +0200 Subject: [PATCH 8/8] update lockfile --- Cargo.lock | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c411a935d0..f4f8333203 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -615,6 +615,8 @@ dependencies = [ [[package]] name = "boilerplate" version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eea9ab3a1bd92007c73dfba0ab14e56d2ec6ac2e4e2e5b14b4e4a477a97d311" dependencies = [ "darling", "mime", @@ -1084,9 +1086,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" dependencies = [ "atty", "humantime", @@ -1804,9 +1806,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.132" +version = "0.2.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" [[package]] name = "libsqlite3-sys" @@ -3235,9 +3237,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" +checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" dependencies = [ "proc-macro2", "quote", @@ -3447,9 +3449,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" dependencies = [ "futures-core", "pin-project-lite",