From 6ae6c27c00c081c457070c07e46bd980d10757c4 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Fri, 25 Nov 2022 11:22:07 -0800 Subject: [PATCH 1/2] Make analyzing sync time easier --- justfile | 3 +++ src/index.rs | 6 +++++- src/index/updater.rs | 12 ++++++------ src/options.rs | 2 ++ src/subcommand.rs | 4 ++-- src/subcommand/info.rs | 37 ++++++++++++++++++++++++++++++++----- tests/index.rs | 26 +++++++------------------- tests/info.rs | 41 ++++++++++++++++++++++++++++++++++++++++- tests/lib.rs | 1 + 9 files changed, 98 insertions(+), 34 deletions(-) diff --git a/justfile b/justfile index 1d778b2995..3493a56b82 100644 --- a/justfile +++ b/justfile @@ -95,6 +95,9 @@ download-log unit='ord' host='ordinals.com': ssh root@{{host}} 'mkdir -p tmp && journalctl -u {{unit}} > tmp/{{unit}}.log' rsync --progress --compress root@{{host}}:tmp/{{unit}}.log tmp/{{unit}}.log +download-index unit='ord' host='ordinals.com': + rsync --progress --compress root@{{host}}:/var/lib/{{unit}}/index.redb tmp/{{unit}}.index.redb + graph log: ./bin/graph $1 diff --git a/src/index.rs b/src/index.rs index e5c0685f71..d29e09b79a 100644 --- a/src/index.rs +++ b/src/index.rs @@ -156,7 +156,11 @@ impl Index { bail!("failed to create data dir `{}`: {err}", data_dir.display()); } - let database_path = data_dir.join("index.redb"); + let database_path = if let Some(database_path) = &options.index { + database_path.clone() + } else { + data_dir.join("index.redb") + }; let database = match unsafe { redb::Database::open(&database_path) } { Ok(database) => database, diff --git a/src/index/updater.rs b/src/index/updater.rs index a72f303f81..5a96b25f3f 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -73,8 +73,8 @@ impl Updater { let rx = Self::fetch_blocks_from(index, self.height)?; - let mut uncomitted = 0; - for i in 0.. { + let mut uncommitted = 0; + loop { let block = match rx.recv() { Ok(block) => block, Err(mpsc::RecvError) => break, @@ -90,11 +90,11 @@ impl Updater { } } - uncomitted += 1; + uncommitted += 1; - if i % 5000 == 0 { + if uncommitted == 5000 { self.commit(wtx)?; - uncomitted = 0; + uncommitted = 0; wtx = index.begin_write()?; let height = wtx .open_table(HEIGHT_TO_BLOCK_HASH)? @@ -124,7 +124,7 @@ impl Updater { } } - if uncomitted > 0 { + if uncommitted > 0 { self.commit(wtx)?; } diff --git a/src/options.rs b/src/options.rs index e823ba420f..feab9b80e0 100644 --- a/src/options.rs +++ b/src/options.rs @@ -15,6 +15,8 @@ pub(crate) struct Options { data_dir: Option, #[clap(long, help = "Limit index to blocks.")] pub(crate) height_limit: Option, + #[clap(long, help = "Use index at .")] + pub(crate) index: Option, #[clap(long, help = "Index ordinal ranges.")] pub(crate) index_ordinals: bool, #[clap(long, help = "Connect to Bitcoin Core RPC at .")] diff --git a/src/subcommand.rs b/src/subcommand.rs index a2a2b284d4..6f5853c7d6 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -17,7 +17,7 @@ pub(crate) enum Subcommand { Epochs, Find(find::Find), Index, - Info, + Info(info::Info), ListRanges(list::List), Parse(parse::Parse), Range(range::Range), @@ -34,7 +34,7 @@ impl Subcommand { Self::Epochs => epochs::run(), Self::Find(find) => find.run(options), Self::Index => index::run(options), - Self::Info => info::run(options), + Self::Info(info) => info.run(options), Self::ListRanges(list) => list.run(options), Self::Parse(parse) => parse.run(), Self::Range(range) => range.run(), diff --git a/src/subcommand/info.rs b/src/subcommand/info.rs index 2376747752..f0c12ef04a 100644 --- a/src/subcommand/info.rs +++ b/src/subcommand/info.rs @@ -1,8 +1,35 @@ use super::*; -pub(crate) fn run(options: Options) -> Result { - let index = Index::open(&options)?; - index.update()?; - serde_json::to_writer(io::stdout(), &index.info()?)?; - Ok(()) +#[derive(Debug, Parser)] +pub(crate) struct Info { + #[clap(long)] + transactions: bool, +} + +impl Info { + pub(crate) fn run(self, options: Options) -> Result { + let index = Index::open(&options)?; + index.update()?; + let info = index.info()?; + + if self.transactions { + println!("start\tend\tblocks\telapsed"); + + for window in info.transactions.windows(2) { + let start = &window[0]; + let end = &window[1]; + println!( + "{}\t{}\t{}\t{:.2}", + start.starting_block_count, + end.starting_block_count, + end.starting_block_count - start.starting_block_count, + (end.starting_timestamp - start.starting_timestamp) as f64 / 1000.0 / 60.0 + ); + } + } else { + serde_json::to_writer(io::stdout(), &info)?; + } + + Ok(()) + } } diff --git a/tests/index.rs b/tests/index.rs index ca4fd413c3..e9d5725375 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -1,29 +1,17 @@ use super::*; #[test] -fn custom_index_size() { +fn custom_index_path() { let rpc_server = test_bitcoincore_rpc::spawn(); rpc_server.mine_blocks(1); - let output = CommandBuilder::new("--max-index-size 1mib --index-ordinals find 0") + let tempdir = TempDir::new().unwrap(); + + let index_path = tempdir.path().join("foo.redb"); + + CommandBuilder::new(format!("--index {} index", index_path.display())) .rpc_server(&rpc_server) - .expected_stdout("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0:0\n") .run(); - assert_eq!( - output - .tempdir - .path() - .join(if cfg!(target_os = "macos") { - "Library/Application Support/" - } else { - ".local/share" - }) - .join("ord") - .join("index.redb") - .metadata() - .unwrap() - .len(), - 1 << 20 - ); + assert!(index_path.is_file()) } diff --git a/tests/info.rs b/tests/info.rs index 95a97fa315..8fd4d4e0af 100644 --- a/tests/info.rs +++ b/tests/info.rs @@ -1,7 +1,7 @@ use super::*; #[test] -fn initial() { +fn json() { let rpc_server = test_bitcoincore_rpc::spawn(); CommandBuilder::new("--index-ordinals info") .rpc_server(&rpc_server) @@ -10,3 +10,42 @@ fn initial() { ) .run(); } + +#[test] +fn transactions() { + let rpc_server = test_bitcoincore_rpc::spawn(); + + let tempdir = TempDir::new().unwrap(); + + let index_path = tempdir.path().join("index.redb"); + + CommandBuilder::new(format!( + "--index-ordinals --index {} info --transactions", + index_path.display() + )) + .rpc_server(&rpc_server) + .expected_stdout("start\tend\tblocks\telapsed\n") + .run(); + + rpc_server.mine_blocks(10); + + CommandBuilder::new(format!( + "--index-ordinals --index {} info --transactions", + index_path.display() + )) + .rpc_server(&rpc_server) + .stdout_regex("start\tend\tblocks\telapsed\n0\t1\t1\t\\d+\\.\\d+\n") + .run(); + + rpc_server.mine_blocks(10); + + CommandBuilder::new(format!( + "--index-ordinals --index {} info --transactions", + index_path.display() + )) + .rpc_server(&rpc_server) + .expected_stdout("start\tend\tblocks\telapsed\n") + .stdout_regex("start\tend\tblocks\telapsed\n0\t1\t1\t\\d+\\.\\d+\n") + .stdout_regex("start\tend\tblocks\telapsed\n0\t1\t1\t\\d+\\.\\d+\n1\t11\t10\t\\d+\\.\\d+\n") + .run(); +} diff --git a/tests/lib.rs b/tests/lib.rs index f4600bc465..3108225a9b 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -37,6 +37,7 @@ mod command_builder; mod epochs; mod expected; mod find; +mod index; mod info; mod list; mod parse; From 8b6e28e0ba258c8436ab0c3597470e16a9fee53c Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Fri, 25 Nov 2022 12:10:05 -0800 Subject: [PATCH 2/2] blocks -> count --- src/subcommand/info.rs | 2 +- tests/info.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/subcommand/info.rs b/src/subcommand/info.rs index f0c12ef04a..73ab5d6675 100644 --- a/src/subcommand/info.rs +++ b/src/subcommand/info.rs @@ -13,7 +13,7 @@ impl Info { let info = index.info()?; if self.transactions { - println!("start\tend\tblocks\telapsed"); + println!("start\tend\tcount\telapsed"); for window in info.transactions.windows(2) { let start = &window[0]; diff --git a/tests/info.rs b/tests/info.rs index 8fd4d4e0af..523108d9cf 100644 --- a/tests/info.rs +++ b/tests/info.rs @@ -24,7 +24,7 @@ fn transactions() { index_path.display() )) .rpc_server(&rpc_server) - .expected_stdout("start\tend\tblocks\telapsed\n") + .expected_stdout("start\tend\tcount\telapsed\n") .run(); rpc_server.mine_blocks(10); @@ -34,7 +34,7 @@ fn transactions() { index_path.display() )) .rpc_server(&rpc_server) - .stdout_regex("start\tend\tblocks\telapsed\n0\t1\t1\t\\d+\\.\\d+\n") + .stdout_regex("start\tend\tcount\telapsed\n0\t1\t1\t\\d+\\.\\d+\n") .run(); rpc_server.mine_blocks(10); @@ -44,8 +44,8 @@ fn transactions() { index_path.display() )) .rpc_server(&rpc_server) - .expected_stdout("start\tend\tblocks\telapsed\n") - .stdout_regex("start\tend\tblocks\telapsed\n0\t1\t1\t\\d+\\.\\d+\n") - .stdout_regex("start\tend\tblocks\telapsed\n0\t1\t1\t\\d+\\.\\d+\n1\t11\t10\t\\d+\\.\\d+\n") + .expected_stdout("start\tend\tcount\telapsed\n") + .stdout_regex("start\tend\tcount\telapsed\n0\t1\t1\t\\d+\\.\\d+\n") + .stdout_regex("start\tend\tcount\telapsed\n0\t1\t1\t\\d+\\.\\d+\n1\t11\t10\t\\d+\\.\\d+\n") .run(); }