From 5ec78778b9bac7723ad68fcec92ad5aa38d1181b Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 18:35:43 -0800 Subject: [PATCH 01/10] Fix empty block crash --- src/index.rs | 9 ++++++--- tests/find.rs | 10 ++++++++++ tests/integration.rs | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/index.rs b/src/index.rs index ba55b2f9cd..ae04817e1e 100644 --- a/src/index.rs +++ b/src/index.rs @@ -55,7 +55,11 @@ impl Index { coinbase_inputs.push_front((start.n(), (start + h.subsidy()).n())); } - for tx in &block.txdata[1..] { + for (i, tx) in block.txdata.iter().enumerate() { + if i == 0 { + continue; + } + let mut input_ordinal_ranges = VecDeque::new(); for input in &tx.input { let mut key = Vec::new(); @@ -100,8 +104,7 @@ impl Index { coinbase_inputs.extend(&input_ordinal_ranges); } - { - let tx = &block.txdata[0]; + if let Some(tx) = block.txdata.first() { for (vout, output) in tx.output.iter().enumerate() { let mut ordinals = Vec::new(); let mut remaining = output.value; diff --git a/tests/find.rs b/tests/find.rs index a643718794..0b925bf907 100644 --- a/tests/find.rs +++ b/tests/find.rs @@ -77,3 +77,13 @@ fn first_satoshi_spent_in_second_block_slot() -> Result { .transaction(&[(0, 0, 0)], 1) .run() } + +#[test] +fn regression_empty_block_crash() -> Result { + Test::new()? + .command("find --blocksdir blocks 0 --slot --as-of-height 1") + .block() + .block_no_coinbase() + .expected_stdout("0.0.0.0\n") + .run() +} diff --git a/tests/integration.rs b/tests/integration.rs index 93b0f6c7c9..89e2e24b5e 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -160,6 +160,25 @@ impl Test { self } + fn block_no_coinbase(mut self) -> Self { + if self.blocks.is_empty() { + self.blocks.push(genesis_block(Network::Bitcoin)); + } else { + self.blocks.push(Block { + header: BlockHeader { + version: 0, + prev_blockhash: self.blocks.last().unwrap().block_hash(), + merkle_root: Default::default(), + time: 0, + bits: 0, + nonce: 0, + }, + txdata: Vec::new(), + }); + } + self + } + fn transaction(mut self, slots: &[(usize, usize, u32)], output_count: u64) -> Self { let value = slots .iter() From 73bfcaa0641db7942b75f6f15fff7acd7552cf94 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 18:42:43 -0800 Subject: [PATCH 02/10] Use new block function --- tests/find.rs | 28 +++++++++++----------- tests/integration.rs | 55 ++++++++++++++++---------------------------- tests/list.rs | 26 ++++++++++----------- 3 files changed, 47 insertions(+), 62 deletions(-) diff --git a/tests/find.rs b/tests/find.rs index 0b925bf907..38597e1d11 100644 --- a/tests/find.rs +++ b/tests/find.rs @@ -5,7 +5,7 @@ fn first_satoshi() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 0") .expected_stdout("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0:0\n") - .block() + .block(true) .run() } @@ -14,7 +14,7 @@ fn first_satoshi_slot() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 0 --slot") .expected_stdout("0.0.0.0\n") - .block() + .block(true) .run() } @@ -23,7 +23,7 @@ fn second_satoshi() -> Result { Test::new()? .command("find --blocksdir blocks 1 --as-of-height 0") .expected_stdout("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0:1\n") - .block() + .block(true) .run() } @@ -32,7 +32,7 @@ fn second_satoshi_slot() -> Result { Test::new()? .command("find --blocksdir blocks 1 --as-of-height 0 --slot") .expected_stdout("0.0.0.1\n") - .block() + .block(true) .run() } @@ -41,8 +41,8 @@ fn first_satoshi_of_second_block() -> Result { Test::new()? .command("find --blocksdir blocks 5000000000 --as-of-height 1") .expected_stdout("9068a11b8769174363376b606af9a4b8b29dd7b13d013f4b0cbbd457db3c3ce5:0:0\n") - .block() - .block() + .block(true) + .block(true) .run() } @@ -51,8 +51,8 @@ fn first_satoshi_of_second_block_slot() -> Result { Test::new()? .command("find --blocksdir blocks 5000000000 --as-of-height 1 --slot") .expected_stdout("1.0.0.0\n") - .block() - .block() + .block(true) + .block(true) .run() } @@ -61,8 +61,8 @@ fn first_satoshi_spent_in_second_block() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 1") .expected_stdout("72e60639a1dcc6263ed214a1db0dc9545bf65d9327e5a60e84bd3db7fbb4c2fa:0:0\n") - .block() - .block() + .block(true) + .block(true) .transaction(&[(0, 0, 0)], 1) .run() } @@ -72,8 +72,8 @@ fn first_satoshi_spent_in_second_block_slot() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 1 --slot") .expected_stdout("1.1.0.0\n") - .block() - .block() + .block(true) + .block(true) .transaction(&[(0, 0, 0)], 1) .run() } @@ -82,8 +82,8 @@ fn first_satoshi_spent_in_second_block_slot() -> Result { fn regression_empty_block_crash() -> Result { Test::new()? .command("find --blocksdir blocks 0 --slot --as-of-height 1") - .block() - .block_no_coinbase() + .block(true) + .block(false) .expected_stdout("0.0.0.0\n") .run() } diff --git a/tests/integration.rs b/tests/integration.rs index 89e2e24b5e..a769781557 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -126,7 +126,7 @@ impl Test { Ok(stdout.to_owned()) } - fn block(mut self) -> Self { + fn block(mut self, coinbase: bool) -> Self { if self.blocks.is_empty() { self.blocks.push(genesis_block(Network::Bitcoin)); } else { @@ -139,41 +139,26 @@ impl Test { bits: 0, nonce: 0, }, - txdata: vec![Transaction { - version: 0, - lock_time: 0, - input: vec![TxIn { - previous_output: OutPoint::null(), - script_sig: script::Builder::new() - .push_scriptint(self.blocks.len().try_into().unwrap()) - .into_script(), - sequence: 0, - witness: vec![], - }], - output: vec![TxOut { - value: 50 * COIN_VALUE, - script_pubkey: script::Builder::new().into_script(), - }], - }], - }); - } - self - } - - fn block_no_coinbase(mut self) -> Self { - if self.blocks.is_empty() { - self.blocks.push(genesis_block(Network::Bitcoin)); - } else { - self.blocks.push(Block { - header: BlockHeader { - version: 0, - prev_blockhash: self.blocks.last().unwrap().block_hash(), - merkle_root: Default::default(), - time: 0, - bits: 0, - nonce: 0, + txdata: if coinbase { + vec![Transaction { + version: 0, + lock_time: 0, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: script::Builder::new() + .push_scriptint(self.blocks.len().try_into().unwrap()) + .into_script(), + sequence: 0, + witness: vec![], + }], + output: vec![TxOut { + value: 50 * COIN_VALUE, + script_pubkey: script::Builder::new().into_script(), + }], + }] + } else { + Vec::new() }, - txdata: Vec::new(), }); } self diff --git a/tests/list.rs b/tests/list.rs index a2284c89a2..397baa3bb6 100644 --- a/tests/list.rs +++ b/tests/list.rs @@ -6,7 +6,7 @@ fn first_coinbase_transaction() -> Result { .command( "list --blocksdir blocks 4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0", ) - .block() + .block(true) .expected_stdout("[0,5000000000)\n") .run() } @@ -17,8 +17,8 @@ fn second_coinbase_transaction() -> Result { .command( "list --blocksdir blocks 9068a11b8769174363376b606af9a4b8b29dd7b13d013f4b0cbbd457db3c3ce5:0", ) - .block() - .block() + .block(true) + .block(true) .expected_stdout("[5000000000,10000000000)\n") .run() } @@ -29,9 +29,9 @@ fn third_coinbase_transaction_is_not_duplicate() -> Result { .command( "list --blocksdir blocks 8aa5103b13b5b233ac417ee31f21820c9284af2b7a2080a142c2d20e1697b0f4:0", ) - .block() - .block() - .block() + .block(true) + .block(true) + .block(true) .expected_stdout("[10000000000,15000000000)\n") .run() } @@ -42,8 +42,8 @@ fn split_ranges_are_tracked_correctly() -> Result { .command( "list --blocksdir blocks 7ab338c0e46c95c119ba8ff0a3f6cf64f56f35ba5553a399fdd38169cca00be3:0", ) - .block() - .block() + .block(true) + .block(true) .transaction(&[(0, 0, 0)], 2) .expected_stdout("[0,2500000000)\n") .run()?; @@ -52,8 +52,8 @@ fn split_ranges_are_tracked_correctly() -> Result { .command( "list --blocksdir blocks 7ab338c0e46c95c119ba8ff0a3f6cf64f56f35ba5553a399fdd38169cca00be3:1", ) - .block() - .block() + .block(true) + .block(true) .transaction(&[(0, 0, 0)], 2) .expected_stdout("[2500000000,5000000000)\n") .run() @@ -65,10 +65,10 @@ fn merge_ranges_are_tracked_correctly() -> Result { .command( "list --blocksdir blocks fe283c08e46269a7bbe36b629bc2be55d604152419818e94330477c9c3487eec:0", ) - .block() - .block() + .block(true) + .block(true) .transaction(&[(0, 0, 0)], 2) - .block() + .block(true) .transaction(&[(1, 1, 0), (1, 1, 1)], 1) .expected_stdout("[0,2500000000)\n[2500000000,5000000000)\n") .run() From 69b20bdc715f2905be0ea6a58b6816ba9b0fda59 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 18:43:25 -0800 Subject: [PATCH 03/10] tweak --- src/index.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/index.rs b/src/index.rs index ae04817e1e..5b38492beb 100644 --- a/src/index.rs +++ b/src/index.rs @@ -55,11 +55,7 @@ impl Index { coinbase_inputs.push_front((start.n(), (start + h.subsidy()).n())); } - for (i, tx) in block.txdata.iter().enumerate() { - if i == 0 { - continue; - } - + for tx in block.txdata.iter().skip(1) { let mut input_ordinal_ranges = VecDeque::new(); for input in &tx.input { let mut key = Vec::new(); From efa8f6e1626a452a66657b77230c39aff495839d Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 18:46:38 -0800 Subject: [PATCH 04/10] tweak --- tests/find.rs | 28 +++++++++++----------- tests/integration.rs | 55 ++++++++++++++++++++++++++++---------------- tests/list.rs | 26 ++++++++++----------- 3 files changed, 62 insertions(+), 47 deletions(-) diff --git a/tests/find.rs b/tests/find.rs index 38597e1d11..2b576b175b 100644 --- a/tests/find.rs +++ b/tests/find.rs @@ -5,7 +5,7 @@ fn first_satoshi() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 0") .expected_stdout("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0:0\n") - .block(true) + .block() .run() } @@ -14,7 +14,7 @@ fn first_satoshi_slot() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 0 --slot") .expected_stdout("0.0.0.0\n") - .block(true) + .block() .run() } @@ -23,7 +23,7 @@ fn second_satoshi() -> Result { Test::new()? .command("find --blocksdir blocks 1 --as-of-height 0") .expected_stdout("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0:1\n") - .block(true) + .block() .run() } @@ -32,7 +32,7 @@ fn second_satoshi_slot() -> Result { Test::new()? .command("find --blocksdir blocks 1 --as-of-height 0 --slot") .expected_stdout("0.0.0.1\n") - .block(true) + .block() .run() } @@ -41,8 +41,8 @@ fn first_satoshi_of_second_block() -> Result { Test::new()? .command("find --blocksdir blocks 5000000000 --as-of-height 1") .expected_stdout("9068a11b8769174363376b606af9a4b8b29dd7b13d013f4b0cbbd457db3c3ce5:0:0\n") - .block(true) - .block(true) + .block() + .block() .run() } @@ -51,8 +51,8 @@ fn first_satoshi_of_second_block_slot() -> Result { Test::new()? .command("find --blocksdir blocks 5000000000 --as-of-height 1 --slot") .expected_stdout("1.0.0.0\n") - .block(true) - .block(true) + .block() + .block() .run() } @@ -61,8 +61,8 @@ fn first_satoshi_spent_in_second_block() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 1") .expected_stdout("72e60639a1dcc6263ed214a1db0dc9545bf65d9327e5a60e84bd3db7fbb4c2fa:0:0\n") - .block(true) - .block(true) + .block() + .block() .transaction(&[(0, 0, 0)], 1) .run() } @@ -72,8 +72,8 @@ fn first_satoshi_spent_in_second_block_slot() -> Result { Test::new()? .command("find --blocksdir blocks 0 --as-of-height 1 --slot") .expected_stdout("1.1.0.0\n") - .block(true) - .block(true) + .block() + .block() .transaction(&[(0, 0, 0)], 1) .run() } @@ -82,8 +82,8 @@ fn first_satoshi_spent_in_second_block_slot() -> Result { fn regression_empty_block_crash() -> Result { Test::new()? .command("find --blocksdir blocks 0 --slot --as-of-height 1") - .block(true) - .block(false) + .block() + .block_without_coinbase() .expected_stdout("0.0.0.0\n") .run() } diff --git a/tests/integration.rs b/tests/integration.rs index a769781557..66d39ea968 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -126,7 +126,7 @@ impl Test { Ok(stdout.to_owned()) } - fn block(mut self, coinbase: bool) -> Self { + fn block(mut self) -> Self { if self.blocks.is_empty() { self.blocks.push(genesis_block(Network::Bitcoin)); } else { @@ -139,26 +139,41 @@ impl Test { bits: 0, nonce: 0, }, - txdata: if coinbase { - vec![Transaction { - version: 0, - lock_time: 0, - input: vec![TxIn { - previous_output: OutPoint::null(), - script_sig: script::Builder::new() - .push_scriptint(self.blocks.len().try_into().unwrap()) - .into_script(), - sequence: 0, - witness: vec![], - }], - output: vec![TxOut { - value: 50 * COIN_VALUE, - script_pubkey: script::Builder::new().into_script(), - }], - }] - } else { - Vec::new() + txdata: vec![Transaction { + version: 0, + lock_time: 0, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: script::Builder::new() + .push_scriptint(self.blocks.len().try_into().unwrap()) + .into_script(), + sequence: 0, + witness: vec![], + }], + output: vec![TxOut { + value: 50 * COIN_VALUE, + script_pubkey: script::Builder::new().into_script(), + }], + }], + }); + } + self + } + + fn block_without_coinbase(mut self) -> Self { + if self.blocks.is_empty() { + self.blocks.push(genesis_block(Network::Bitcoin)); + } else { + self.blocks.push(Block { + header: BlockHeader { + version: 0, + prev_blockhash: self.blocks.last().unwrap().block_hash(), + merkle_root: Default::default(), + time: 0, + bits: 0, + nonce: 0, }, + txdata: Vec::new(), }); } self diff --git a/tests/list.rs b/tests/list.rs index 397baa3bb6..a2284c89a2 100644 --- a/tests/list.rs +++ b/tests/list.rs @@ -6,7 +6,7 @@ fn first_coinbase_transaction() -> Result { .command( "list --blocksdir blocks 4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0", ) - .block(true) + .block() .expected_stdout("[0,5000000000)\n") .run() } @@ -17,8 +17,8 @@ fn second_coinbase_transaction() -> Result { .command( "list --blocksdir blocks 9068a11b8769174363376b606af9a4b8b29dd7b13d013f4b0cbbd457db3c3ce5:0", ) - .block(true) - .block(true) + .block() + .block() .expected_stdout("[5000000000,10000000000)\n") .run() } @@ -29,9 +29,9 @@ fn third_coinbase_transaction_is_not_duplicate() -> Result { .command( "list --blocksdir blocks 8aa5103b13b5b233ac417ee31f21820c9284af2b7a2080a142c2d20e1697b0f4:0", ) - .block(true) - .block(true) - .block(true) + .block() + .block() + .block() .expected_stdout("[10000000000,15000000000)\n") .run() } @@ -42,8 +42,8 @@ fn split_ranges_are_tracked_correctly() -> Result { .command( "list --blocksdir blocks 7ab338c0e46c95c119ba8ff0a3f6cf64f56f35ba5553a399fdd38169cca00be3:0", ) - .block(true) - .block(true) + .block() + .block() .transaction(&[(0, 0, 0)], 2) .expected_stdout("[0,2500000000)\n") .run()?; @@ -52,8 +52,8 @@ fn split_ranges_are_tracked_correctly() -> Result { .command( "list --blocksdir blocks 7ab338c0e46c95c119ba8ff0a3f6cf64f56f35ba5553a399fdd38169cca00be3:1", ) - .block(true) - .block(true) + .block() + .block() .transaction(&[(0, 0, 0)], 2) .expected_stdout("[2500000000,5000000000)\n") .run() @@ -65,10 +65,10 @@ fn merge_ranges_are_tracked_correctly() -> Result { .command( "list --blocksdir blocks fe283c08e46269a7bbe36b629bc2be55d604152419818e94330477c9c3487eec:0", ) - .block(true) - .block(true) + .block() + .block() .transaction(&[(0, 0, 0)], 2) - .block(true) + .block() .transaction(&[(1, 1, 0), (1, 1, 1)], 1) .expected_stdout("[0,2500000000)\n[2500000000,5000000000)\n") .run() From c8d9952971937a5aab82a6669ca25fb08e4c44de Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 19:03:12 -0800 Subject: [PATCH 05/10] Write failing test for indexing multiple blockfiles --- tests/find.rs | 12 ++++++++++++ tests/integration.rs | 32 +++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/tests/find.rs b/tests/find.rs index 2b576b175b..7cf5c71ba7 100644 --- a/tests/find.rs +++ b/tests/find.rs @@ -87,3 +87,15 @@ fn regression_empty_block_crash() -> Result { .expected_stdout("0.0.0.0\n") .run() } + +#[test] +fn index_multiple_blockfiles() -> Result { + Test::new()? + .command("find --blocksdir blocks 0 --as-of-height 1 --slot") + .expected_stdout("1.1.0.0\n") + .block() + .end_blockfile() + .block() + .transaction(&[(0, 0, 0)], 1) + .run() +} diff --git a/tests/integration.rs b/tests/integration.rs index 66d39ea968..e097ed2c0b 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -30,24 +30,26 @@ type Result = std::result::Result>; struct Test { args: Vec, - expected_stdout: String, - expected_stderr: String, + blockfile_ends: Vec, + blocks: Vec, expected_status: i32, + expected_stderr: String, + expected_stdout: String, ignore_stdout: bool, tempdir: TempDir, - blocks: Vec, } impl Test { fn new() -> Result { Ok(Self { args: Vec::new(), - expected_stdout: String::new(), - expected_stderr: String::new(), + blockfile_ends: Vec::new(), + blocks: Vec::new(), expected_status: 0, + expected_stderr: String::new(), + expected_stdout: String::new(), ignore_stdout: false, tempdir: TempDir::new()?, - blocks: Vec::new(), }) } @@ -214,12 +216,28 @@ impl Test { self } + fn end_blockfile(mut self) -> Self { + self.blockfile_ends.push(self.blocks.len()); + self + } + fn populate_blocksdir(&self) -> io::Result<()> { let blocksdir = self.tempdir.path().join("blocks"); fs::create_dir(&blocksdir)?; + let mut blockfile = File::create(blocksdir.join("blk00000.dat"))?; - for block in &self.blocks { + for (i, block) in self.blocks.iter().enumerate() { + if i + >= self + .blockfile_ends + .last() + .copied() + .unwrap_or(usize::max_value()) + { + blockfile = File::create(blocksdir.join(format!("blk{:05}.dat", i)))?; + } + let mut encoded = Vec::new(); block.consensus_encode(&mut encoded)?; blockfile.write_all(&[0xf9, 0xbe, 0xb4, 0xd9])?; From 9c1356b5b6c59ad24664a66ae55684f123d173c2 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 22:57:23 -0800 Subject: [PATCH 06/10] Read multiple blockfiles --- src/index.rs | 42 ++++++++--------- src/main.rs | 2 +- tests/find.rs | 4 +- tests/integration.rs | 105 ++++++++++++++++++++----------------------- 4 files changed, 73 insertions(+), 80 deletions(-) diff --git a/src/index.rs b/src/index.rs index 5b38492beb..f62eeee241 100644 --- a/src/index.rs +++ b/src/index.rs @@ -6,9 +6,9 @@ pub(crate) struct Index { } impl Index { + const HASH_TO_BLOCK: &'static str = "HASH_TO_BLOCK"; const HASH_TO_CHILDREN: &'static str = "HASH_TO_CHILDREN"; const HASH_TO_HEIGHT: &'static str = "HASH_TO_HEIGHT"; - const HASH_TO_OFFSET: &'static str = "HASH_TO_OFFSET"; const HEIGHT_TO_HASH: &'static str = "HEIGHT_TO_HASH"; const OUTPOINT_TO_ORDINAL_RANGES: &'static str = "OUTPOINT_TO_ORDINAL_RANGES"; @@ -138,15 +138,24 @@ impl Index { } fn index_blockfile(&self) -> Result { - { + for i in 0.. { + let blocks = match fs::read(self.blocksdir.join(format!("blk{:05}.dat", i))) { + Ok(blocks) => blocks, + Err(err) => { + if err.kind() == io::ErrorKind::NotFound { + break; + } else { + return Err(err.into()); + } + } + }; + let tx = self.database.begin_write()?; let mut hash_to_children: MultimapTable<[u8], [u8]> = tx.open_multimap_table(Self::HASH_TO_CHILDREN)?; - let mut hash_to_offset: Table<[u8], u64> = tx.open_table(Self::HASH_TO_OFFSET)?; - - let blocks = fs::read(self.blocksdir.join("blk00000.dat"))?; + let mut hash_to_block: Table<[u8], [u8]> = tx.open_table(Self::HASH_TO_BLOCK)?; let mut offset = 0; @@ -163,7 +172,7 @@ impl Index { hash_to_children.insert(&block.header.prev_blockhash, &block.block_hash())?; - hash_to_offset.insert(&block.block_hash(), &(offset as u64))?; + hash_to_block.insert(&block.block_hash(), &blocks[range.clone()])?; offset = range.end; @@ -224,15 +233,14 @@ impl Index { Some(guard) => { let hash = guard.to_value(); - let hash_to_offset: ReadOnlyTable<[u8], u64> = tx.open_table(Self::HASH_TO_OFFSET)?; - let offset = hash_to_offset - .get(hash)? - .ok_or("Could not find offset to block in index")? - .to_value() as usize; - - let blocks = fs::read(self.blocksdir.join("blk00000.dat"))?; + let hash_to_block: ReadOnlyTable<[u8], [u8]> = tx.open_table(Self::HASH_TO_BLOCK)?; - Ok(Some(Self::decode_block_at(&blocks, offset)?)) + Ok(Some(Block::consensus_decode( + hash_to_block + .get(hash)? + .ok_or("Could not find block in index")? + .to_value(), + )?)) } } } @@ -247,12 +255,6 @@ impl Index { Ok(offset..offset + len) } - fn decode_block_at(blocks: &[u8], offset: usize) -> Result { - Ok(Block::consensus_decode( - &blocks[Self::block_range_at(blocks, offset)?], - )?) - } - pub(crate) fn list(&self, outpoint: OutPoint) -> Result> { let rtx = self.database.begin_read()?; let outpoint_to_ordinal_ranges: ReadOnlyTable<[u8], [u8]> = diff --git a/src/main.rs b/src/main.rs index 06173b1c6a..974b1dc494 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ use { cmp::Ordering, collections::VecDeque, fmt::{self, Display, Formatter}, - fs, + fs, io, ops::{Add, AddAssign, Deref, Range, Sub}, path::{Path, PathBuf}, process, diff --git a/tests/find.rs b/tests/find.rs index 7cf5c71ba7..98859d09a6 100644 --- a/tests/find.rs +++ b/tests/find.rs @@ -83,7 +83,7 @@ fn regression_empty_block_crash() -> Result { Test::new()? .command("find --blocksdir blocks 0 --slot --as-of-height 1") .block() - .block_without_coinbase() + .block_with_coinbase(false) .expected_stdout("0.0.0.0\n") .run() } @@ -94,7 +94,7 @@ fn index_multiple_blockfiles() -> Result { .command("find --blocksdir blocks 0 --as-of-height 1 --slot") .expected_stdout("1.1.0.0\n") .block() - .end_blockfile() + .blockfile() .block() .transaction(&[(0, 0, 0)], 1) .run() diff --git a/tests/integration.rs b/tests/integration.rs index e097ed2c0b..392a2169d9 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -30,7 +30,7 @@ type Result = std::result::Result>; struct Test { args: Vec, - blockfile_ends: Vec, + blockfiles: Vec, blocks: Vec, expected_status: i32, expected_stderr: String, @@ -43,7 +43,7 @@ impl Test { fn new() -> Result { Ok(Self { args: Vec::new(), - blockfile_ends: Vec::new(), + blockfiles: Vec::new(), blocks: Vec::new(), expected_status: 0, expected_stderr: String::new(), @@ -128,41 +128,11 @@ impl Test { Ok(stdout.to_owned()) } - fn block(mut self) -> Self { - if self.blocks.is_empty() { - self.blocks.push(genesis_block(Network::Bitcoin)); - } else { - self.blocks.push(Block { - header: BlockHeader { - version: 0, - prev_blockhash: self.blocks.last().unwrap().block_hash(), - merkle_root: Default::default(), - time: 0, - bits: 0, - nonce: 0, - }, - txdata: vec![Transaction { - version: 0, - lock_time: 0, - input: vec![TxIn { - previous_output: OutPoint::null(), - script_sig: script::Builder::new() - .push_scriptint(self.blocks.len().try_into().unwrap()) - .into_script(), - sequence: 0, - witness: vec![], - }], - output: vec![TxOut { - value: 50 * COIN_VALUE, - script_pubkey: script::Builder::new().into_script(), - }], - }], - }); - } - self + fn block(self) -> Self { + self.block_with_coinbase(true) } - fn block_without_coinbase(mut self) -> Self { + fn block_with_coinbase(mut self, coinbase: bool) -> Self { if self.blocks.is_empty() { self.blocks.push(genesis_block(Network::Bitcoin)); } else { @@ -175,7 +145,26 @@ impl Test { bits: 0, nonce: 0, }, - txdata: Vec::new(), + txdata: if coinbase { + vec![Transaction { + version: 0, + lock_time: 0, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: script::Builder::new() + .push_scriptint(self.blocks.len().try_into().unwrap()) + .into_script(), + sequence: 0, + witness: vec![], + }], + output: vec![TxOut { + value: 50 * COIN_VALUE, + script_pubkey: script::Builder::new().into_script(), + }], + }] + } else { + Vec::new() + }, }); } self @@ -216,8 +205,8 @@ impl Test { self } - fn end_blockfile(mut self) -> Self { - self.blockfile_ends.push(self.blocks.len()); + fn blockfile(mut self) -> Self { + self.blockfiles.push(self.blocks.len()); self } @@ -225,27 +214,29 @@ impl Test { let blocksdir = self.tempdir.path().join("blocks"); fs::create_dir(&blocksdir)?; - let mut blockfile = File::create(blocksdir.join("blk00000.dat"))?; - - for (i, block) in self.blocks.iter().enumerate() { - if i - >= self - .blockfile_ends - .last() - .copied() - .unwrap_or(usize::max_value()) - { - blockfile = File::create(blocksdir.join(format!("blk{:05}.dat", i)))?; - } + let mut start = 0; - let mut encoded = Vec::new(); - block.consensus_encode(&mut encoded)?; - blockfile.write_all(&[0xf9, 0xbe, 0xb4, 0xd9])?; - blockfile.write_all(&(encoded.len() as u32).to_le_bytes())?; - blockfile.write_all(&encoded)?; - for tx in &block.txdata { - eprintln!("{}", tx.txid()); + for (i, end) in self + .blockfiles + .iter() + .copied() + .chain([self.blocks.len()]) + .enumerate() + { + let mut blockfile = File::create(blocksdir.join(format!("blk{:05}.dat", i)))?; + + for block in &self.blocks[start..end] { + let mut encoded = Vec::new(); + block.consensus_encode(&mut encoded)?; + blockfile.write_all(&[0xf9, 0xbe, 0xb4, 0xd9])?; + blockfile.write_all(&(encoded.len() as u32).to_le_bytes())?; + blockfile.write_all(&encoded)?; + for tx in &block.txdata { + eprintln!("{}", tx.txid()); + } } + + start = end; } Ok(()) From 1fe0a64f9b8f690dd347981232abb38e2ad5b4d4 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 23:02:43 -0800 Subject: [PATCH 07/10] Add scary caveat section to readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index a9e20dc08d..1b035e9476 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,13 @@ current owner can sign a message proving that they own a given UTXO, wich also serves as proof of ownership of all the NFTs assigned to satoshis within that UTXO. +## Index and Caveats + +The `ord` command builds an index using the contents of a local `bitcoind`'s +data directory, which must be halted while the index is built. Currently, the +index is built every time the `ord` runs, but that is a temporary limitation. +Reorgs are not currently properly handled. + ## Numbering Satoshis are assigned ordinal numbers in the order in which they are mined. From 5855ffe15d1ce2a8fbb0457ab7f00a4b5dacc6dd Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 23:07:39 -0800 Subject: [PATCH 08/10] More caveats --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b035e9476..42bd215612 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ the NFT creator would create a message that assigned a new NFT Y to the satoshi with ordinal X. The owner of the UTXO containing the satoshi with ordinal X owns NFT Y, and can transfer that ownership to another person with a transaction that sends ordinal Y to a UTXO that the new owner controls. The -current owner can sign a message proving that they own a given UTXO, wich also +current owner can sign a message proving that they own a given UTXO, which also serves as proof of ownership of all the NFTs assigned to satoshis within that UTXO. @@ -20,6 +20,10 @@ data directory, which must be halted while the index is built. Currently, the index is built every time the `ord` runs, but that is a temporary limitation. Reorgs are not currently properly handled. +The index is stored in `index.redb`, and should not be concurrently modified +while an instance of `ord` is running, or used by two `ord` instances +simultaneously. + ## Numbering Satoshis are assigned ordinal numbers in the order in which they are mined. From 9e6648511f98342e490dded79bf32740714b2775 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 23:08:18 -0800 Subject: [PATCH 09/10] tweak --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42bd215612..6f6c91dea5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ UTXO. The `ord` command builds an index using the contents of a local `bitcoind`'s data directory, which must be halted while the index is built. Currently, the index is built every time the `ord` runs, but that is a temporary limitation. -Reorgs are not currently properly handled. +Reorgs are also not properly handled. The index is stored in `index.redb`, and should not be concurrently modified while an instance of `ord` is running, or used by two `ord` instances From f5b1457ac15b55029aab85d6a7677fa3a1136fea Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 31 Jan 2022 23:11:20 -0800 Subject: [PATCH 10/10] Use iter once --- tests/integration.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration.rs b/tests/integration.rs index 392a2169d9..210f5b4460 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -11,6 +11,7 @@ use { error::Error, fs::{self, File}, io::{self, Write}, + iter, process::Command, str, }, @@ -220,7 +221,7 @@ impl Test { .blockfiles .iter() .copied() - .chain([self.blocks.len()]) + .chain(iter::once(self.blocks.len())) .enumerate() { let mut blockfile = File::create(blocksdir.join(format!("blk{:05}.dat", i)))?;