diff --git a/README.md b/README.md index a9e20dc08d..6f6c91dea5 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,21 @@ the NFT creator would create a message that assigned a new NFT Y to the satoshi with ordinal X. The owner of the UTXO containing the satoshi with ordinal X owns NFT Y, and can transfer that ownership to another person with a transaction that sends ordinal Y to a UTXO that the new owner controls. The -current owner can sign a message proving that they own a given UTXO, wich also +current owner can sign a message proving that they own a given UTXO, which also serves as proof of ownership of all the NFTs assigned to satoshis within that UTXO. +## Index and Caveats + +The `ord` command builds an index using the contents of a local `bitcoind`'s +data directory, which must be halted while the index is built. Currently, the +index is built every time the `ord` runs, but that is a temporary limitation. +Reorgs are also not properly handled. + +The index is stored in `index.redb`, and should not be concurrently modified +while an instance of `ord` is running, or used by two `ord` instances +simultaneously. + ## Numbering Satoshis are assigned ordinal numbers in the order in which they are mined. diff --git a/src/index.rs b/src/index.rs index 5b38492beb..f62eeee241 100644 --- a/src/index.rs +++ b/src/index.rs @@ -6,9 +6,9 @@ pub(crate) struct Index { } impl Index { + const HASH_TO_BLOCK: &'static str = "HASH_TO_BLOCK"; const HASH_TO_CHILDREN: &'static str = "HASH_TO_CHILDREN"; const HASH_TO_HEIGHT: &'static str = "HASH_TO_HEIGHT"; - const HASH_TO_OFFSET: &'static str = "HASH_TO_OFFSET"; const HEIGHT_TO_HASH: &'static str = "HEIGHT_TO_HASH"; const OUTPOINT_TO_ORDINAL_RANGES: &'static str = "OUTPOINT_TO_ORDINAL_RANGES"; @@ -138,15 +138,24 @@ impl Index { } fn index_blockfile(&self) -> Result { - { + for i in 0.. { + let blocks = match fs::read(self.blocksdir.join(format!("blk{:05}.dat", i))) { + Ok(blocks) => blocks, + Err(err) => { + if err.kind() == io::ErrorKind::NotFound { + break; + } else { + return Err(err.into()); + } + } + }; + let tx = self.database.begin_write()?; let mut hash_to_children: MultimapTable<[u8], [u8]> = tx.open_multimap_table(Self::HASH_TO_CHILDREN)?; - let mut hash_to_offset: Table<[u8], u64> = tx.open_table(Self::HASH_TO_OFFSET)?; - - let blocks = fs::read(self.blocksdir.join("blk00000.dat"))?; + let mut hash_to_block: Table<[u8], [u8]> = tx.open_table(Self::HASH_TO_BLOCK)?; let mut offset = 0; @@ -163,7 +172,7 @@ impl Index { hash_to_children.insert(&block.header.prev_blockhash, &block.block_hash())?; - hash_to_offset.insert(&block.block_hash(), &(offset as u64))?; + hash_to_block.insert(&block.block_hash(), &blocks[range.clone()])?; offset = range.end; @@ -224,15 +233,14 @@ impl Index { Some(guard) => { let hash = guard.to_value(); - let hash_to_offset: ReadOnlyTable<[u8], u64> = tx.open_table(Self::HASH_TO_OFFSET)?; - let offset = hash_to_offset - .get(hash)? - .ok_or("Could not find offset to block in index")? - .to_value() as usize; - - let blocks = fs::read(self.blocksdir.join("blk00000.dat"))?; + let hash_to_block: ReadOnlyTable<[u8], [u8]> = tx.open_table(Self::HASH_TO_BLOCK)?; - Ok(Some(Self::decode_block_at(&blocks, offset)?)) + Ok(Some(Block::consensus_decode( + hash_to_block + .get(hash)? + .ok_or("Could not find block in index")? + .to_value(), + )?)) } } } @@ -247,12 +255,6 @@ impl Index { Ok(offset..offset + len) } - fn decode_block_at(blocks: &[u8], offset: usize) -> Result { - Ok(Block::consensus_decode( - &blocks[Self::block_range_at(blocks, offset)?], - )?) - } - pub(crate) fn list(&self, outpoint: OutPoint) -> Result> { let rtx = self.database.begin_read()?; let outpoint_to_ordinal_ranges: ReadOnlyTable<[u8], [u8]> = diff --git a/src/main.rs b/src/main.rs index 06173b1c6a..974b1dc494 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ use { cmp::Ordering, collections::VecDeque, fmt::{self, Display, Formatter}, - fs, + fs, io, ops::{Add, AddAssign, Deref, Range, Sub}, path::{Path, PathBuf}, process, diff --git a/tests/find.rs b/tests/find.rs index 2b576b175b..98859d09a6 100644 --- a/tests/find.rs +++ b/tests/find.rs @@ -83,7 +83,19 @@ fn regression_empty_block_crash() -> Result { Test::new()? .command("find --blocksdir blocks 0 --slot --as-of-height 1") .block() - .block_without_coinbase() + .block_with_coinbase(false) .expected_stdout("0.0.0.0\n") .run() } + +#[test] +fn index_multiple_blockfiles() -> Result { + Test::new()? + .command("find --blocksdir blocks 0 --as-of-height 1 --slot") + .expected_stdout("1.1.0.0\n") + .block() + .blockfile() + .block() + .transaction(&[(0, 0, 0)], 1) + .run() +} diff --git a/tests/integration.rs b/tests/integration.rs index 66d39ea968..210f5b4460 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -11,6 +11,7 @@ use { error::Error, fs::{self, File}, io::{self, Write}, + iter, process::Command, str, }, @@ -30,24 +31,26 @@ type Result = std::result::Result>; struct Test { args: Vec, - expected_stdout: String, - expected_stderr: String, + blockfiles: Vec, + blocks: Vec, expected_status: i32, + expected_stderr: String, + expected_stdout: String, ignore_stdout: bool, tempdir: TempDir, - blocks: Vec, } impl Test { fn new() -> Result { Ok(Self { args: Vec::new(), - expected_stdout: String::new(), - expected_stderr: String::new(), + blockfiles: Vec::new(), + blocks: Vec::new(), expected_status: 0, + expected_stderr: String::new(), + expected_stdout: String::new(), ignore_stdout: false, tempdir: TempDir::new()?, - blocks: Vec::new(), }) } @@ -126,41 +129,11 @@ impl Test { Ok(stdout.to_owned()) } - fn block(mut self) -> Self { - if self.blocks.is_empty() { - self.blocks.push(genesis_block(Network::Bitcoin)); - } else { - self.blocks.push(Block { - header: BlockHeader { - version: 0, - prev_blockhash: self.blocks.last().unwrap().block_hash(), - merkle_root: Default::default(), - time: 0, - bits: 0, - nonce: 0, - }, - txdata: vec![Transaction { - version: 0, - lock_time: 0, - input: vec![TxIn { - previous_output: OutPoint::null(), - script_sig: script::Builder::new() - .push_scriptint(self.blocks.len().try_into().unwrap()) - .into_script(), - sequence: 0, - witness: vec![], - }], - output: vec![TxOut { - value: 50 * COIN_VALUE, - script_pubkey: script::Builder::new().into_script(), - }], - }], - }); - } - self + fn block(self) -> Self { + self.block_with_coinbase(true) } - fn block_without_coinbase(mut self) -> Self { + fn block_with_coinbase(mut self, coinbase: bool) -> Self { if self.blocks.is_empty() { self.blocks.push(genesis_block(Network::Bitcoin)); } else { @@ -173,7 +146,26 @@ impl Test { bits: 0, nonce: 0, }, - txdata: Vec::new(), + txdata: if coinbase { + vec![Transaction { + version: 0, + lock_time: 0, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: script::Builder::new() + .push_scriptint(self.blocks.len().try_into().unwrap()) + .into_script(), + sequence: 0, + witness: vec![], + }], + output: vec![TxOut { + value: 50 * COIN_VALUE, + script_pubkey: script::Builder::new().into_script(), + }], + }] + } else { + Vec::new() + }, }); } self @@ -214,20 +206,38 @@ impl Test { self } + fn blockfile(mut self) -> Self { + self.blockfiles.push(self.blocks.len()); + self + } + fn populate_blocksdir(&self) -> io::Result<()> { let blocksdir = self.tempdir.path().join("blocks"); fs::create_dir(&blocksdir)?; - let mut blockfile = File::create(blocksdir.join("blk00000.dat"))?; - - for block in &self.blocks { - let mut encoded = Vec::new(); - block.consensus_encode(&mut encoded)?; - blockfile.write_all(&[0xf9, 0xbe, 0xb4, 0xd9])?; - blockfile.write_all(&(encoded.len() as u32).to_le_bytes())?; - blockfile.write_all(&encoded)?; - for tx in &block.txdata { - eprintln!("{}", tx.txid()); + + let mut start = 0; + + for (i, end) in self + .blockfiles + .iter() + .copied() + .chain(iter::once(self.blocks.len())) + .enumerate() + { + let mut blockfile = File::create(blocksdir.join(format!("blk{:05}.dat", i)))?; + + for block in &self.blocks[start..end] { + let mut encoded = Vec::new(); + block.consensus_encode(&mut encoded)?; + blockfile.write_all(&[0xf9, 0xbe, 0xb4, 0xd9])?; + blockfile.write_all(&(encoded.len() as u32).to_le_bytes())?; + blockfile.write_all(&encoded)?; + for tx in &block.txdata { + eprintln!("{}", tx.txid()); + } } + + start = end; } Ok(())