diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7cc6bd0692..85ea05852f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,18 +44,13 @@ jobs: cargo update --locked --package ord - name: Test - run: | - cargo test --all - cargo test --all --features redb + run: cargo test --all - name: Clippy - run: | - cargo clippy --all --all-targets - cargo clippy --all --all-targets --features redb + run: cargo clippy --all --all-targets - name: Format - run: | - cargo fmt --all -- --check + run: cargo fmt --all -- --check - name: Check for Forbidden Words if: ${{ matrix.os == 'ubuntu-latest' }} diff --git a/Cargo.lock b/Cargo.lock index 06cafa36f6..719303d296 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,7 +86,7 @@ checksum = "ab2504b827a8bef941ba3dd64bdffe9cf56ca182908a147edd6189c95fbcae7d" dependencies = [ "async-trait", "axum-core", - "bitflags 1.3.2", + "bitflags", "bytes", "futures-util", "http", @@ -231,12 +231,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "bitflags" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" - [[package]] name = "bitflags" version = "1.3.2" @@ -325,7 +319,7 @@ version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "bitflags 1.3.2", + "bitflags", "textwrap 0.11.0", "unicode-width", ] @@ -337,7 +331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" dependencies = [ "atty", - "bitflags 1.3.2", + "bitflags", "clap_derive", "clap_lex", "indexmap", @@ -375,7 +369,7 @@ version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] @@ -786,12 +780,6 @@ dependencies = [ "slab", ] -[[package]] -name = "gcc" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" - [[package]] name = "getrandom" version = "0.1.16" @@ -1291,15 +1279,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "memmap2" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5172b50c23043ff43dd53e51392f36519d9b35a8f3a410d30ece5d1aedd58ae" -dependencies = [ - "libc", -] - [[package]] name = "memoffset" version = "0.6.5" @@ -1380,7 +1359,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg-if 1.0.0", "libc", "memoffset", @@ -1464,7 +1443,7 @@ version = "0.10.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1534,7 +1513,6 @@ dependencies = [ "lazy_static", "log", "nix", - "ord-lmdb-zero", "qrcode-generator", "rayon", "redb", @@ -1550,28 +1528,6 @@ dependencies = [ "unindent", ] -[[package]] -name = "ord-liblmdb-sys" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0708b9f740bd632d4fe7468b314c4c45d994811a811e82acc193ced0f6b358ba" -dependencies = [ - "gcc", - "libc", -] - -[[package]] -name = "ord-lmdb-zero" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6186bbfce46615119f72e26acfe341b79a88c18c2d994d3e4b2c9029a24bf09" -dependencies = [ - "bitflags 0.9.1", - "libc", - "ord-liblmdb-sys", - "supercow", -] - [[package]] name = "os_str_bytes" version = "6.1.0" @@ -1687,7 +1643,7 @@ version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc38c0ad57efb786dd57b9864e5b18bae478c00c824dc55a38bbc9da95dde3ba" dependencies = [ - "bitflags 1.3.2", + "bitflags", "crc32fast", "deflate", "miniz_oxide", @@ -1949,12 +1905,11 @@ dependencies = [ [[package]] name = "redb" -version = "0.0.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49648fe45ea8e0205024fae151ac253398a05dd2cc1eebda32ac0d0e2674e7c1" +checksum = "73ef901de04afd870ef0bb84980dc24e3cf8425b6cef49102122a446cebf7f2a" dependencies = [ "libc", - "memmap2", ] [[package]] @@ -1963,7 +1918,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] @@ -2123,7 +2078,7 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ - "bitflags 1.3.2", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -2236,12 +2191,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "supercow" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171758edb47aa306a78dfa4ab9aeb5167405bd4e3dc2b64e88f6a84bbe98bd63" - [[package]] name = "syn" version = "1.0.96" @@ -2483,7 +2432,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae" dependencies = [ - "bitflags 1.3.2", + "bitflags", "bytes", "futures-core", "futures-util", diff --git a/Cargo.toml b/Cargo.toml index 33a0471f5a..401bf83da4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,10 +27,9 @@ integer-sqrt = "0.1.5" jsonrpc = "0.12.1" lazy_static = "1.4.0" log = "0.4.14" -ord-lmdb-zero = "0.4.5" qrcode-generator = "4.1.6" rayon = "1.5.1" -redb = { version = "0.0.5", optional = true } +redb = "0.2.0" reqwest = { version = "0.11.10", features = ["blocking", "json"] } secp256k1 = { version = "0.22.1", features = ["rand", "rand-std", "global-context"] } serde = { version = "1.0.137", features = ["derive"] } diff --git a/justfile b/justfile index 2b141ab2f5..f19b59e24a 100644 --- a/justfile +++ b/justfile @@ -38,7 +38,6 @@ test-deploy: --exclude .git \ --exclude target \ --exclude .vagrant \ - --exclude index.lmdb \ --exclude index.redb \ . root@192.168.56.4:ord ssh root@192.168.56.4 'cd ord && ./deploy/setup' diff --git a/src/index.rs b/src/index.rs index 2585fc65b9..af1de39ff9 100644 --- a/src/index.rs +++ b/src/index.rs @@ -4,6 +4,10 @@ use { rayon::iter::{IntoParallelRefIterator, ParallelIterator}, }; +const HEIGHT_TO_HASH: TableDefinition = TableDefinition::new("HEIGHT_TO_HASH"); +const OUTPOINT_TO_ORDINAL_RANGES: TableDefinition<[u8], [u8]> = + TableDefinition::new("OUTPOINT_TO_ORDINAL_RANGES"); + pub(crate) struct Index { client: Client, database: Database, @@ -24,10 +28,22 @@ impl Index { ) .context("Failed to connect to RPC URL")?; - Ok(Self { - client, - database: Database::open(options).context("Failed to open database")?, - }) + let database = match unsafe { redb::Database::open("index.redb") } { + Ok(database) => database, + Err(redb::Error::Io(error)) if error.kind() == io::ErrorKind::NotFound => unsafe { + redb::Database::create("index.redb", options.max_index_size.0)? + }, + Err(error) => return Err(error.into()), + }; + + let tx = database.begin_write()?; + + tx.open_table(HEIGHT_TO_HASH)?; + tx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; + + tx.commit()?; + + Ok(Self { client, database }) } #[allow(clippy::self_named_constructors)] @@ -40,7 +56,34 @@ impl Index { } pub(crate) fn print_info(&self) -> Result { - self.database.print_info() + let wtx = self.database.begin_write()?; + + let height_to_hash = wtx.open_table(HEIGHT_TO_HASH)?; + + let blocks_indexed = height_to_hash + .range(0..)? + .rev() + .next() + .map(|(height, _hash)| height + 1) + .unwrap_or(0); + + let outputs_indexed = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?.len()?; + + let stats = wtx.stats()?; + + println!("blocks indexed: {}", blocks_indexed); + println!("outputs indexed: {}", outputs_indexed); + println!("tree height: {}", stats.tree_height()); + println!("free pages: {}", stats.free_pages()); + println!("stored: {}", Bytes(stats.stored_bytes())); + println!("overhead: {}", Bytes(stats.metadata_bytes())); + println!("fragmented: {}", Bytes(stats.fragmented_bytes())); + println!( + "index size: {}", + Bytes(std::fs::metadata("index.redb")?.len().try_into()?) + ); + + Ok(()) } pub(crate) fn decode_ordinal_range(bytes: [u8; 11]) -> (u64, u64) { @@ -58,120 +101,147 @@ impl Index { } pub(crate) fn index_ranges(&self) -> Result { - let mut wtx = self.database.begin_write()?; - loop { - let start = Instant::now(); - let mut ordinal_ranges_written = 0; + let mut wtx = self.database.begin_write()?; - let height = wtx.height()?; + let done = self.index_block(&mut wtx)?; - let block = match self.block(height)? { - Some(block) => block, - None => { - wtx.commit()?; - return Ok(()); - } - }; + wtx.commit()?; - let time: DateTime = DateTime::from_utc( - NaiveDateTime::from_timestamp(block.header.time as i64, 0), - Utc, - ); + if done || INTERRUPTS.load(atomic::Ordering::Relaxed) > 0 { + break; + } + } - log::info!( - "Block {height} at {} with {} transactions…", - time, - block.txdata.len() - ); + Ok(()) + } - if let Some(prev_height) = height.checked_sub(1) { - let prev_hash = wtx.blockhash_at_height(prev_height)?.unwrap(); + pub(crate) fn index_block(&self, wtx: &mut WriteTransaction) -> Result { + let mut height_to_hash = wtx.open_table(HEIGHT_TO_HASH)?; + let mut outpoint_to_ordinal_ranges = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; - if prev_hash != block.header.prev_blockhash.as_ref() { - return Err(anyhow!("Reorg detected at or before {prev_height}")); - } - } + let start = Instant::now(); + let mut ordinal_ranges_written = 0; - let mut coinbase_inputs = VecDeque::new(); + let height = height_to_hash + .range(0..)? + .rev() + .next() + .map(|(height, _hash)| height + 1) + .unwrap_or(0); - let h = Height(height); - if h.subsidy() > 0 { - let start = h.starting_ordinal(); - coinbase_inputs.push_front((start.n(), (start + h.subsidy()).n())); + let block = match self.block(height)? { + Some(block) => block, + None => { + return Ok(true); } + }; - let txdata = block - .txdata - .as_slice() - .par_iter() - .map(|tx| (tx.txid(), tx)) - .collect::>(); + let time: DateTime = DateTime::from_utc( + NaiveDateTime::from_timestamp(block.header.time as i64, 0), + Utc, + ); - for (tx_offset, (txid, tx)) in txdata.iter().enumerate().skip(1) { - log::trace!("Indexing transaction {tx_offset}…"); + log::info!( + "Block {height} at {} with {} transactions…", + time, + block.txdata.len() + ); - let mut input_ordinal_ranges = VecDeque::new(); + if let Some(prev_height) = height.checked_sub(1) { + let prev_hash = height_to_hash.get(&prev_height)?.unwrap(); - for input in &tx.input { - let mut key = Vec::new(); - input.previous_output.consensus_encode(&mut key)?; + if prev_hash != block.header.prev_blockhash.as_ref() { + return Err(anyhow!("Reorg detected at or before {prev_height}")); + } + } - let ordinal_ranges = wtx - .get_ordinal_ranges(key.as_slice())? - .ok_or_else(|| anyhow!("Could not find outpoint in index"))?; + let mut coinbase_inputs = VecDeque::new(); - for chunk in ordinal_ranges.chunks_exact(11) { - input_ordinal_ranges.push_back(Self::decode_ordinal_range(chunk.try_into().unwrap())); - } + let h = Height(height); + if h.subsidy() > 0 { + let start = h.starting_ordinal(); + coinbase_inputs.push_front((start.n(), (start + h.subsidy()).n())); + } - wtx.remove_outpoint(&key)?; - } + let txdata = block + .txdata + .as_slice() + .par_iter() + .map(|tx| (tx.txid(), tx)) + .collect::>(); - self.index_transaction( - *txid, - tx, - &mut wtx, - &mut input_ordinal_ranges, - &mut ordinal_ranges_written, - )?; + for (tx_offset, (txid, tx)) in txdata.iter().enumerate().skip(1) { + log::trace!("Indexing transaction {tx_offset}…"); - coinbase_inputs.extend(input_ordinal_ranges); - } + let mut input_ordinal_ranges = VecDeque::new(); - if let Some((txid, tx)) = txdata.first() { - self.index_transaction( - *txid, - tx, - &mut wtx, - &mut coinbase_inputs, - &mut ordinal_ranges_written, - )?; - } + for input in &tx.input { + let mut key = Vec::new(); + input.previous_output.consensus_encode(&mut key)?; - wtx.set_blockhash_at_height(height, block.block_hash())?; - if height % 1000 == 0 { - wtx.commit()?; - wtx = self.database.begin_write()?; - } + let ordinal_ranges = outpoint_to_ordinal_ranges + .get(key.as_slice())? + .ok_or_else(|| anyhow!("Could not find outpoint in index"))?; - log::info!( - "Wrote {ordinal_ranges_written} ordinal ranges in {}ms", - (Instant::now() - start).as_millis(), - ); + for chunk in ordinal_ranges.chunks_exact(11) { + input_ordinal_ranges.push_back(Self::decode_ordinal_range(chunk.try_into().unwrap())); + } - if INTERRUPTS.load(atomic::Ordering::Relaxed) > 0 { - wtx.commit()?; - return Ok(()); + outpoint_to_ordinal_ranges.remove(&key)?; } + + self.index_transaction( + *txid, + tx, + &mut outpoint_to_ordinal_ranges, + &mut input_ordinal_ranges, + &mut ordinal_ranges_written, + )?; + + coinbase_inputs.extend(input_ordinal_ranges); } + + if let Some((txid, tx)) = txdata.first() { + self.index_transaction( + *txid, + tx, + &mut outpoint_to_ordinal_ranges, + &mut coinbase_inputs, + &mut ordinal_ranges_written, + )?; + } + + height_to_hash.insert(&height, &block.block_hash())?; + + log::info!( + "Wrote {ordinal_ranges_written} ordinal ranges in {}ms", + (Instant::now() - start).as_millis(), + ); + + Ok(false) + } + + pub(crate) fn height(&self) -> Result { + let tx = self.database.begin_read()?; + + let height_to_hash = tx.open_table(HEIGHT_TO_HASH)?; + + Ok( + height_to_hash + .range(0..)? + .rev() + .next() + .map(|(height, _hash)| height + 1) + .unwrap_or(0), + ) } fn index_transaction( &self, txid: Txid, tx: &Transaction, - wtx: &mut WriteTransaction, + outpoint_to_ordinal_ranges: &mut Table<[u8], [u8]>, input_ordinal_ranges: &mut VecDeque<(u64, u64)>, ordinal_ranges_written: &mut u64, ) -> Result { @@ -212,7 +282,7 @@ impl Index { let mut outpoint_encoded = Vec::new(); outpoint.consensus_encode(&mut outpoint_encoded)?; - wtx.insert_outpoint(&outpoint_encoded, &ordinals)?; + outpoint_to_ordinal_ranges.insert(&outpoint_encoded, &ordinals)?; } Ok(()) @@ -229,17 +299,49 @@ impl Index { } pub(crate) fn find(&self, ordinal: Ordinal) -> Result> { - if self.database.height()? <= ordinal.height().0 { + if self.height()? <= ordinal.height().0 { return Ok(None); } - self.database.find(ordinal) + let rtx = self.database.begin_read()?; + + let outpoint_to_ordinal_ranges = rtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?; + + let mut cursor = outpoint_to_ordinal_ranges.range([]..)?; + + while let Some((key, value)) = cursor.next() { + let mut offset = 0; + for chunk in value.chunks_exact(11) { + let (start, end) = Index::decode_ordinal_range(chunk.try_into().unwrap()); + if start <= ordinal.0 && ordinal.0 < end { + let outpoint: OutPoint = Decodable::consensus_decode(key)?; + return Ok(Some(SatPoint { + outpoint, + offset: offset + ordinal.0 - start, + })); + } + offset += end - start; + } + } + + Ok(None) + } + + pub(crate) fn list_inner(&self, outpoint: &[u8]) -> Result>> { + Ok( + self + .database + .begin_read()? + .open_table(OUTPOINT_TO_ORDINAL_RANGES)? + .get(outpoint)? + .map(|outpoint| outpoint.to_vec()), + ) } pub(crate) fn list(&self, outpoint: OutPoint) -> Result>> { let mut outpoint_encoded = Vec::new(); outpoint.consensus_encode(&mut outpoint_encoded)?; - let ordinal_ranges = self.database.list(&outpoint_encoded)?; + let ordinal_ranges = self.list_inner(&outpoint_encoded)?; match ordinal_ranges { Some(ordinal_ranges) => { let mut output = Vec::new(); diff --git a/src/lmdb_database.rs b/src/lmdb_database.rs deleted file mode 100644 index 6957e9aaa0..0000000000 --- a/src/lmdb_database.rs +++ /dev/null @@ -1,213 +0,0 @@ -use { - super::*, - ord_lmdb_zero::{self as lmdb, EnvBuilder, Environment}, - std::fs, -}; - -const HEIGHT_TO_HASH: &str = "HEIGHT_TO_HASH"; -const OUTPOINT_TO_ORDINAL_RANGES: &str = "OUTPOINT_TO_ORDINAL_RANGES"; - -trait LmdbResultExt { - fn into_option(self) -> Result>; -} - -impl LmdbResultExt for lmdb::Result { - fn into_option(self) -> Result> { - match self { - Ok(value) => Ok(Some(value)), - Err(lmdb::Error::Code(-30798)) => Ok(None), - Err(error) => Err(error.into()), - } - } -} - -pub(crate) struct Database { - environment: Arc, - height_to_hash: lmdb::Database<'static>, - outpoint_to_ordinal_ranges: lmdb::Database<'static>, -} - -impl Database { - pub(crate) fn open(options: &Options) -> Result { - let path = "index.lmdb"; - - fs::create_dir_all(path)?; - - let environment = unsafe { - let mut builder = EnvBuilder::new()?; - - builder.set_maxdbs(3)?; - builder.set_mapsize(options.index_size.0)?; - - Arc::new(builder.open(path, lmdb::open::Flags::empty(), 0o600)?) - }; - - let height_to_hash = lmdb::Database::open( - environment.clone(), - Some(HEIGHT_TO_HASH), - &lmdb::DatabaseOptions::new(lmdb::db::CREATE), - )?; - - let outpoint_to_ordinal_ranges = lmdb::Database::open( - environment.clone(), - Some(OUTPOINT_TO_ORDINAL_RANGES), - &lmdb::DatabaseOptions::new(lmdb::db::CREATE), - )?; - - Ok(Self { - environment, - height_to_hash, - outpoint_to_ordinal_ranges, - }) - } - - pub(crate) fn begin_write(&self) -> Result { - WriteTransaction::new(self) - } - - pub(crate) fn print_info(&self) -> Result { - let stat = self.environment.stat()?; - - let blocks_indexed = self.height()?; - - println!("blocks indexed: {}", blocks_indexed); - println!( - "data and metadata: {}", - ((stat.branch_pages + stat.leaf_pages + stat.overflow_pages) as u64) * stat.psize as u64 - ); - - Ok(()) - } - - pub(crate) fn height(&self) -> Result { - let tx = lmdb::ReadTransaction::new(self.environment.clone())?; - - let height = tx - .cursor(&self.height_to_hash)? - .last::<[u8], [u8]>(&tx.access()) - .into_option()? - .map(|(key, _value)| u64::from_be_bytes(key.try_into().unwrap()) + 1) - .unwrap_or_default(); - - Ok(height) - } - - pub(crate) fn list(&self, outpoint: &[u8]) -> Result>> { - Ok( - lmdb::ReadTransaction::new(self.environment.clone())? - .access() - .get::<[u8], [u8]>(&self.outpoint_to_ordinal_ranges, outpoint) - .into_option()? - .map(|ranges| ranges.to_vec()), - ) - } - - pub(crate) fn find(&self, ordinal: Ordinal) -> Result> { - let tx = lmdb::ReadTransaction::new(self.environment.clone())?; - - let access = tx.access(); - - let mut cursor = tx.cursor(&self.outpoint_to_ordinal_ranges)?; - - while let Some((key, value)) = cursor.next::<[u8], [u8]>(&access).into_option()? { - let mut offset = 0; - for chunk in value.chunks_exact(11) { - let (start, end) = Index::decode_ordinal_range(chunk.try_into().unwrap()); - if start <= ordinal.0 && ordinal.0 < end { - let outpoint: OutPoint = Decodable::consensus_decode(key)?; - return Ok(Some(SatPoint { - outpoint, - offset: offset + ordinal.0 - start, - })); - } - offset += end - start; - } - } - - Ok(None) - } -} - -pub(crate) struct WriteTransaction<'a> { - height_to_hash: &'a lmdb::Database<'static>, - lmdb_write_transaction: lmdb::WriteTransaction<'a>, - outpoint_to_ordinal_ranges: &'a lmdb::Database<'static>, -} - -impl<'a> WriteTransaction<'a> { - pub(crate) fn new(database: &'a Database) -> Result { - let lmdb_write_transaction = lmdb::WriteTransaction::new(database.environment.clone())?; - - Ok(Self { - lmdb_write_transaction, - height_to_hash: &database.height_to_hash, - outpoint_to_ordinal_ranges: &database.outpoint_to_ordinal_ranges, - }) - } - - pub(crate) fn commit(self) -> Result { - Ok(self.lmdb_write_transaction.commit()?) - } - - pub(crate) fn height(&self) -> Result { - Ok( - self - .lmdb_write_transaction - .cursor(self.height_to_hash)? - .last::<[u8], [u8]>(&self.lmdb_write_transaction.access()) - .into_option()? - .map(|(key, _value)| u64::from_be_bytes(key.try_into().unwrap()) + 1) - .unwrap_or_default(), - ) - } - - pub(crate) fn blockhash_at_height(&self, height: u64) -> Result>> { - Ok( - self - .lmdb_write_transaction - .access() - .get::<[u8], [u8]>(self.height_to_hash, &height.to_be_bytes()) - .into_option()? - .map(|value| value.to_vec()), - ) - } - - pub(crate) fn set_blockhash_at_height(&mut self, height: u64, blockhash: BlockHash) -> Result { - self.lmdb_write_transaction.access().put( - self.height_to_hash, - &height.to_be_bytes(), - blockhash.as_ref(), - lmdb::put::Flags::empty(), - )?; - Ok(()) - } - - pub(crate) fn insert_outpoint(&mut self, outpoint: &[u8], ordinal_ranges: &[u8]) -> Result { - self.lmdb_write_transaction.access().put( - self.outpoint_to_ordinal_ranges, - outpoint, - ordinal_ranges, - lmdb::put::Flags::empty(), - )?; - Ok(()) - } - - pub(crate) fn remove_outpoint(&mut self, outpoint: &[u8]) -> Result { - self - .lmdb_write_transaction - .access() - .del_key(self.outpoint_to_ordinal_ranges, outpoint)?; - Ok(()) - } - - pub(crate) fn get_ordinal_ranges(&self, outpoint: &[u8]) -> Result>> { - Ok( - self - .lmdb_write_transaction - .access() - .get::<[u8], [u8]>(self.outpoint_to_ordinal_ranges, outpoint) - .into_option()? - .map(|value| value.to_vec()), - ) - } -} diff --git a/src/main.rs b/src/main.rs index 2f037082de..1c881b823a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,7 @@ use { bech32::{FromBase32, ToBase32}, bitcoin::{ blockdata::constants::COIN_VALUE, consensus::Decodable, consensus::Encodable, - util::key::PrivateKey, Address, Block, BlockHash, Network, OutPoint, Transaction, Txid, + util::key::PrivateKey, Address, Block, Network, OutPoint, Transaction, Txid, }, bitcoin_hashes::{sha256, Hash, HashEngine}, chrono::{DateTime, NaiveDateTime, Utc}, @@ -21,6 +21,7 @@ use { integer_sqrt::IntegerSquareRoot, lazy_static::lazy_static, qrcode_generator::QrCodeEcc, + redb::{Database, ReadableTable, Table, TableDefinition, WriteTransaction}, secp256k1::{rand, schnorr::Signature, KeyPair, Secp256k1, SecretKey, XOnlyPublicKey}, serde::{Deserialize, Serialize}, std::{ @@ -46,24 +47,14 @@ use { tower_http::cors::{Any, CorsLayer}, }; -#[cfg(feature = "redb")] -use redb_database::{Database, WriteTransaction}; - -#[cfg(not(feature = "redb"))] -use lmdb_database::{Database, WriteTransaction}; - mod arguments; mod bytes; mod epoch; mod height; mod index; -#[cfg(not(feature = "redb"))] -mod lmdb_database; mod nft; mod options; mod ordinal; -#[cfg(feature = "redb")] -mod redb_database; mod sat_point; mod subcommand; diff --git a/src/options.rs b/src/options.rs index 6e6a166247..99b1e75028 100644 --- a/src/options.rs +++ b/src/options.rs @@ -3,7 +3,7 @@ use super::*; #[derive(Parser)] pub(crate) struct Options { #[clap(long, default_value = "1MiB")] - pub(crate) index_size: Bytes, + pub(crate) max_index_size: Bytes, #[clap(long)] pub(crate) cookie_file: Option, #[clap(long)] diff --git a/src/redb_database.rs b/src/redb_database.rs deleted file mode 100644 index eed365852a..0000000000 --- a/src/redb_database.rs +++ /dev/null @@ -1,179 +0,0 @@ -use { - super::*, - redb::{ReadableTable, TableDefinition}, -}; - -const HEIGHT_TO_HASH: TableDefinition = TableDefinition::new("HEIGHT_TO_HASH"); -const OUTPOINT_TO_ORDINAL_RANGES: TableDefinition<[u8], [u8]> = - TableDefinition::new("OUTPOINT_TO_ORDINAL_RANGES"); - -pub(crate) struct Database(redb::Database); - -impl Database { - pub(crate) fn open(options: &Options) -> Result { - let database = match unsafe { redb::Database::open("index.redb") } { - Ok(database) => database, - Err(redb::Error::Io(error)) if error.kind() == io::ErrorKind::NotFound => unsafe { - redb::Database::create("index.redb", options.index_size.0)? - }, - Err(error) => return Err(error.into()), - }; - - let tx = database.begin_write()?; - - tx.open_table(&HEIGHT_TO_HASH)?; - tx.open_table(&OUTPOINT_TO_ORDINAL_RANGES)?; - - tx.commit()?; - - Ok(Self(database)) - } - - pub(crate) fn begin_write(&self) -> Result { - WriteTransaction::new(&self.0) - } - - pub(crate) fn print_info(&self) -> Result { - let tx = self.0.begin_read()?; - - let height_to_hash = tx.open_table(&HEIGHT_TO_HASH)?; - - let blocks_indexed = height_to_hash - .range(0..)? - .rev() - .next() - .map(|(height, _hash)| height + 1) - .unwrap_or(0); - - let outputs_indexed = tx.open_table(&OUTPOINT_TO_ORDINAL_RANGES)?.len()?; - - let stats = self.0.stats()?; - - println!("blocks indexed: {}", blocks_indexed); - println!("outputs indexed: {}", outputs_indexed); - println!("tree height: {}", stats.tree_height()); - println!("free pages: {}", stats.free_pages()); - println!("stored: {}", Bytes(stats.stored_bytes())); - println!("overhead: {}", Bytes(stats.overhead_bytes())); - println!("fragmented: {}", Bytes(stats.fragmented_bytes())); - println!( - "index size: {}", - Bytes(std::fs::metadata("index.redb")?.len().try_into()?) - ); - - Ok(()) - } - - pub(crate) fn height(&self) -> Result { - let tx = self.0.begin_read()?; - - let height_to_hash = tx.open_table(&HEIGHT_TO_HASH)?; - - Ok( - height_to_hash - .range(0..)? - .rev() - .next() - .map(|(height, _hash)| height + 1) - .unwrap_or(0), - ) - } - - pub(crate) fn find(&self, ordinal: Ordinal) -> Result> { - let rtx = self.0.begin_read()?; - - let outpoint_to_ordinal_ranges = rtx.open_table(&OUTPOINT_TO_ORDINAL_RANGES)?; - - let mut cursor = outpoint_to_ordinal_ranges.range([]..)?; - - while let Some((key, value)) = cursor.next() { - let mut offset = 0; - for chunk in value.chunks_exact(11) { - let (start, end) = Index::decode_ordinal_range(chunk.try_into().unwrap()); - if start <= ordinal.0 && ordinal.0 < end { - let outpoint: OutPoint = Decodable::consensus_decode(key)?; - return Ok(Some(SatPoint { - outpoint, - offset: offset + ordinal.0 - start, - })); - } - offset += end - start; - } - } - - Ok(None) - } - - pub(crate) fn list(&self, outpoint: &[u8]) -> Result>> { - Ok( - self - .0 - .begin_read()? - .open_table(&OUTPOINT_TO_ORDINAL_RANGES)? - .get(outpoint)? - .map(|outpoint| outpoint.to_vec()), - ) - } -} - -pub(crate) struct WriteTransaction<'a> { - inner: redb::DatabaseTransaction<'a>, - height_to_hash: redb::Table<'a, u64, [u8]>, - outpoint_to_ordinal_ranges: redb::Table<'a, [u8], [u8]>, -} - -impl<'a> WriteTransaction<'a> { - pub(crate) fn new(database: &'a redb::Database) -> Result { - let inner = database.begin_write()?; - let height_to_hash = inner.open_table(&HEIGHT_TO_HASH)?; - let outpoint_to_ordinal_ranges = inner.open_table(&OUTPOINT_TO_ORDINAL_RANGES)?; - - Ok(Self { - inner, - height_to_hash, - outpoint_to_ordinal_ranges, - }) - } - - pub(crate) fn commit(self) -> Result { - self.inner.commit()?; - Ok(()) - } - - pub(crate) fn height(&self) -> Result { - Ok( - self - .height_to_hash - .range(0..)? - .rev() - .next() - .map(|(height, _hash)| height + 1) - .unwrap_or(0), - ) - } - - pub(crate) fn blockhash_at_height(&self, height: u64) -> Result> { - Ok(self.height_to_hash.get(&height)?) - } - - pub(crate) fn set_blockhash_at_height(&mut self, height: u64, blockhash: BlockHash) -> Result { - self.height_to_hash.insert(&height, &blockhash)?; - Ok(()) - } - - pub(crate) fn insert_outpoint(&mut self, outpoint: &[u8], ordinal_ranges: &[u8]) -> Result { - self - .outpoint_to_ordinal_ranges - .insert(outpoint, ordinal_ranges)?; - Ok(()) - } - - pub(crate) fn remove_outpoint(&mut self, outpoint: &[u8]) -> Result { - self.outpoint_to_ordinal_ranges.remove(outpoint)?; - Ok(()) - } - - pub(crate) fn get_ordinal_ranges(&self, outpoint: &[u8]) -> Result> { - Ok(self.outpoint_to_ordinal_ranges.get(outpoint)?) - } -} diff --git a/tests/index.rs b/tests/index.rs index 0a1d2adcdc..7a23a36eab 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -23,40 +23,23 @@ fn incremental_indexing() -> Result { } #[test] -#[cfg(feature = "redb")] -fn custom_index_size() -> Result { - let tempdir = Test::new()? - .command("--index-size 2097152 find 0") - .expected_stdout("0396bc915f141f7de025f72ae9b6bb8dcdb5f444fc245d8fac486ba67a38eef9:0:0\n") - .block() - .output()? - .tempdir; - - assert_eq!(tempdir.path().join("index.redb").metadata()?.len(), 2 << 20); - - Ok(()) -} - -#[test] -#[cfg(feature = "redb")] -fn human_readable_index_size() -> Result { +fn default_index_size() -> Result { let tempdir = Test::new()? - .command("--index-size 2mib find 0") + .command("find 0") .expected_stdout("0396bc915f141f7de025f72ae9b6bb8dcdb5f444fc245d8fac486ba67a38eef9:0:0\n") .block() .output()? .tempdir; - assert_eq!(tempdir.path().join("index.redb").metadata()?.len(), 2 << 20); + assert_eq!(tempdir.path().join("index.redb").metadata()?.len(), 1 << 20); Ok(()) } #[test] -#[cfg(feature = "redb")] -fn default_index_size() -> Result { +fn custom_index_size() -> Result { let tempdir = Test::new()? - .command("find 0") + .command("--max-index-size 1mib find 0") .expected_stdout("0396bc915f141f7de025f72ae9b6bb8dcdb5f444fc245d8fac486ba67a38eef9:0:0\n") .block() .output()? diff --git a/tests/info.rs b/tests/info.rs index ccb3ec0cb4..44c108f4df 100644 --- a/tests/info.rs +++ b/tests/info.rs @@ -1,7 +1,6 @@ use super::*; #[test] -#[cfg(feature = "redb")] fn basic() -> Result { let output = Test::new()?.command("index").block().output()?; @@ -22,20 +21,3 @@ fn basic() -> Result { ) .run() } - -#[test] -#[cfg(not(feature = "redb"))] -fn basic() -> Result { - let output = Test::new()?.command("index").block().output()?; - - Test::with_tempdir(output.tempdir) - .command("info") - .stdout_regex( - r" - blocks indexed: 1 - data and metadata: \d+ - " - .unindent(), - ) - .run() -}