From 362e83d5f58672ab8492d5afa7dbf88568f3c7a7 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Thu, 21 Dec 2023 22:56:59 +0800 Subject: [PATCH 1/2] Index transactions --- deploy/ord.service | 1 + src/index.rs | 111 +++++++++++++++++++----------- src/index/updater.rs | 13 ++++ src/options.rs | 2 + test-bitcoincore-rpc/src/lib.rs | 4 ++ test-bitcoincore-rpc/src/state.rs | 4 ++ tests/server.rs | 27 ++++++++ tests/test_server.rs | 2 +- 8 files changed, 124 insertions(+), 40 deletions(-) diff --git a/deploy/ord.service b/deploy/ord.service index e04b8dd803..c068bde8af 100644 --- a/deploy/ord.service +++ b/deploy/ord.service @@ -15,6 +15,7 @@ ExecStart=/usr/local/bin/ord \ --data-dir /var/lib/ord \ --index-runes \ --index-sats \ + --index-transactions \ server \ --acme-contact mailto:casey@rodarmor.com \ --csp-origin https://ordinals.com \ diff --git a/src/index.rs b/src/index.rs index 60b74b7157..bd37937627 100644 --- a/src/index.rs +++ b/src/index.rs @@ -20,8 +20,8 @@ use { log::log_enabled, redb::{ Database, DatabaseError, MultimapTable, MultimapTableDefinition, MultimapTableHandle, - ReadableMultimapTable, ReadableTable, RedbKey, RedbValue, StorageError, Table, TableDefinition, - TableHandle, WriteTransaction, + ReadOnlyTable, ReadableMultimapTable, ReadableTable, RedbKey, RedbValue, StorageError, Table, + TableDefinition, TableHandle, WriteTransaction, }, std::{ collections::{BTreeSet, HashMap}, @@ -75,6 +75,7 @@ define_table! { SEQUENCE_NUMBER_TO_RUNE_ID, u32, RuneIdValue } define_table! { SEQUENCE_NUMBER_TO_SATPOINT, u32, &SatPointValue } define_table! { STATISTIC_TO_COUNT, u64, u64 } define_table! { TRANSACTION_ID_TO_RUNE, &TxidValue, u128 } +define_table! { TRANSACTION_ID_TO_TRANSACTION, &TxidValue, &[u8] } define_table! { WRITE_TRANSACTION_STARTING_BLOCK_COUNT_TO_TIMESTAMP, u32, u128 } #[derive(Debug, PartialEq)] @@ -97,6 +98,7 @@ pub(crate) enum Statistic { Runes, SatRanges, UnboundInscriptions, + IndexTransactions, } impl Statistic { @@ -195,6 +197,7 @@ pub struct Index { height_limit: Option, index_runes: bool, index_sats: bool, + index_transactions: bool, options: Options, path: PathBuf, started: DateTime, @@ -237,6 +240,7 @@ impl Index { let index_runes; let index_sats; + let index_transactions; let index_path = path.clone(); let once = Once::new(); @@ -252,40 +256,32 @@ impl Index { Ok(database) => { { let tx = database.begin_read()?; - let schema_version = tx - .open_table(STATISTIC_TO_COUNT)? + let statistics = tx.open_table(STATISTIC_TO_COUNT)?; + + let schema_version = statistics .get(&Statistic::Schema.key())? .map(|x| x.value()) .unwrap_or(0); match schema_version.cmp(&SCHEMA_VERSION) { - cmp::Ordering::Less => - bail!( - "index at `{}` appears to have been built with an older, incompatible version of ord, consider deleting and rebuilding the index: index schema {schema_version}, ord schema {SCHEMA_VERSION}", - path.display() - ), - cmp::Ordering::Greater => - bail!( - "index at `{}` appears to have been built with a newer, incompatible version of ord, consider updating ord: index schema {schema_version}, ord schema {SCHEMA_VERSION}", - path.display() - ), - cmp::Ordering::Equal => { + cmp::Ordering::Less => + bail!( + "index at `{}` appears to have been built with an older, incompatible version of ord, consider deleting and rebuilding the index: index schema {schema_version}, ord schema {SCHEMA_VERSION}", + path.display() + ), + cmp::Ordering::Greater => + bail!( + "index at `{}` appears to have been built with a newer, incompatible version of ord, consider updating ord: index schema {schema_version}, ord schema {SCHEMA_VERSION}", + path.display() + ), + cmp::Ordering::Equal => { + } } - } - let statistics = tx.open_table(STATISTIC_TO_COUNT)?; - index_runes = statistics - .get(&Statistic::IndexRunes.key())? - .unwrap() - .value() - != 0; - - index_sats = statistics - .get(&Statistic::IndexSats.key())? - .unwrap() - .value() - != 0; + index_runes = Self::is_statistic_set(&statistics, Statistic::IndexRunes)?; + index_sats = Self::is_statistic_set(&statistics, Statistic::IndexSats)?; + index_transactions = Self::is_statistic_set(&statistics, Statistic::IndexTransactions)?; } database @@ -330,15 +326,12 @@ impl Index { index_runes = options.index_runes(); index_sats = options.index_sats; + index_transactions = options.index_transactions; - statistics.insert( - &Statistic::IndexRunes.key(), - &u64::from(index_runes), - )?; - - statistics.insert(&Statistic::IndexSats.key(), &u64::from(index_sats))?; - - statistics.insert(&Statistic::Schema.key(), &SCHEMA_VERSION)?; + Self::set_statistic(&mut statistics, Statistic::IndexRunes, u64::from(index_runes))?; + Self::set_statistic(&mut statistics, Statistic::IndexSats, u64::from(index_sats))?; + Self::set_statistic(&mut statistics, Statistic::IndexSats, u64::from(index_transactions))?; + Self::set_statistic(&mut statistics, Statistic::Schema, SCHEMA_VERSION)?; } tx.commit()?; @@ -361,6 +354,7 @@ impl Index { height_limit: options.height_limit, index_runes, index_sats, + index_transactions, options: options.clone(), path, started: Utc::now(), @@ -610,6 +604,12 @@ impl Index { insert_table_info(&mut tables, &wtx, total_bytes, SEQUENCE_NUMBER_TO_SATPOINT); insert_table_info(&mut tables, &wtx, total_bytes, STATISTIC_TO_COUNT); insert_table_info(&mut tables, &wtx, total_bytes, TRANSACTION_ID_TO_RUNE); + insert_table_info( + &mut tables, + &wtx, + total_bytes, + TRANSACTION_ID_TO_TRANSACTION, + ); insert_table_info( &mut tables, &wtx, @@ -792,6 +792,28 @@ impl Index { Ok(()) } + pub(crate) fn set_statistic( + statistics: &mut Table, + statistic: Statistic, + value: u64, + ) -> Result<()> { + statistics.insert(&statistic.key(), &value)?; + Ok(()) + } + + pub(crate) fn is_statistic_set( + statistics: &ReadOnlyTable, + statistic: Statistic, + ) -> Result { + Ok( + statistics + .get(&statistic.key())? + .map(|guard| guard.value()) + .unwrap_or_default() + != 0, + ) + } + #[cfg(test)] pub(crate) fn statistic(&self, statistic: Statistic) -> u64 { self @@ -1436,10 +1458,21 @@ impl Index { pub(crate) fn get_transaction(&self, txid: Txid) -> Result> { if txid == self.genesis_block_coinbase_txid { - Ok(Some(self.genesis_block_coinbase_transaction.clone())) - } else { - self.client.get_raw_transaction(&txid, None).into_option() + return Ok(Some(self.genesis_block_coinbase_transaction.clone())); + } + + if self.index_transactions { + if let Some(transaction) = self + .database + .begin_read()? + .open_table(TRANSACTION_ID_TO_TRANSACTION)? + .get(&txid.store())? + { + return Ok(Some(consensus::encode::deserialize(transaction.value())?)); + } } + + self.client.get_raw_transaction(&txid, None).into_option() } pub(crate) fn get_transaction_blockhash(&self, txid: Txid) -> Result> { diff --git a/src/index/updater.rs b/src/index/updater.rs index 471999956e..0e016192d9 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -602,6 +602,19 @@ impl<'index> Updater<'_> { } } + if index.index_transactions { + let mut transaction_id_to_transaction = wtx.open_table(TRANSACTION_ID_TO_TRANSACTION)?; + + let mut buffer = Vec::new(); + for (transaction, txid) in block.txdata { + transaction + .consensus_encode(&mut buffer) + .expect("in-memory writers don't error"); + transaction_id_to_transaction.insert(&txid.store(), buffer.as_slice())?; + buffer.clear(); + } + } + height_to_block_header.insert(&self.height, &block.header.store())?; self.height += 1; diff --git a/src/options.rs b/src/options.rs index 86c60d5857..0830c27739 100644 --- a/src/options.rs +++ b/src/options.rs @@ -49,6 +49,8 @@ pub struct Options { pub(crate) index_runes: bool, #[arg(long, help = "Track location of all satoshis.")] pub(crate) index_sats: bool, + #[arg(long, help = "Store transactions in index.")] + pub(crate) index_transactions: bool, #[arg( long, short, diff --git a/test-bitcoincore-rpc/src/lib.rs b/test-bitcoincore-rpc/src/lib.rs index b12cf61fbf..2e1276ef5e 100644 --- a/test-bitcoincore-rpc/src/lib.rs +++ b/test-bitcoincore-rpc/src/lib.rs @@ -206,6 +206,10 @@ impl Handle { self.state.lock().unwrap() } + pub fn clear_state(&self) { + self.state.lock().unwrap().clear(); + } + pub fn wallets(&self) -> BTreeSet { self.state().wallets.clone() } diff --git a/test-bitcoincore-rpc/src/state.rs b/test-bitcoincore-rpc/src/state.rs index c77a3d5c31..fc962056d7 100644 --- a/test-bitcoincore-rpc/src/state.rs +++ b/test-bitcoincore-rpc/src/state.rs @@ -47,6 +47,10 @@ impl State { } } + pub(crate) fn clear(&mut self) { + *self = Self::new(self.network, self.version, self.fail_lock_unspent); + } + pub(crate) fn push_block(&mut self, subsidy: u64) -> Block { let coinbase = Transaction { version: 2, diff --git a/tests/server.rs b/tests/server.rs index 1d716ec713..4aa0a4ee24 100644 --- a/tests/server.rs +++ b/tests/server.rs @@ -459,3 +459,30 @@ fn sat_recursive_endpoints_without_sat_index_return_404() { StatusCode::NOT_FOUND, ); } + +#[test] +fn transactions_are_stored_with_transaction_index() { + let rpc_server = test_bitcoincore_rpc::spawn(); + + rpc_server.mine_blocks(2); + + let server = TestServer::spawn_with_args(&rpc_server, &["--index-transactions"]); + + assert_eq!( + server + .request("/tx/f02151fda57850f323fa22b3d74c1e7039658ac788566677725fe682efb1fe3b") + .status(), + StatusCode::OK, + ); + + rpc_server.clear_state(); + + rpc_server.mine_blocks(1); + + assert_eq!( + server + .request("/tx/f02151fda57850f323fa22b3d74c1e7039658ac788566677725fe682efb1fe3b") + .status(), + StatusCode::OK, + ); +} diff --git a/tests/test_server.rs b/tests/test_server.rs index de20c59893..87a7f794cc 100644 --- a/tests/test_server.rs +++ b/tests/test_server.rs @@ -104,7 +104,7 @@ impl TestServer { for i in 0.. { let response = reqwest::blocking::get(self.url().join("/blockcount").unwrap()).unwrap(); assert_eq!(response.status(), StatusCode::OK); - if response.text().unwrap().parse::().unwrap() == chain_block_count { + if response.text().unwrap().parse::().unwrap() >= chain_block_count { break; } else if i == 20 { panic!("index failed to synchronize with chain"); From d23621773404778cac2d26560772eca4856f1fb7 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 25 Dec 2023 12:42:29 -0800 Subject: [PATCH 2/2] Bump schema --- src/index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.rs b/src/index.rs index bd37937627..92b2239ed8 100644 --- a/src/index.rs +++ b/src/index.rs @@ -41,7 +41,7 @@ mod updater; #[cfg(test)] pub(crate) mod testing; -const SCHEMA_VERSION: u64 = 15; +const SCHEMA_VERSION: u64 = 16; macro_rules! define_table { ($name:ident, $key:ty, $value:ty) => {