Merge branch 'dev' of https://github.com/okx/ord into pending2

.github/workflows/ci.yaml

@@ -4,9 +4,11 @@ on:
   push:
     branches:
     - master
+    - dev
   pull_request:
     branches:
     - master
+    - dev
 
 defaults:
   run:
@@ -97,9 +99,9 @@ jobs:
     strategy:
       matrix:
         os:
-        - macos-latest
+#        - macos-latest
         - ubuntu-latest
-        - windows-latest
+#        - windows-latest
 
     runs-on: ${{matrix.os}}
 

bin/forbid

@@ -9,5 +9,5 @@ which rg > /dev/null
   --glob '!docs/src/bounty/frequency.tsv' \
   --glob '!docs/po/*' \
   --ignore-case \
-  'dbg!|fixme|todo|xxx' \
+  'dbg!|fixme|xxx' \
   .

src/index.rs

@@ -854,7 +854,7 @@ impl Index {
     if let Some(height) = ord_height {
       if query_btc {
         let btc_height = match self.client.get_blockchain_info() {
-          Ok(info) => Height(info.headers as u32),
+          Ok(info) => Height(u32::try_from(info.headers).unwrap()),
           Err(e) => {
             return Err(anyhow!(
               "failed to get blockchain info from bitcoin node: {}",
@@ -2259,7 +2259,7 @@ impl Index {
       if let Some(tx_blockhash) = tx.blockhash {
         let tx_bh = self.client.get_block_header_info(&tx_blockhash)?;
         let parsed_height = self.block_height()?;
-        if parsed_height.is_none() || tx_bh.height as u32 > parsed_height.unwrap().0 {
+        if parsed_height.is_none() || u32::try_from(tx_bh.height)? > parsed_height.unwrap().0 {
           return Ok(None);
         }
       } else {
@@ -2317,10 +2317,7 @@ impl Index {
   ) -> Result<Vec<brc20::Balance>> {
     let rtx = self.database.begin_read().unwrap();
     let table = rtx.open_table(BRC20_BALANCES)?;
-    Ok(get_balances(
-      &table,
-      &ScriptKey::from_address(address.clone()),
-    )?)
+    get_balances(&table, &ScriptKey::from_address(address.clone()))
   }
 
   pub(crate) fn brc20_get_tx_events_by_txid(
@@ -2336,7 +2333,7 @@ impl Index {
       if let Some(tx_blockhash) = tx.blockhash {
         let tx_bh = self.client.get_block_header_info(&tx_blockhash)?;
         let parsed_height = self.begin_read()?.block_height()?;
-        if parsed_height.is_none() || tx_bh.height as u32 > parsed_height.unwrap().0 {
+        if parsed_height.is_none() || u32::try_from(tx_bh.height)? > parsed_height.unwrap().0 {
           return Ok(None);
         }
       } else {

src/index/simulator/mod.rs

@@ -1,4 +1,14 @@
 pub mod error;
+#[allow(clippy::needless_return)]
+#[allow(clippy::map_identity)]
+#[allow(clippy::unnecessary_cast)]
+#[allow(clippy::clone_on_copy)]
 pub mod processor;
+#[allow(clippy::needless_return)]
+#[allow(clippy::map_identity)]
+#[allow(clippy::unnecessary_cast)]
+#[allow(clippy::clone_on_copy)]
+#[allow(clippy::needless_borrow)]
+#[allow(clippy::let_and_return)]
 pub mod simulate;
 pub mod types;

src/index/simulator/processor.rs

@@ -38,8 +38,7 @@ impl IndexWrapper {
   ) -> crate::Result<T> {
     let rtx = self.internal.begin_read()?;
     let table = rtx.0.open_table(table_def)?;
-    let ret = f(table);
-    ret
+    f(table)
   }
   pub fn new(internal: Arc<Index>) -> Self {
     Self { internal }
@@ -83,7 +82,7 @@ unsafe impl<'a, 'db, 'tx> Sync for StorageProcessor<'a, 'db, 'tx> {}
 impl<'a, 'db, 'tx> StorageProcessor<'a, 'db, 'tx> {
   pub fn get_transaction(&self, tx_id: &Txid) -> crate::Result<Option<Transaction>> {
     let client = self.client.as_ref().unwrap();
-    let ret = client.get_transaction_by_tx_id(tx_id.clone())?;
+    let ret = client.get_transaction_by_tx_id(*tx_id)?;
     Ok(ret)
   }
   pub fn create_context(&self) -> crate::Result<SimulateContext<'a, 'db, 'tx>> {

src/index/simulator/simulate.rs

@@ -249,22 +249,14 @@ impl SimulatorServer {
       .get_block_by_height(height)?
       .unwrap();
     let home_inscriptions = wtx.open_table(HOME_INSCRIPTIONS)?;
-    let inscription_id_to_sequence_number =
-      wtx.open_table(INSCRIPTION_ID_TO_SEQUENCE_NUMBER)?;
-    let inscription_number_to_sequence_number = wtx
-      .open_table(INSCRIPTION_NUMBER_TO_SEQUENCE_NUMBER)
-      ?;
+    let inscription_id_to_sequence_number = wtx.open_table(INSCRIPTION_ID_TO_SEQUENCE_NUMBER)?;
+    let inscription_number_to_sequence_number =
+      wtx.open_table(INSCRIPTION_NUMBER_TO_SEQUENCE_NUMBER)?;
     let sat_to_sequence_number = wtx.open_multimap_table(SAT_TO_SEQUENCE_NUMBER)?;
-    let satpoint_to_sequence_number = wtx
-      .open_multimap_table(SATPOINT_TO_SEQUENCE_NUMBER)
-      ?;
-    let sequence_number_to_children = wtx
-      .open_multimap_table(SEQUENCE_NUMBER_TO_CHILDREN)
-      ?;
+    let satpoint_to_sequence_number = wtx.open_multimap_table(SATPOINT_TO_SEQUENCE_NUMBER)?;
+    let sequence_number_to_children = wtx.open_multimap_table(SEQUENCE_NUMBER_TO_CHILDREN)?;
     let sequence_number_to_inscription_entry = Rc::new(RefCell::new(
-      wtx
-        .open_table(SEQUENCE_NUMBER_TO_INSCRIPTION_ENTRY)
-        ?,
+      wtx.open_table(SEQUENCE_NUMBER_TO_INSCRIPTION_ENTRY)?,
     ));
     let sequence_number_to_satpoint = wtx.open_table(SEQUENCE_NUMBER_TO_SATPOINT)?;
     let transaction_id_to_transaction = wtx.open_table(TRANSACTION_ID_TO_TRANSACTION)?;
@@ -305,7 +297,7 @@ impl SimulatorServer {
 
     let db_receipts = ctx.get_transaction_receipts(&tx.txid())?;
     let operations = ctx.get_transaction_operations(&tx.txid())?;
-    if db_receipts.len() > 0 || operations.len() > 0 {
+    if !db_receipts.is_empty() || !operations.is_empty() {
       ret.ord_operations = operations;
       ret.brc20_receipts = db_receipts;
       info!("tx:{:?} already simulated", tx.txid());
@@ -361,7 +353,7 @@ impl SimulatorServer {
       error!("simulate tx:{:?} depth:{:?} over 20", tx.txid(), depth);
       return Err(SimulateError::StackOverFlow);
     }
-    *depth = *depth + 1;
+    *depth += 1;
     let tx_id = tx.txid();
     let mut need_handle_first = vec![];
     for input in &tx.input {
@@ -692,7 +684,7 @@ pub fn start_simulator(ops: Options, internal: Arc<Index>) -> Option<SimulatorSe
   let rt = Arc::new(Runtime::new().unwrap());
 
   let ret = rt.block_on(async { do_start_simulator(ops, internal.clone()).await });
-  return ret;
+  ret
 }
 
 async fn do_start_simulator(ops: Options, internal: Arc<Index>) -> Option<SimulatorServer> {
@@ -744,6 +736,7 @@ async fn do_start_simulator(ops: Options, internal: Arc<Index>) -> Option<Simula
 mod tests {
   use super::*;
   use bitcoincore_rpc::RpcApi;
+  use indexer_sdk::client::SyncClient;
   use indexer_sdk::factory::common::new_client_for_test;
   use log::LevelFilter;
   use std::path::PathBuf;
@@ -815,6 +808,8 @@ mod tests {
       simulate_bitcoin_rpc_user: Some("bitcoinrpc".to_string()),
       simulate_index: Some("./simulate".to_string().into()),
       rx: None,
+      commit_height_interval: 0,
+      commit_persist_interval: 0,
     };
     opt
   }

src/index/updater.rs

@@ -92,7 +92,16 @@ impl<'index> Updater<'_> {
 
     let (mut outpoint_sender, mut tx_out_receiver) = Self::spawn_fetcher(self.index)?;
 
+    let commit_height_interval = self.index.options.commit_height_interval();
+    let commit_persist_interval = self.index.options.commit_persist_interval();
+    log::info!(
+      "commit height interval: {}, commit persist interval: {}",
+      commit_height_interval,
+      commit_persist_interval
+    );
+
     let mut uncommitted = 0;
+    let mut unpersisted = 0;
     let mut tx_out_cache = SimpleLru::new(self.index.options.lru_size);
     while let Ok(block) = rx.recv() {
       tx_out_cache.refresh();
@@ -119,7 +128,15 @@ impl<'index> Updater<'_> {
 
       uncommitted += 1;
 
-      if uncommitted == 200 {
+      let should_break = SHUTTING_DOWN.load(atomic::Ordering::Relaxed);
+      if uncommitted >= commit_height_interval {
+        unpersisted += 1;
+        if unpersisted < commit_persist_interval && !should_break {
+          wtx.set_durability(redb::Durability::None);
+          log::info!("set wtx durability to none");
+        } else {
+          unpersisted = 0;
+        }
         self.commit(wtx)?;
         uncommitted = 0;
         wtx = self.index.begin_write()?;
@@ -135,6 +152,9 @@ impl<'index> Updater<'_> {
           // write transaction
           break;
         }
+        if should_break {
+          break;
+        }
         wtx
           .open_table(WRITE_TRANSACTION_STARTING_BLOCK_COUNT_TO_TIMESTAMP)?
           .insert(
@@ -144,14 +164,12 @@ impl<'index> Updater<'_> {
               .map(|duration| duration.as_millis())
               .unwrap_or(0),
           )?;
-      }
-
-      if SHUTTING_DOWN.load(atomic::Ordering::Relaxed) {
+      } else if should_break {
         break;
       }
     }
 
-    if uncommitted > 0 {
+    if uncommitted > 0 || unpersisted > 0 {
       self.commit(wtx)?;
     }
 

src/index/updater/inscription_updater.rs

@@ -23,6 +23,7 @@ pub(super) struct Flotsam {
   pub origin: Origin,
 }
 
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug, Clone)]
 pub enum Origin {
   New {
@@ -636,11 +637,7 @@ impl<'a, 'db, 'tx> InscriptionUpdater<'a, 'db, 'tx> {
     } else {
       new_satpoint.store()
     };
-    let transfer_to_coin_base = if new_satpoint.outpoint.txid == flotsam.txid {
-      false
-    } else {
-      true
-    };
+    let transfer_to_coin_base = new_satpoint.outpoint.txid != flotsam.txid;
     self
       .operations
       .entry(flotsam.txid)

src/okx/datastore/brc20/redb/table.rs

@@ -152,8 +152,8 @@ where
 }
 
 // BRC20_BALANCES
-pub fn update_token_balance<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static str, &'static [u8]>,
+pub fn update_token_balance(
+  table: &mut Table<'_, '_, &'static str, &'static [u8]>,
   script_key: &ScriptKey,
   new_balance: Balance,
 ) -> crate::Result<()> {
@@ -165,8 +165,8 @@ pub fn update_token_balance<'db, 'txn>(
 }
 
 // BRC20_TOKEN
-pub fn insert_token_info<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static str, &'static [u8]>,
+pub fn insert_token_info(
+  table: &mut Table<'_, '_, &'static str, &'static [u8]>,
   tick: &Tick,
   new_info: &TokenInfo,
 ) -> crate::Result<()> {
@@ -178,8 +178,8 @@ pub fn insert_token_info<'db, 'txn>(
 }
 
 // BRC20_TOKEN
-pub fn update_mint_token_info<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static str, &'static [u8]>,
+pub fn update_mint_token_info(
+  table: &mut Table<'_, '_, &'static str, &'static [u8]>,
   tick: &Tick,
   minted_amt: u128,
   minted_block_number: u32,
@@ -198,8 +198,8 @@ pub fn update_mint_token_info<'db, 'txn>(
 }
 
 // BRC20_EVENTS
-pub fn save_transaction_receipts<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static TxidValue, &'static [u8]>,
+pub fn save_transaction_receipts(
+  table: &mut Table<'_, '_, &'static TxidValue, &'static [u8]>,
   txid: &Txid,
   receipts: &[Receipt],
 ) -> crate::Result<()> {
@@ -211,8 +211,8 @@ pub fn save_transaction_receipts<'db, 'txn>(
 }
 
 // BRC20_TRANSFERABLELOG
-pub fn insert_transferable<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static str, &'static [u8]>,
+pub fn insert_transferable(
+  table: &mut Table<'_, '_, &'static str, &'static [u8]>,
   script: &ScriptKey,
   tick: &Tick,
   inscription: &TransferableLog,
@@ -225,8 +225,8 @@ pub fn insert_transferable<'db, 'txn>(
 }
 
 // BRC20_TRANSFERABLELOG
-pub fn remove_transferable<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static str, &'static [u8]>,
+pub fn remove_transferable(
+  table: &mut Table<'_, '_, &'static str, &'static [u8]>,
   script: &ScriptKey,
   tick: &Tick,
   inscription_id: &InscriptionId,
@@ -236,8 +236,8 @@ pub fn remove_transferable<'db, 'txn>(
 }
 
 // BRC20_INSCRIBE_TRANSFER
-pub fn insert_inscribe_transfer_inscription<'db, 'txn>(
-  table: &mut Table<'db, 'txn, InscriptionIdValue, &'static [u8]>,
+pub fn insert_inscribe_transfer_inscription(
+  table: &mut Table<'_, '_, InscriptionIdValue, &'static [u8]>,
   inscription_id: &InscriptionId,
   transfer_info: TransferInfo,
 ) -> crate::Result<()> {
@@ -249,8 +249,8 @@ pub fn insert_inscribe_transfer_inscription<'db, 'txn>(
 }
 
 // BRC20_INSCRIBE_TRANSFER
-pub fn remove_inscribe_transfer_inscription<'db, 'txn>(
-  table: &mut Table<'db, 'txn, InscriptionIdValue, &'static [u8]>,
+pub fn remove_inscribe_transfer_inscription(
+  table: &mut Table<'_, '_, InscriptionIdValue, &'static [u8]>,
   inscription_id: &InscriptionId,
 ) -> crate::Result<()> {
   table.remove(&inscription_id.store())?;

src/okx/datastore/ord/operation.rs

@@ -17,6 +17,7 @@ pub struct InscriptionOp {
 }
 
 // the act of marking an inscription.
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
 pub enum Action {
   New {
@@ -55,6 +56,7 @@ mod tests {
       pub new_satpoint: Option<SatPoint>,
     }
 
+    #[allow(clippy::large_enum_variant)]
     #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
     enum OldAction {
       New {

src/okx/datastore/ord/redb/table.rs

@@ -68,8 +68,8 @@ where
 }
 
 // ORD_TX_TO_OPERATIONS
-pub fn save_transaction_operations<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static TxidValue, &'static [u8]>,
+pub fn save_transaction_operations(
+  table: &mut Table<'_, '_, &'static TxidValue, &'static [u8]>,
   txid: &Txid,
   operations: &[InscriptionOp],
 ) -> crate::Result<()> {
@@ -78,8 +78,8 @@ pub fn save_transaction_operations<'db, 'txn>(
 }
 
 // COLLECTIONS_KEY_TO_INSCRIPTION_ID
-pub fn set_inscription_by_collection_key<'db, 'txn>(
-  table: &mut Table<'db, 'txn, &'static str, InscriptionIdValue>,
+pub fn set_inscription_by_collection_key(
+  table: &mut Table<'_, '_, &'static str, InscriptionIdValue>,
   key: &str,
   inscription_id: &InscriptionId,
 ) -> crate::Result<()> {
@@ -88,8 +88,8 @@ pub fn set_inscription_by_collection_key<'db, 'txn>(
 }
 
 // COLLECTIONS_INSCRIPTION_ID_TO_KINDS
-pub fn set_inscription_attributes<'db, 'txn>(
-  table: &mut Table<'db, 'txn, InscriptionIdValue, &'static [u8]>,
+pub fn set_inscription_attributes(
+  table: &mut Table<'_, '_, InscriptionIdValue, &'static [u8]>,
   inscription_id: &InscriptionId,
   kind: &[CollectionKind],
 ) -> crate::Result<()> {