From 53fa35096fbeea2b82b28987e8ef6f7d39ffc80b Mon Sep 17 00:00:00 2001 From: LLFourn Date: Fri, 31 May 2024 13:52:49 +1000 Subject: [PATCH 1/2] refactor(electrum)!: put the tx cache in electrum Previously there was a tx cache that you passed in as part of the sync request. This seems bad and the example show'd that you should copy all your transactions from the transaction graph into the sync request every time you sync'd. If you forgot to do this then you would always download everything. Instead just do a plain old simple cache inside the electrum client. This way at least you only download transactions once. You can pre-populate the cache with a method also and I did this in the examples. --- crates/chain/src/indexed_tx_graph.rs | 6 + crates/chain/src/spk_client.rs | 65 +-- ...electrum_ext.rs => bdk_electrum_client.rs} | 536 +++++++++--------- crates/electrum/src/lib.rs | 11 +- crates/electrum/tests/test_electrum.rs | 8 +- crates/wallet/src/wallet/mod.rs | 2 - example-crates/example_electrum/src/main.rs | 11 +- example-crates/wallet_electrum/src/main.rs | 16 +- 8 files changed, 302 insertions(+), 353 deletions(-) rename crates/electrum/src/{electrum_ext.rs => bdk_electrum_client.rs} (57%) diff --git a/crates/chain/src/indexed_tx_graph.rs b/crates/chain/src/indexed_tx_graph.rs index e5e1f7536..ff6686332 100644 --- a/crates/chain/src/indexed_tx_graph.rs +++ b/crates/chain/src/indexed_tx_graph.rs @@ -352,3 +352,9 @@ pub trait Indexer { /// Determines whether the transaction should be included in the index. fn is_tx_relevant(&self, tx: &Transaction) -> bool; } + +impl AsRef> for IndexedTxGraph { + fn as_ref(&self) -> &TxGraph { + &self.graph + } +} diff --git a/crates/chain/src/spk_client.rs b/crates/chain/src/spk_client.rs index 19813c560..fdc3be35b 100644 --- a/crates/chain/src/spk_client.rs +++ b/crates/chain/src/spk_client.rs @@ -1,19 +1,12 @@ //! Helper types for spk-based blockchain clients. use crate::{ - collections::{BTreeMap, HashMap}, - local_chain::CheckPoint, - ConfirmationTimeHeightAnchor, TxGraph, + collections::BTreeMap, local_chain::CheckPoint, ConfirmationTimeHeightAnchor, TxGraph, }; -use alloc::{boxed::Box, sync::Arc, vec::Vec}; -use bitcoin::{OutPoint, Script, ScriptBuf, Transaction, Txid}; +use alloc::{boxed::Box, vec::Vec}; +use bitcoin::{OutPoint, Script, ScriptBuf, Txid}; use core::{fmt::Debug, marker::PhantomData, ops::RangeBounds}; -/// A cache of [`Arc`]-wrapped full transactions, identified by their [`Txid`]s. -/// -/// This is used by the chain-source to avoid re-fetching full transactions. -pub type TxCache = HashMap>; - /// Data required to perform a spk-based blockchain client sync. /// /// A client sync fetches relevant chain data for a known list of scripts, transaction ids and @@ -24,8 +17,6 @@ pub struct SyncRequest { /// /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip pub chain_tip: CheckPoint, - /// Cache of full transactions, so the chain-source can avoid re-fetching. - pub tx_cache: TxCache, /// Transactions that spend from or to these indexed script pubkeys. pub spks: Box + Send>, /// Transactions with these txids. @@ -39,36 +30,12 @@ impl SyncRequest { pub fn from_chain_tip(cp: CheckPoint) -> Self { Self { chain_tip: cp, - tx_cache: TxCache::new(), spks: Box::new(core::iter::empty()), txids: Box::new(core::iter::empty()), outpoints: Box::new(core::iter::empty()), } } - /// Add to the [`TxCache`] held by the request. - /// - /// This consumes the [`SyncRequest`] and returns the updated one. - #[must_use] - pub fn cache_txs(mut self, full_txs: impl IntoIterator) -> Self - where - T: Into>, - { - self.tx_cache = full_txs - .into_iter() - .map(|(txid, tx)| (txid, tx.into())) - .collect(); - self - } - - /// Add all transactions from [`TxGraph`] into the [`TxCache`]. - /// - /// This consumes the [`SyncRequest`] and returns the updated one. - #[must_use] - pub fn cache_graph_txs(self, graph: &TxGraph) -> Self { - self.cache_txs(graph.full_txs().map(|tx_node| (tx_node.txid, tx_node.tx))) - } - /// Set the [`Script`]s that will be synced against. /// /// This consumes the [`SyncRequest`] and returns the updated one. @@ -227,8 +194,6 @@ pub struct FullScanRequest { /// /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip pub chain_tip: CheckPoint, - /// Cache of full transactions, so the chain-source can avoid re-fetching. - pub tx_cache: TxCache, /// Iterators of script pubkeys indexed by the keychain index. pub spks_by_keychain: BTreeMap + Send>>, } @@ -239,34 +204,10 @@ impl FullScanRequest { pub fn from_chain_tip(chain_tip: CheckPoint) -> Self { Self { chain_tip, - tx_cache: TxCache::new(), spks_by_keychain: BTreeMap::new(), } } - /// Add to the [`TxCache`] held by the request. - /// - /// This consumes the [`SyncRequest`] and returns the updated one. - #[must_use] - pub fn cache_txs(mut self, full_txs: impl IntoIterator) -> Self - where - T: Into>, - { - self.tx_cache = full_txs - .into_iter() - .map(|(txid, tx)| (txid, tx.into())) - .collect(); - self - } - - /// Add all transactions from [`TxGraph`] into the [`TxCache`]. - /// - /// This consumes the [`SyncRequest`] and returns the updated one. - #[must_use] - pub fn cache_graph_txs(self, graph: &TxGraph) -> Self { - self.cache_txs(graph.full_txs().map(|tx_node| (tx_node.txid, tx_node.tx))) - } - /// Construct a new [`FullScanRequest`] from a given `chain_tip` and `index`. /// /// Unbounded script pubkey iterators for each keychain (`K`) are extracted using diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/bdk_electrum_client.rs similarity index 57% rename from crates/electrum/src/electrum_ext.rs rename to crates/electrum/src/bdk_electrum_client.rs index d02a7dcec..17480cc57 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/bdk_electrum_client.rs @@ -2,19 +2,69 @@ use bdk_chain::{ bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, collections::{BTreeMap, HashMap, HashSet}, local_chain::CheckPoint, - spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult, TxCache}, + spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult}, tx_graph::TxGraph, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, }; use core::str::FromStr; use electrum_client::{ElectrumApi, Error, HeaderNotification}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; /// We include a chain suffix of a certain length for the purpose of robustness. const CHAIN_SUFFIX_LENGTH: u32 = 8; -/// Trait to extend [`electrum_client::Client`] functionality. -pub trait ElectrumExt { +/// Wrapper around an [`electrum_client::ElectrumApi`] which includes an internal in-memory +/// transaction cache to avoid re-fetching already downloaded transactions. +#[derive(Debug)] +pub struct BdkElectrumClient { + /// The internal [`electrum_client::ElectrumApi`] + pub inner: E, + /// The transaction cache + tx_cache: Mutex>>, +} + +impl BdkElectrumClient { + /// Creates a new bdk client from a [`electrum_client::ElectrumApi`] + pub fn new(client: E) -> Self { + Self { + inner: client, + tx_cache: Default::default(), + } + } + + /// Inserts transactions into the transaction cache so that the client will not fetch these + /// transactions. + pub fn populate_tx_cache(&self, tx_graph: impl AsRef>) { + let txs = tx_graph + .as_ref() + .full_txs() + .map(|tx_node| (tx_node.txid, tx_node.tx)); + + let mut tx_cache = self.tx_cache.lock().unwrap(); + for (txid, tx) in txs { + tx_cache.insert(txid, tx); + } + } + + /// Fetch transaction of given `txid`. + /// + /// If it hits the cache it will return the cached version and avoid making the request. + pub fn fetch_tx(&self, txid: Txid) -> Result, Error> { + let tx_cache = self.tx_cache.lock().unwrap(); + + if let Some(tx) = tx_cache.get(&txid) { + return Ok(Arc::clone(tx)); + } + + drop(tx_cache); + + let tx = Arc::new(self.inner.transaction_get(&txid)?); + + self.tx_cache.lock().unwrap().insert(txid, Arc::clone(&tx)); + + Ok(tx) + } + /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and /// returns updates for [`bdk_chain`] data structures. /// @@ -25,44 +75,12 @@ pub trait ElectrumExt { /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch /// request /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee - /// calculation - fn full_scan( + pub fn full_scan( &self, request: FullScanRequest, stop_gap: usize, batch_size: usize, fetch_prev_txouts: bool, - ) -> Result, Error>; - - /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified - /// and returns updates for [`bdk_chain`] data structures. - /// - /// - `request`: struct with data required to perform a spk-based blockchain client sync, - /// see [`SyncRequest`] - /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch - /// request - /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee - /// calculation - /// - /// If the scripts to sync are unknown, such as when restoring or importing a keychain that - /// may include scripts that have been used, use [`full_scan`] with the keychain. - /// - /// [`full_scan`]: ElectrumExt::full_scan - fn sync( - &self, - request: SyncRequest, - batch_size: usize, - fetch_prev_txouts: bool, - ) -> Result; -} - -impl ElectrumExt for E { - fn full_scan( - &self, - mut request: FullScanRequest, - stop_gap: usize, - batch_size: usize, - fetch_prev_txouts: bool, ) -> Result, Error> { let mut request_spks = request.spks_by_keychain; @@ -75,7 +93,7 @@ impl ElectrumExt for E { let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new(); let update = loop { - let (tip, _) = construct_update_tip(self, request.chain_tip.clone())?; + let (tip, _) = construct_update_tip(&self.inner, request.chain_tip.clone())?; let mut graph_update = TxGraph::::default(); let cps = tip .iter() @@ -85,24 +103,22 @@ impl ElectrumExt for E { if !request_spks.is_empty() { if !scanned_spks.is_empty() { - scanned_spks.append(&mut populate_with_spks( - self, - &cps, - &mut request.tx_cache, - &mut graph_update, - &mut scanned_spks - .iter() - .map(|(i, (spk, _))| (i.clone(), spk.clone())), - stop_gap, - batch_size, - )?); + scanned_spks.append( + &mut self.populate_with_spks( + &cps, + &mut graph_update, + &mut scanned_spks + .iter() + .map(|(i, (spk, _))| (i.clone(), spk.clone())), + stop_gap, + batch_size, + )?, + ); } for (keychain, keychain_spks) in &mut request_spks { scanned_spks.extend( - populate_with_spks( - self, + self.populate_with_spks( &cps, - &mut request.tx_cache, &mut graph_update, keychain_spks, stop_gap, @@ -115,14 +131,14 @@ impl ElectrumExt for E { } // check for reorgs during scan process - let server_blockhash = self.block_header(tip.height() as usize)?.block_hash(); + let server_blockhash = self.inner.block_header(tip.height() as usize)?.block_hash(); if tip.hash() != server_blockhash { continue; // reorg } // Fetch previous `TxOut`s for fee calculation if flag is enabled. if fetch_prev_txouts { - fetch_prev_txout(self, &mut request.tx_cache, &mut graph_update)?; + self.fetch_prev_txout(&mut graph_update)?; } let chain_update = tip; @@ -148,46 +164,45 @@ impl ElectrumExt for E { Ok(ElectrumFullScanResult(update)) } - fn sync( + /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified + /// and returns updates for [`bdk_chain`] data structures. + /// + /// - `request`: struct with data required to perform a spk-based blockchain client sync, + /// see [`SyncRequest`] + /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch + /// request + /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee + /// calculation + /// + /// If the scripts to sync are unknown, such as when restoring or importing a keychain that + /// may include scripts that have been used, use [`full_scan`] with the keychain. + /// + /// [`full_scan`]: Self::full_scan + pub fn sync( &self, request: SyncRequest, batch_size: usize, fetch_prev_txouts: bool, ) -> Result { - let mut tx_cache = request.tx_cache.clone(); - let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone()) - .cache_txs(request.tx_cache) .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk))); let mut full_scan_res = self .full_scan(full_scan_req, usize::MAX, batch_size, false)? .with_confirmation_height_anchor(); - let (tip, _) = construct_update_tip(self, request.chain_tip)?; + let (tip, _) = construct_update_tip(&self.inner, request.chain_tip)?; let cps = tip .iter() .take(10) .map(|cp| (cp.height(), cp)) .collect::>(); - populate_with_txids( - self, - &cps, - &mut tx_cache, - &mut full_scan_res.graph_update, - request.txids, - )?; - populate_with_outpoints( - self, - &cps, - &mut tx_cache, - &mut full_scan_res.graph_update, - request.outpoints, - )?; + self.populate_with_txids(&cps, &mut full_scan_res.graph_update, request.txids)?; + self.populate_with_outpoints(&cps, &mut full_scan_res.graph_update, request.outpoints)?; // Fetch previous `TxOut`s for fee calculation if flag is enabled. if fetch_prev_txouts { - fetch_prev_txout(self, &mut tx_cache, &mut full_scan_res.graph_update)?; + self.fetch_prev_txout(&mut full_scan_res.graph_update)?; } Ok(ElectrumSyncResult(SyncResult { @@ -195,9 +210,180 @@ impl ElectrumExt for E { graph_update: full_scan_res.graph_update, })) } + + /// Populate the `graph_update` with transactions/anchors associated with the given `spks`. + /// + /// Transactions that contains an output with requested spk, or spends form an output with + /// requested spk will be added to `graph_update`. Anchors of the aforementioned transactions are + /// also included. + /// + /// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory. + fn populate_with_spks( + &self, + cps: &BTreeMap, + graph_update: &mut TxGraph, + spks: &mut impl Iterator, + stop_gap: usize, + batch_size: usize, + ) -> Result, Error> { + let mut unused_spk_count = 0_usize; + let mut scanned_spks = BTreeMap::new(); + + loop { + let spks = (0..batch_size) + .map_while(|_| spks.next()) + .collect::>(); + if spks.is_empty() { + return Ok(scanned_spks); + } + + let spk_histories = self + .inner + .batch_script_get_history(spks.iter().map(|(_, s)| s.as_script()))?; + + for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) { + if spk_history.is_empty() { + scanned_spks.insert(spk_index, (spk, false)); + unused_spk_count += 1; + if unused_spk_count > stop_gap { + return Ok(scanned_spks); + } + continue; + } else { + scanned_spks.insert(spk_index, (spk, true)); + unused_spk_count = 0; + } + + for tx_res in spk_history { + let _ = graph_update.insert_tx(self.fetch_tx(tx_res.tx_hash)?); + if let Some(anchor) = determine_tx_anchor(cps, tx_res.height, tx_res.tx_hash) { + let _ = graph_update.insert_anchor(tx_res.tx_hash, anchor); + } + } + } + } + } + + // Helper function which fetches the `TxOut`s of our relevant transactions' previous transactions, + // which we do not have by default. This data is needed to calculate the transaction fee. + fn fetch_prev_txout( + &self, + graph_update: &mut TxGraph, + ) -> Result<(), Error> { + let full_txs: Vec> = + graph_update.full_txs().map(|tx_node| tx_node.tx).collect(); + for tx in full_txs { + for vin in &tx.input { + let outpoint = vin.previous_output; + let vout = outpoint.vout; + let prev_tx = self.fetch_tx(outpoint.txid)?; + let txout = prev_tx.output[vout as usize].clone(); + let _ = graph_update.insert_txout(outpoint, txout); + } + } + Ok(()) + } + + /// Populate the `graph_update` with associated transactions/anchors of `outpoints`. + /// + /// Transactions in which the outpoint resides, and transactions that spend from the outpoint are + /// included. Anchors of the aforementioned transactions are included. + /// + /// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory. + fn populate_with_outpoints( + &self, + cps: &BTreeMap, + graph_update: &mut TxGraph, + outpoints: impl IntoIterator, + ) -> Result<(), Error> { + for outpoint in outpoints { + let op_txid = outpoint.txid; + let op_tx = self.fetch_tx(op_txid)?; + let op_txout = match op_tx.output.get(outpoint.vout as usize) { + Some(txout) => txout, + None => continue, + }; + debug_assert_eq!(op_tx.txid(), op_txid); + + // attempt to find the following transactions (alongside their chain positions), and + // add to our sparsechain `update`: + let mut has_residing = false; // tx in which the outpoint resides + let mut has_spending = false; // tx that spends the outpoint + for res in self.inner.script_get_history(&op_txout.script_pubkey)? { + if has_residing && has_spending { + break; + } + + if !has_residing && res.tx_hash == op_txid { + has_residing = true; + let _ = graph_update.insert_tx(Arc::clone(&op_tx)); + if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { + let _ = graph_update.insert_anchor(res.tx_hash, anchor); + } + } + + if !has_spending && res.tx_hash != op_txid { + let res_tx = self.fetch_tx(res.tx_hash)?; + // we exclude txs/anchors that do not spend our specified outpoint(s) + has_spending = res_tx + .input + .iter() + .any(|txin| txin.previous_output == outpoint); + if !has_spending { + continue; + } + let _ = graph_update.insert_tx(Arc::clone(&res_tx)); + if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { + let _ = graph_update.insert_anchor(res.tx_hash, anchor); + } + } + } + } + Ok(()) + } + + /// Populate the `graph_update` with transactions/anchors of the provided `txids`. + fn populate_with_txids( + &self, + cps: &BTreeMap, + graph_update: &mut TxGraph, + txids: impl IntoIterator, + ) -> Result<(), Error> { + for txid in txids { + let tx = match self.fetch_tx(txid) { + Ok(tx) => tx, + Err(electrum_client::Error::Protocol(_)) => continue, + Err(other_err) => return Err(other_err), + }; + + let spk = tx + .output + .first() + .map(|txo| &txo.script_pubkey) + .expect("tx must have an output"); + + // because of restrictions of the Electrum API, we have to use the `script_get_history` + // call to get confirmation status of our transaction + let anchor = match self + .inner + .script_get_history(spk)? + .into_iter() + .find(|r| r.tx_hash == txid) + { + Some(r) => determine_tx_anchor(cps, r.height, txid), + None => continue, + }; + + let _ = graph_update.insert_tx(tx); + if let Some(anchor) = anchor { + let _ = graph_update.insert_anchor(txid, anchor); + } + } + Ok(()) + } } -/// The result of [`ElectrumExt::full_scan`]. +/// The result of [`BdkElectrumClient::full_scan`]. /// /// This can be transformed into a [`FullScanResult`] with either [`ConfirmationHeightAnchor`] or /// [`ConfirmationTimeHeightAnchor`] anchor types. @@ -214,18 +400,18 @@ impl ElectrumFullScanResult { /// This requires additional calls to the Electrum server. pub fn with_confirmation_time_height_anchor( self, - client: &impl ElectrumApi, + client: &BdkElectrumClient, ) -> Result, Error> { let res = self.0; Ok(FullScanResult { - graph_update: try_into_confirmation_time_result(res.graph_update, client)?, + graph_update: try_into_confirmation_time_result(res.graph_update, &client.inner)?, chain_update: res.chain_update, last_active_indices: res.last_active_indices, }) } } -/// The result of [`ElectrumExt::sync`]. +/// The result of [`BdkElectrumClient::sync`]. /// /// This can be transformed into a [`SyncResult`] with either [`ConfirmationHeightAnchor`] or /// [`ConfirmationTimeHeightAnchor`] anchor types. @@ -242,11 +428,11 @@ impl ElectrumSyncResult { /// This requires additional calls to the Electrum server. pub fn with_confirmation_time_height_anchor( self, - client: &impl ElectrumApi, + client: &BdkElectrumClient, ) -> Result, Error> { let res = self.0; Ok(SyncResult { - graph_update: try_into_confirmation_time_result(res.graph_update, client)?, + graph_update: try_into_confirmation_time_result(res.graph_update, &client.inner)?, chain_update: res.chain_update, }) } @@ -394,193 +580,3 @@ fn determine_tx_anchor( } } } - -/// Populate the `graph_update` with associated transactions/anchors of `outpoints`. -/// -/// Transactions in which the outpoint resides, and transactions that spend from the outpoint are -/// included. Anchors of the aforementioned transactions are included. -/// -/// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory. -fn populate_with_outpoints( - client: &impl ElectrumApi, - cps: &BTreeMap, - tx_cache: &mut TxCache, - graph_update: &mut TxGraph, - outpoints: impl IntoIterator, -) -> Result<(), Error> { - for outpoint in outpoints { - let op_txid = outpoint.txid; - let op_tx = fetch_tx(client, tx_cache, op_txid)?; - let op_txout = match op_tx.output.get(outpoint.vout as usize) { - Some(txout) => txout, - None => continue, - }; - debug_assert_eq!(op_tx.txid(), op_txid); - - // attempt to find the following transactions (alongside their chain positions), and - // add to our sparsechain `update`: - let mut has_residing = false; // tx in which the outpoint resides - let mut has_spending = false; // tx that spends the outpoint - for res in client.script_get_history(&op_txout.script_pubkey)? { - if has_residing && has_spending { - break; - } - - if !has_residing && res.tx_hash == op_txid { - has_residing = true; - let _ = graph_update.insert_tx(Arc::clone(&op_tx)); - if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { - let _ = graph_update.insert_anchor(res.tx_hash, anchor); - } - } - - if !has_spending && res.tx_hash != op_txid { - let res_tx = fetch_tx(client, tx_cache, res.tx_hash)?; - // we exclude txs/anchors that do not spend our specified outpoint(s) - has_spending = res_tx - .input - .iter() - .any(|txin| txin.previous_output == outpoint); - if !has_spending { - continue; - } - let _ = graph_update.insert_tx(Arc::clone(&res_tx)); - if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { - let _ = graph_update.insert_anchor(res.tx_hash, anchor); - } - } - } - } - Ok(()) -} - -/// Populate the `graph_update` with transactions/anchors of the provided `txids`. -fn populate_with_txids( - client: &impl ElectrumApi, - cps: &BTreeMap, - tx_cache: &mut TxCache, - graph_update: &mut TxGraph, - txids: impl IntoIterator, -) -> Result<(), Error> { - for txid in txids { - let tx = match fetch_tx(client, tx_cache, txid) { - Ok(tx) => tx, - Err(electrum_client::Error::Protocol(_)) => continue, - Err(other_err) => return Err(other_err), - }; - - let spk = tx - .output - .first() - .map(|txo| &txo.script_pubkey) - .expect("tx must have an output"); - - // because of restrictions of the Electrum API, we have to use the `script_get_history` - // call to get confirmation status of our transaction - let anchor = match client - .script_get_history(spk)? - .into_iter() - .find(|r| r.tx_hash == txid) - { - Some(r) => determine_tx_anchor(cps, r.height, txid), - None => continue, - }; - - let _ = graph_update.insert_tx(tx); - if let Some(anchor) = anchor { - let _ = graph_update.insert_anchor(txid, anchor); - } - } - Ok(()) -} - -/// Fetch transaction of given `txid`. -/// -/// We maintain a `tx_cache` so that we won't need to fetch from Electrum with every call. -fn fetch_tx( - client: &C, - tx_cache: &mut TxCache, - txid: Txid, -) -> Result, Error> { - use bdk_chain::collections::hash_map::Entry; - Ok(match tx_cache.entry(txid) { - Entry::Occupied(entry) => entry.get().clone(), - Entry::Vacant(entry) => entry - .insert(Arc::new(client.transaction_get(&txid)?)) - .clone(), - }) -} - -// Helper function which fetches the `TxOut`s of our relevant transactions' previous transactions, -// which we do not have by default. This data is needed to calculate the transaction fee. -fn fetch_prev_txout( - client: &C, - tx_cache: &mut TxCache, - graph_update: &mut TxGraph, -) -> Result<(), Error> { - let full_txs: Vec> = - graph_update.full_txs().map(|tx_node| tx_node.tx).collect(); - for tx in full_txs { - for vin in &tx.input { - let outpoint = vin.previous_output; - let vout = outpoint.vout; - let prev_tx = fetch_tx(client, tx_cache, outpoint.txid)?; - let txout = prev_tx.output[vout as usize].clone(); - let _ = graph_update.insert_txout(outpoint, txout); - } - } - Ok(()) -} - -/// Populate the `graph_update` with transactions/anchors associated with the given `spks`. -/// -/// Transactions that contains an output with requested spk, or spends form an output with -/// requested spk will be added to `graph_update`. Anchors of the aforementioned transactions are -/// also included. -/// -/// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory. -fn populate_with_spks( - client: &impl ElectrumApi, - cps: &BTreeMap, - tx_cache: &mut TxCache, - graph_update: &mut TxGraph, - spks: &mut impl Iterator, - stop_gap: usize, - batch_size: usize, -) -> Result, Error> { - let mut unused_spk_count = 0_usize; - let mut scanned_spks = BTreeMap::new(); - - loop { - let spks = (0..batch_size) - .map_while(|_| spks.next()) - .collect::>(); - if spks.is_empty() { - return Ok(scanned_spks); - } - - let spk_histories = - client.batch_script_get_history(spks.iter().map(|(_, s)| s.as_script()))?; - - for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) { - if spk_history.is_empty() { - scanned_spks.insert(spk_index, (spk, false)); - unused_spk_count += 1; - if unused_spk_count > stop_gap { - return Ok(scanned_spks); - } - continue; - } else { - scanned_spks.insert(spk_index, (spk, true)); - unused_spk_count = 0; - } - - for tx_res in spk_history { - let _ = graph_update.insert_tx(fetch_tx(client, tx_cache, tx_res.tx_hash)?); - if let Some(anchor) = determine_tx_anchor(cps, tx_res.height, tx_res.tx_hash) { - let _ = graph_update.insert_anchor(tx_res.tx_hash, anchor); - } - } - } - } -} diff --git a/crates/electrum/src/lib.rs b/crates/electrum/src/lib.rs index eaa2405bf..d303ee403 100644 --- a/crates/electrum/src/lib.rs +++ b/crates/electrum/src/lib.rs @@ -1,9 +1,9 @@ //! This crate is used for updating structures of [`bdk_chain`] with data from an Electrum server. //! -//! The two primary methods are [`ElectrumExt::sync`] and [`ElectrumExt::full_scan`]. In most cases -//! [`ElectrumExt::sync`] is used to sync the transaction histories of scripts that the application +//! The two primary methods are [`BdkElectrumClient::sync`] and [`BdkElectrumClient::full_scan`]. In most cases +//! [`BdkElectrumClient::sync`] is used to sync the transaction histories of scripts that the application //! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it -//! has shown a user. [`ElectrumExt::full_scan`] is meant to be used when importing or restoring a +//! has shown a user. [`BdkElectrumClient::full_scan`] is meant to be used when importing or restoring a //! keychain where the range of possibly used scripts is not known. In this case it is necessary to //! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a //! sync or full scan the user receives relevant blockchain data and output updates for @@ -15,7 +15,8 @@ #![warn(missing_docs)] -mod electrum_ext; +mod bdk_electrum_client; +pub use bdk_electrum_client::*; + pub use bdk_chain; pub use electrum_client; -pub use electrum_ext::*; diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index dd7ee6a92..4e7911bd3 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -5,7 +5,7 @@ use bdk_chain::{ spk_client::SyncRequest, ConfirmationTimeHeightAnchor, IndexedTxGraph, SpkTxOutIndex, }; -use bdk_electrum::ElectrumExt; +use bdk_electrum::BdkElectrumClient; use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv}; fn get_balance( @@ -31,7 +31,8 @@ fn scan_detects_confirmed_tx() -> anyhow::Result<()> { const SEND_AMOUNT: Amount = Amount::from_sat(10_000); let env = TestEnv::new()?; - let client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); // Setup addresses. let addr_to_mine = env @@ -122,7 +123,8 @@ fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> { const SEND_AMOUNT: Amount = Amount::from_sat(10_000); let env = TestEnv::new()?; - let client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); // Setup addresses. let addr_to_mine = env diff --git a/crates/wallet/src/wallet/mod.rs b/crates/wallet/src/wallet/mod.rs index e80584dc3..f91f83efe 100644 --- a/crates/wallet/src/wallet/mod.rs +++ b/crates/wallet/src/wallet/mod.rs @@ -2499,7 +2499,6 @@ impl Wallet { /// start a blockchain sync with a spk based blockchain client. pub fn start_sync_with_revealed_spks(&self) -> SyncRequest { SyncRequest::from_chain_tip(self.chain.tip()) - .cache_graph_txs(self.tx_graph()) .populate_with_revealed_spks(&self.indexed_graph.index, ..) } @@ -2513,7 +2512,6 @@ impl Wallet { /// in which the list of used scripts is not known. pub fn start_full_scan(&self) -> FullScanRequest { FullScanRequest::from_keychain_txout_index(self.chain.tip(), &self.indexed_graph.index) - .cache_graph_txs(self.tx_graph()) } } diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index e88b1e6fc..8467d2699 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -14,7 +14,7 @@ use bdk_chain::{ }; use bdk_electrum::{ electrum_client::{self, Client, ElectrumApi}, - ElectrumExt, + BdkElectrumClient, }; use example_cli::{ anyhow::{self, Context}, @@ -146,7 +146,10 @@ fn main() -> anyhow::Result<()> { } }; - let client = electrum_cmd.electrum_args().client(args.network)?; + let client = BdkElectrumClient::new(electrum_cmd.electrum_args().client(args.network)?); + + // Tell the electrum client about the txs we've already got locally so it doesn't re-download them + client.populate_tx_cache(&*graph.lock().unwrap()); let (chain_update, mut graph_update, keychain_update) = match electrum_cmd.clone() { ElectrumCommands::Scan { @@ -159,7 +162,6 @@ fn main() -> anyhow::Result<()> { let chain = &*chain.lock().unwrap(); FullScanRequest::from_chain_tip(chain.tip()) - .cache_graph_txs(graph.graph()) .set_spks_for_keychain( Keychain::External, graph @@ -220,8 +222,7 @@ fn main() -> anyhow::Result<()> { } let chain_tip = chain.tip(); - let mut request = - SyncRequest::from_chain_tip(chain_tip.clone()).cache_graph_txs(graph.graph()); + let mut request = SyncRequest::from_chain_tip(chain_tip.clone()); if all_spks { let all_spks = graph diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index c411713ff..017902c8a 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -6,10 +6,8 @@ const BATCH_SIZE: usize = 5; use std::io::Write; use std::str::FromStr; -use bdk_electrum::{ - electrum_client::{self, ElectrumApi}, - ElectrumExt, -}; +use bdk_electrum::electrum_client::{self, ElectrumApi}; +use bdk_electrum::BdkElectrumClient; use bdk_file_store::Store; use bdk_wallet::bitcoin::{Address, Amount}; use bdk_wallet::chain::collections::HashSet; @@ -37,7 +35,13 @@ fn main() -> Result<(), anyhow::Error> { println!("Wallet balance before syncing: {} sats", balance.total()); print!("Syncing..."); - let client = electrum_client::Client::new("ssl://electrum.blockstream.info:60002")?; + let client = BdkElectrumClient::new(electrum_client::Client::new( + "ssl://electrum.blockstream.info:60002", + )?); + + // Populate the electrum client's transaction cache so it doesn't redownload transaction we + // already have. + client.populate_tx_cache(&wallet); let request = wallet .start_full_scan() @@ -89,7 +93,7 @@ fn main() -> Result<(), anyhow::Error> { assert!(finalized); let tx = psbt.extract_tx()?; - client.transaction_broadcast(&tx)?; + client.inner.transaction_broadcast(&tx)?; println!("Tx broadcasted! Txid: {}", tx.txid()); Ok(()) From 2d2656acfa83ab4c4846c0aab14072efb64c5cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 4 Jun 2024 11:59:39 +0800 Subject: [PATCH 2/2] feat(electrum): re-export `transaction_broadcast` method Also: update `wallet_electrum` example to use the method. --- crates/electrum/src/bdk_electrum_client.rs | 7 +++++++ example-crates/wallet_electrum/src/main.rs | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/electrum/src/bdk_electrum_client.rs b/crates/electrum/src/bdk_electrum_client.rs index 17480cc57..5f7d59b22 100644 --- a/crates/electrum/src/bdk_electrum_client.rs +++ b/crates/electrum/src/bdk_electrum_client.rs @@ -65,6 +65,13 @@ impl BdkElectrumClient { Ok(tx) } + /// Broadcasts a transaction to the network. + /// + /// This is a re-export of [`ElectrumApi::transaction_broadcast`]. + pub fn transaction_broadcast(&self, tx: &Transaction) -> Result { + self.inner.transaction_broadcast(tx) + } + /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and /// returns updates for [`bdk_chain`] data structures. /// diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index 017902c8a..73a75184b 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -6,7 +6,7 @@ const BATCH_SIZE: usize = 5; use std::io::Write; use std::str::FromStr; -use bdk_electrum::electrum_client::{self, ElectrumApi}; +use bdk_electrum::electrum_client; use bdk_electrum::BdkElectrumClient; use bdk_file_store::Store; use bdk_wallet::bitcoin::{Address, Amount}; @@ -93,7 +93,7 @@ fn main() -> Result<(), anyhow::Error> { assert!(finalized); let tx = psbt.extract_tx()?; - client.inner.transaction_broadcast(&tx)?; + client.transaction_broadcast(&tx)?; println!("Tx broadcasted! Txid: {}", tx.txid()); Ok(())