Merge branch 'master' into sync_pipeline

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Overhauled sync logic for electrum and esplora.
 - Unify ureq and reqwest esplora backends to have the same configuration parameters. This means reqwest now has a timeout parameter and ureq has a concurrency parameter.
 - Fixed esplora fee estimation.
+- Update the `Database` trait to store the last sync timestamp and block height
+- Rename `ConfirmationTime` to `BlockTime`
 
 ## [v0.13.0] - [v0.12.0]
 

Cargo.toml

@@ -94,7 +94,7 @@ test-md-docs = ["electrum"]
 lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
-electrsd = { version= "0.12", features = ["trigger", "bitcoind_0_21_1"] }
+electrsd = { version= "0.12", features = ["trigger", "bitcoind_22_0"] }
 
 [[example]]
 name = "address_validator"

src/blockchain/compact_filters/mod.rs

@@ -71,7 +71,7 @@ use super::{Blockchain, Capability, ConfigurableBlockchain, Progress};
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
 use crate::types::{KeychainKind, LocalUtxo, TransactionDetails};
-use crate::{ConfirmationTime, FeeRate};
+use crate::{BlockTime, FeeRate};
 
 use peer::*;
 use store::*;
@@ -206,7 +206,7 @@ impl CompactFiltersBlockchain {
                 transaction: Some(tx.clone()),
                 received: incoming,
                 sent: outgoing,
-                confirmation_time: ConfirmationTime::new(height, timestamp),
+                confirmation_time: BlockTime::new(height, timestamp),
                 verified: height.is_some(),
                 fee: Some(inputs_sum.saturating_sub(outputs_sum)),
             };

src/blockchain/electrum.rs

@@ -37,7 +37,7 @@ use super::script_sync::Request;
 use super::*;
 use crate::database::{BatchDatabase, Database};
 use crate::error::Error;
-use crate::{ConfirmationTime, FeeRate};
+use crate::{BlockTime, FeeRate};
 
 /// Wrapper over an Electrum Client that implements the required blockchain traits
 ///
@@ -146,7 +146,7 @@ impl Blockchain for ElectrumBlockchain {
                                 .map(|height| {
                                     let timestamp =
                                         *block_times.get(height).ok_or_else(electrum_goof)?;
-                                    Result::<_, Error>::Ok(ConfirmationTime {
+                                    Result::<_, Error>::Ok(BlockTime {
                                         height: *height,
                                         timestamp: timestamp.into(),
                                     })

src/blockchain/esplora/api.rs

@@ -1,7 +1,7 @@
 //! structs from the esplora API
 //!
 //! see: <https://github.com/Blockstream/esplora/blob/master/API.md>
-use crate::ConfirmationTime;
+use crate::BlockTime;
 use bitcoin::{OutPoint, Script, Transaction, TxIn, TxOut, Txid};
 
 #[derive(serde::Deserialize, Clone, Debug)]
@@ -78,13 +78,13 @@ impl Tx {
         }
     }
 
-    pub fn confirmation_time(&self) -> Option<ConfirmationTime> {
+    pub fn confirmation_time(&self) -> Option<BlockTime> {
         match self.status {
             TxStatus {
                 confirmed: true,
                 block_height: Some(height),
                 block_time: Some(timestamp),
-            } => Some(ConfirmationTime { timestamp, height }),
+            } => Some(BlockTime { timestamp, height }),
             _ => None,
         }
     }

src/blockchain/rpc.rs

@@ -37,7 +37,7 @@ use crate::blockchain::{Blockchain, Capability, ConfigurableBlockchain, Progress
 use crate::database::{BatchDatabase, DatabaseUtils};
 use crate::descriptor::{get_checksum, IntoWalletDescriptor};
 use crate::wallet::utils::SecpCtx;
-use crate::{ConfirmationTime, Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails};
+use crate::{BlockTime, Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails};
 use bitcoincore_rpc::json::{
     GetAddressInfoResultLabel, ImportMultiOptions, ImportMultiRequest,
     ImportMultiRequestScriptPubkey, ImportMultiRescanSince,
@@ -230,7 +230,7 @@ impl Blockchain for RpcBlockchain {
             list_txs_ids.insert(txid);
             if let Some(mut known_tx) = known_txs.get_mut(&txid) {
                 let confirmation_time =
-                    ConfirmationTime::new(tx_result.info.blockheight, tx_result.info.blocktime);
+                    BlockTime::new(tx_result.info.blockheight, tx_result.info.blocktime);
                 if confirmation_time != known_tx.confirmation_time {
                     // reorg may change tx height
                     debug!(
@@ -266,7 +266,7 @@ impl Blockchain for RpcBlockchain {
                 let td = TransactionDetails {
                     transaction: Some(tx),
                     txid: tx_result.info.txid,
-                    confirmation_time: ConfirmationTime::new(
+                    confirmation_time: BlockTime::new(
                         tx_result.info.blockheight,
                         tx_result.info.blocktime,
                     ),

src/blockchain/script_sync.rs

@@ -6,7 +6,7 @@ returns associated transactions i.e. electrum.
 use crate::{
     database::{BatchDatabase, BatchOperations, DatabaseUtils},
     wallet::time::Instant,
-    ConfirmationTime, Error, KeychainKind, LocalUtxo, TransactionDetails,
+    BlockTime, Error, KeychainKind, LocalUtxo, TransactionDetails,
 };
 use bitcoin::{OutPoint, Script, Transaction, TxOut, Txid};
 use log::*;
@@ -246,7 +246,7 @@ impl<'a, D: BatchDatabase> ConftimeReq<'a, D> {
 
     pub fn satisfy(
         mut self,
-        confirmation_times: Vec<Option<ConfirmationTime>>,
+        confirmation_times: Vec<Option<BlockTime>>,
     ) -> Result<Request<'a, D>, Error> {
         let conftime_needed = self
             .request()

src/database/any.rs

@@ -144,6 +144,9 @@ impl BatchOperations for AnyDatabase {
     fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
         impl_inner_method!(AnyDatabase, self, set_last_index, keychain, value)
     }
+    fn set_sync_time(&mut self, sync_time: SyncTime) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_sync_time, sync_time)
+    }
 
     fn del_script_pubkey_from_path(
         &mut self,
@@ -180,6 +183,9 @@ impl BatchOperations for AnyDatabase {
     fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
         impl_inner_method!(AnyDatabase, self, del_last_index, keychain)
     }
+    fn del_sync_time(&mut self) -> Result<Option<SyncTime>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_sync_time)
+    }
 }
 
 impl Database for AnyDatabase {
@@ -241,6 +247,9 @@ impl Database for AnyDatabase {
     fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
         impl_inner_method!(AnyDatabase, self, get_last_index, keychain)
     }
+    fn get_sync_time(&self) -> Result<Option<SyncTime>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_sync_time)
+    }
 
     fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
         impl_inner_method!(AnyDatabase, self, increment_last_index, keychain)
@@ -272,6 +281,9 @@ impl BatchOperations for AnyBatch {
     fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
         impl_inner_method!(AnyBatch, self, set_last_index, keychain, value)
     }
+    fn set_sync_time(&mut self, sync_time: SyncTime) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_sync_time, sync_time)
+    }
 
     fn del_script_pubkey_from_path(
         &mut self,
@@ -302,6 +314,9 @@ impl BatchOperations for AnyBatch {
     fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
         impl_inner_method!(AnyBatch, self, del_last_index, keychain)
     }
+    fn del_sync_time(&mut self) -> Result<Option<SyncTime>, Error> {
+        impl_inner_method!(AnyBatch, self, del_sync_time)
+    }
 }
 
 impl BatchDatabase for AnyDatabase {

src/database/keyvalue.rs

@@ -18,7 +18,7 @@ use bitcoin::hash_types::Txid;
 use bitcoin::{OutPoint, Script, Transaction};
 
 use crate::database::memory::MapKey;
-use crate::database::{BatchDatabase, BatchOperations, Database};
+use crate::database::{BatchDatabase, BatchOperations, Database, SyncTime};
 use crate::error::Error;
 use crate::types::*;
 
@@ -82,6 +82,13 @@ macro_rules! impl_batch_operations {
             Ok(())
         }
 
+        fn set_sync_time(&mut self, data: SyncTime) -> Result<(), Error> {
+            let key = MapKey::SyncTime.as_map_key();
+            self.insert(key, serde_json::to_vec(&data)?)$($after_insert)*;
+
+            Ok(())
+        }
+
         fn del_script_pubkey_from_path(&mut self, keychain: KeychainKind, path: u32) -> Result<Option<Script>, Error> {
             let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
             let res = self.remove(key);
@@ -168,6 +175,14 @@ macro_rules! impl_batch_operations {
                 }
             }
         }
+
+        fn del_sync_time(&mut self) -> Result<Option<SyncTime>, Error> {
+            let key = MapKey::SyncTime.as_map_key();
+            let res = self.remove(key);
+            let res = $process_delete!(res);
+
+            Ok(res.map(|b| serde_json::from_slice(&b)).transpose()?)
+        }
     }
 }
 
@@ -342,6 +357,14 @@ impl Database for Tree {
             .transpose()
     }
 
+    fn get_sync_time(&self) -> Result<Option<SyncTime>, Error> {
+        let key = MapKey::SyncTime.as_map_key();
+        Ok(self
+            .get(key)?
+            .map(|b| serde_json::from_slice(&b))
+            .transpose()?)
+    }
+
     // inserts 0 if not present
     fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
         let key = MapKey::LastIndex(keychain).as_map_key();
@@ -470,4 +493,9 @@ mod test {
     fn test_last_index() {
         crate::database::test::test_last_index(get_tree());
     }
+
+    #[test]
+    fn test_sync_time() {
+        crate::database::test::test_sync_time(get_tree());
+    }
 }

src/database/memory.rs

@@ -22,7 +22,7 @@ use bitcoin::consensus::encode::{deserialize, serialize};
 use bitcoin::hash_types::Txid;
 use bitcoin::{OutPoint, Script, Transaction};
 
-use crate::database::{BatchDatabase, BatchOperations, ConfigurableDatabase, Database};
+use crate::database::{BatchDatabase, BatchOperations, ConfigurableDatabase, Database, SyncTime};
 use crate::error::Error;
 use crate::types::*;
 
@@ -33,6 +33,7 @@ use crate::types::*;
 // transactions         t<txid> -> tx details
 // deriv indexes        c{i,e} -> u32
 // descriptor checksum  d{i,e} -> vec<u8>
+// last sync time       l -> { height, timestamp }
 
 pub(crate) enum MapKey<'a> {
     Path((Option<KeychainKind>, Option<u32>)),
@@ -41,6 +42,7 @@ pub(crate) enum MapKey<'a> {
     RawTx(Option<&'a Txid>),
     Transaction(Option<&'a Txid>),
     LastIndex(KeychainKind),
+    SyncTime,
     DescriptorChecksum(KeychainKind),
 }
 
@@ -59,6 +61,7 @@ impl MapKey<'_> {
             MapKey::RawTx(_) => b"r".to_vec(),
             MapKey::Transaction(_) => b"t".to_vec(),
             MapKey::LastIndex(st) => [b"c", st.as_ref()].concat(),
+            MapKey::SyncTime => b"l".to_vec(),
             MapKey::DescriptorChecksum(st) => [b"d", st.as_ref()].concat(),
         }
     }
@@ -180,6 +183,12 @@ impl BatchOperations for MemoryDatabase {
 
         Ok(())
     }
+    fn set_sync_time(&mut self, data: SyncTime) -> Result<(), Error> {
+        let key = MapKey::SyncTime.as_map_key();
+        self.map.insert(key, Box::new(data));
+
+        Ok(())
+    }
 
     fn del_script_pubkey_from_path(
         &mut self,
@@ -270,6 +279,13 @@ impl BatchOperations for MemoryDatabase {
             Some(b) => Ok(Some(*b.downcast_ref().unwrap())),
         }
     }
+    fn del_sync_time(&mut self) -> Result<Option<SyncTime>, Error> {
+        let key = MapKey::SyncTime.as_map_key();
+        let res = self.map.remove(&key);
+        self.deleted_keys.push(key);
+
+        Ok(res.map(|b| b.downcast_ref().cloned().unwrap()))
+    }
 }
 
 impl Database for MemoryDatabase {
@@ -407,6 +423,14 @@ impl Database for MemoryDatabase {
         Ok(self.map.get(&key).map(|b| *b.downcast_ref().unwrap()))
     }
 
+    fn get_sync_time(&self) -> Result<Option<SyncTime>, Error> {
+        let key = MapKey::SyncTime.as_map_key();
+        Ok(self
+            .map
+            .get(&key)
+            .map(|b| b.downcast_ref().cloned().unwrap()))
+    }
+
     // inserts 0 if not present
     fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
         let key = MapKey::LastIndex(keychain).as_map_key();
@@ -479,12 +503,10 @@ macro_rules! populate_test_db {
         };
 
         let txid = tx.txid();
-        let confirmation_time = tx_meta
-            .min_confirmations
-            .map(|conf| $crate::ConfirmationTime {
-                height: current_height.unwrap().checked_sub(conf as u32).unwrap(),
-                timestamp: 0,
-            });
+        let confirmation_time = tx_meta.min_confirmations.map(|conf| $crate::BlockTime {
+            height: current_height.unwrap().checked_sub(conf as u32).unwrap(),
+            timestamp: 0,
+        });
 
         let tx_details = $crate::TransactionDetails {
             transaction: Some(tx.clone()),
@@ -590,4 +612,9 @@ mod test {
     fn test_last_index() {
         crate::database::test::test_last_index(get_tree());
     }
+
+    #[test]
+    fn test_sync_time() {
+        crate::database::test::test_sync_time(get_tree());
+    }
 }