Merge currency specific bitcoin wallets

bitcoin/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 default = []
 regtest-manual-mining = []
 cli = ["clap"]
-uses-bitcoind = []
+uses-bitcoind = ["regtest-manual-mining"]
 light-client = []
 
 [dependencies]

bitcoin/src/iter.rs

@@ -207,15 +207,16 @@ mod tests {
             async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result<Block, Error>;
             fn get_balance(&self, min_confirmations: Option<u32>) -> Result<Amount, Error>;
             fn list_transactions(&self, max_count: Option<usize>) -> Result<Vec<json::ListTransactionResult>, Error>;
+            fn list_addresses(&self) -> Result<Vec<Address>, Error>;
             async fn get_block_count(&self) -> Result<u64, Error>;
             async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result<Vec<u8>, Error>;
             async fn get_transaction(&self, txid: &Txid, block_hash: Option<BlockHash>) -> Result<Transaction, Error>;
             async fn get_proof(&self, txid: Txid, block_hash: &BlockHash) -> Result<Vec<u8>, Error>;
             async fn get_block_hash(&self, height: u32) -> Result<BlockHash, Error>;
             async fn get_new_address(&self) -> Result<Address, Error>;
             async fn get_new_public_key(&self) -> Result<PublicKey, Error>;
-            fn dump_derivation_key(&self, public_key: &PublicKey) -> Result<PrivateKey, Error>;
-            fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), Error>;
+            fn dump_private_key(&self, address: &Address) -> Result<PrivateKey, Error>;
+            fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), Error>;
             async fn add_new_deposit_key(
                 &self,
                 public_key: PublicKey,

bitcoin/src/lib.rs

@@ -51,6 +51,7 @@ pub use sp_core::H256;
 use std::{
     convert::TryInto,
     future::Future,
+    str::FromStr,
     sync::Arc,
     time::{Duration, Instant},
 };
@@ -139,6 +140,8 @@ pub trait BitcoinCoreApi {
 
     fn list_transactions(&self, max_count: Option<usize>) -> Result<Vec<json::ListTransactionResult>, Error>;
 
+    fn list_addresses(&self) -> Result<Vec<Address>, Error>;
+
     async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result<Vec<u8>, Error>;
 
     async fn get_transaction(&self, txid: &Txid, block_hash: Option<BlockHash>) -> Result<Transaction, Error>;
@@ -151,9 +154,9 @@ pub trait BitcoinCoreApi {
 
     async fn get_new_public_key(&self) -> Result<PublicKey, Error>;
 
-    fn dump_derivation_key(&self, public_key: &PublicKey) -> Result<PrivateKey, Error>;
+    fn dump_private_key(&self, address: &Address) -> Result<PrivateKey, Error>;
 
-    fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), Error>;
+    fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), Error>;
 
     async fn add_new_deposit_key(&self, public_key: PublicKey, secret_key: Vec<u8>) -> Result<(), Error>;
 
@@ -531,10 +534,15 @@ impl BitcoinCore {
     }
 
     #[cfg(feature = "regtest-manual-mining")]
-    pub fn mine_block(&self) -> Result<BlockHash, Error> {
-        Ok(self
-            .rpc
-            .generate_to_address(1, &self.rpc.get_new_address(None, Some(AddressType::Bech32))?)?[0])
+    pub fn mine_blocks(&self, block_num: u64, maybe_address: Option<Address>) -> BlockHash {
+        let address =
+            maybe_address.unwrap_or_else(|| self.rpc.get_new_address(None, Some(AddressType::Bech32)).unwrap());
+        self.rpc
+            .generate_to_address(block_num, &address)
+            .unwrap()
+            .last()
+            .unwrap()
+            .clone()
     }
 
     async fn with_retry_on_timeout<F, R, T>(&self, call: F) -> Result<T, Error>
@@ -692,6 +700,20 @@ impl BitcoinCoreApi for BitcoinCore {
             .list_transactions(None, max_count.or(Some(DEFAULT_MAX_TX_COUNT)), None, None)?)
     }
 
+    // TODO: remove this once the wallet migration has completed
+    fn list_addresses(&self) -> Result<Vec<Address>, Error> {
+        // Lists groups of addresses which have had their common ownership
+        // made public by common use as inputs or as the resulting change
+        // in past transactions
+        let groupings: Vec<Vec<Vec<serde_json::Value>>> = self.rpc.call("listaddressgroupings", &[])?;
+        let addresses = groupings
+            .into_iter()
+            .flatten()
+            .filter_map(|group| group.get(0).and_then(|v| v.as_str()).map(Address::from_str)?.ok())
+            .collect::<Vec<_>>();
+        Ok(addresses)
+    }
+
     /// Get the raw transaction identified by `Txid` and stored
     /// in the specified block.
     ///
@@ -753,15 +775,16 @@ impl BitcoinCoreApi for BitcoinCore {
         Ok(public_key)
     }
 
-    fn dump_derivation_key(&self, public_key: &PublicKey) -> Result<PrivateKey, Error> {
-        let address = Address::p2wpkh(public_key, self.network).map_err(ConversionError::from)?;
-        Ok(self.rpc.dump_private_key(&address)?)
+    fn dump_private_key(&self, address: &Address) -> Result<PrivateKey, Error> {
+        Ok(self.rpc.dump_private_key(address)?)
     }
 
-    fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), Error> {
-        Ok(self
-            .rpc
-            .import_private_key(private_key, Some(DERIVATION_KEY_LABEL), Some(false))?)
+    fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), Error> {
+        Ok(self.rpc.import_private_key(
+            private_key,
+            is_derivation_key.then_some(DERIVATION_KEY_LABEL),
+            Some(false),
+        )?)
     }
 
     /// Derive and import the private key for the master public key and public secret

bitcoin/src/light/mod.rs

@@ -133,6 +133,12 @@ impl BitcoinCoreApi for BitcoinLight {
         Ok(Default::default())
     }
 
+    // TODO: remove this later
+    fn list_addresses(&self) -> Result<Vec<Address>, BitcoinError> {
+        // don't need to migrate keys
+        Ok(Default::default())
+    }
+
     async fn get_raw_tx(&self, txid: &Txid, _block_hash: &BlockHash) -> Result<Vec<u8>, BitcoinError> {
         Ok(self.electrs.get_raw_tx(txid).await?)
     }
@@ -162,11 +168,11 @@ impl BitcoinCoreApi for BitcoinLight {
         Ok(self.private_key.public_key(&self.secp_ctx))
     }
 
-    fn dump_derivation_key(&self, _public_key: &PublicKey) -> Result<PrivateKey, BitcoinError> {
-        Ok(self.private_key)
+    fn dump_private_key(&self, _: &Address) -> Result<PrivateKey, BitcoinError> {
+        Err(Error::InvalidAddress.into())
     }
 
-    fn import_derivation_key(&self, _private_key: &PrivateKey) -> Result<(), BitcoinError> {
+    fn import_private_key(&self, _private_key: &PrivateKey, _is_derivation_key: bool) -> Result<(), BitcoinError> {
         // nothing to do
         Ok(())
     }

bitcoin/tests/integration_test.rs

@@ -26,6 +26,22 @@ async fn should_get_new_address() -> Result<(), Error> {
     Ok(())
 }
 
+#[tokio::test]
+async fn should_list_addresses() -> Result<(), Error> {
+    let btc_rpc = new_bitcoin_core(Some("Alice".to_string()))?;
+    btc_rpc.create_or_load_wallet().await?;
+
+    let address = btc_rpc.get_new_address().await?;
+    btc_rpc.mine_blocks(101, Some(address.clone()));
+
+    assert!(
+        btc_rpc.list_addresses()?.contains(&address),
+        "Address not found in groupings"
+    );
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn should_get_new_public_key() -> Result<(), Error> {
     let btc_rpc = new_bitcoin_core(Some("Bob".to_string()))?;

runtime/src/integration/bitcoin_simulator.rs

@@ -326,6 +326,9 @@ impl BitcoinCoreApi for MockBitcoinCore {
     fn list_transactions(&self, max_count: Option<usize>) -> Result<Vec<json::ListTransactionResult>, BitcoinError> {
         Ok(vec![])
     }
+    fn list_addresses(&self) -> Result<Vec<Address>, BitcoinError> {
+        Ok(vec![])
+    }
     async fn get_block_count(&self) -> Result<u64, BitcoinError> {
         Ok((self.blocks.read().await.len() - 1).try_into().unwrap())
     }
@@ -385,10 +388,10 @@ impl BitcoinCoreApi for MockBitcoinCore {
         let public_key = secp256k1::PublicKey::from_secret_key(&secp, &secret_key);
         Ok(PublicKey::new(public_key))
     }
-    fn dump_derivation_key(&self, public_key: &PublicKey) -> Result<PrivateKey, BitcoinError> {
+    fn dump_private_key(&self, address: &Address) -> Result<PrivateKey, BitcoinError> {
         todo!()
     }
-    fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), BitcoinError> {
+    fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), BitcoinError> {
         todo!()
     }
     async fn add_new_deposit_key(&self, _public_key: PublicKey, _secret_key: Vec<u8>) -> Result<(), BitcoinError> {

service/src/lib.rs

@@ -28,7 +28,8 @@ pub trait Service<Config, InnerError> {
 
     fn new_service(
         btc_parachain: BtcParachain,
-        bitcoin_core: DynBitcoinCoreApi,
+        bitcoin_core_master: DynBitcoinCoreApi,
+        bitcoin_core_shared: DynBitcoinCoreApi,
         config: Config,
         monitoring_config: MonitoringConfig,
         shutdown: ShutdownSender,
@@ -87,7 +88,7 @@ impl<Config: Clone + Send + 'static, F: Fn()> ConnectionManager<Config, F> {
             let shutdown_tx = ShutdownSender::new();
 
             let prefix = self.wallet_name.clone().unwrap_or_else(|| "vault".to_string());
-            let bitcoin_core = self.bitcoin_config.new_client(Some(format!("{prefix}-master"))).await?;
+            let bitcoin_core_master = self.bitcoin_config.new_client(Some(format!("{prefix}-master"))).await?;
 
             // only open connection to parachain after bitcoind sync to prevent timeout
             let signer = self.signer.clone();
@@ -102,7 +103,14 @@ impl<Config: Clone + Send + 'static, F: Fn()> ConnectionManager<Config, F> {
             .await?;
 
             let config_copy = self.bitcoin_config.clone();
-            let network_copy = bitcoin_core.network();
+            let network_copy = bitcoin_core_master.network();
+
+            // use a separate wallet for all bitcoin transactions
+            // to make exporting the private key easier from the
+            // master wallet if we switch to descriptor wallets
+            let bitcoin_core_shared =
+                config_copy.new_client_with_network(Some(format!("{prefix}-shared")), network_copy)?;
+
             let constructor = move |vault_id: VaultId| {
                 let collateral_currency: CurrencyId = vault_id.collateral_currency();
                 let wrapped_currency: CurrencyId = vault_id.wrapped_currency();
@@ -121,7 +129,8 @@ impl<Config: Clone + Send + 'static, F: Fn()> ConnectionManager<Config, F> {
 
             let service = S::new_service(
                 btc_parachain,
-                bitcoin_core,
+                bitcoin_core_master,
+                bitcoin_core_shared,
                 config,
                 self.monitoring_config.clone(),
                 shutdown_tx.clone(),

vault/src/execution.rs

@@ -655,7 +655,7 @@ fn get_request_for_btc_tx(tx: &Transaction, hash_map: &HashMap<H256, Request>) -
     }
 }
 
-#[cfg(all(test, feature = "parachain-metadata-kintsugi-testnet"))]
+#[cfg(all(test, feature = "parachain-metadata-kintsugi"))]
 mod tests {
     use super::*;
     use crate::metrics::PerCurrencyMetrics;
@@ -699,6 +699,7 @@ mod tests {
             async fn get_foreign_asset_metadata(&self, id: u32) -> Result<AssetMetadata, RuntimeError>;
             async fn get_lend_tokens(&self) -> Result<Vec<(CurrencyId, CurrencyId)>, RuntimeError>;
         }
+
         #[async_trait]
         pub trait VaultRegistryPallet {
             async fn get_vault(&self, vault_id: &VaultId) -> Result<InterBtcVault, RuntimeError>;
@@ -793,6 +794,7 @@ mod tests {
             async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result<Block, BitcoinError>;
             fn get_balance(&self, min_confirmations: Option<u32>) -> Result<Amount, BitcoinError>;
             fn list_transactions(&self, max_count: Option<usize>) -> Result<Vec<json::ListTransactionResult>, BitcoinError>;
+            fn list_addresses(&self) -> Result<Vec<Address>, BitcoinError>;
             async fn get_block_count(&self) -> Result<u64, BitcoinError>;
             async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result<Vec<u8>, BitcoinError>;
             async fn get_transaction(&self, txid: &Txid, block_hash: Option<BlockHash>) -> Result<Transaction, BitcoinError>;
@@ -801,8 +803,8 @@ mod tests {
             async fn get_pruned_height(&self) -> Result<u64, BitcoinError>;
             async fn get_new_address(&self) -> Result<Address, BitcoinError>;
             async fn get_new_public_key(&self) -> Result<PublicKey, BitcoinError>;
-            fn dump_derivation_key(&self, public_key: &PublicKey) -> Result<PrivateKey, BitcoinError>;
-            fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), BitcoinError>;
+            fn dump_private_key(&self, address: &Address) -> Result<PrivateKey, BitcoinError>;
+            fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), BitcoinError>;
             async fn add_new_deposit_key(&self, public_key: PublicKey, secret_key: Vec<u8>) -> Result<(), BitcoinError>;
             async fn get_best_block_hash(&self) -> Result<BlockHash, BitcoinError>;
             async fn get_block(&self, hash: &BlockHash) -> Result<Block, BitcoinError>;

vault/src/issue.rs

@@ -1,12 +1,13 @@
 use crate::{
-    delay::RandomDelay, metrics::publish_expected_bitcoin_balance, Error, Event, IssueRequests, VaultIdManager,
+    delay::RandomDelay, metrics::publish_expected_bitcoin_balance, system::DatabaseConfig, Error, Event, IssueRequests,
+    VaultIdManager,
 };
 use bitcoin::{BlockHash, Error as BitcoinError, PublicKey, Transaction, TransactionExt};
 use futures::{channel::mpsc::Sender, future, SinkExt, StreamExt, TryFutureExt};
 use runtime::{
-    BtcAddress, BtcPublicKey, BtcRelayPallet, CancelIssueEvent, ExecuteIssueEvent, H256Le, InterBtcIssueRequest,
-    InterBtcParachain, IssuePallet, IssueRequestStatus, PartialAddress, PrettyPrint, RequestIssueEvent, UtilFuncs,
-    VaultId, H256,
+    AccountId, BtcAddress, BtcPublicKey, BtcRelayPallet, CancelIssueEvent, ExecuteIssueEvent, H256Le,
+    InterBtcIssueRequest, InterBtcParachain, IssuePallet, IssueRequestStatus, PartialAddress, PrettyPrint,
+    RequestIssueEvent, UtilFuncs, H256,
 };
 use service::{DynBitcoinCoreApi, Error as ServiceError};
 use sha2::{Digest, Sha256};
@@ -82,10 +83,11 @@ struct RescanStatus {
     newest_issue_height: u32,
     queued_rescan_range: Option<(usize, usize)>, // start, end(including)
 }
+
 impl RescanStatus {
     // there was a bug pre-v2 that set rescanning status to an invalid range.
     // by changing the keyname we effectively force a reset
-    const KEY: &str = "rescan-status-v2";
+    const KEY: &str = "rescan-status-v3";
     fn update(&mut self, mut issues: Vec<InterBtcIssueRequest>, current_bitcoin_height: usize) {
         // Only look at issues that haven't been processed yet
         issues.retain(|issue| issue.opentime > self.newest_issue_height);
@@ -133,38 +135,34 @@ impl RescanStatus {
         Some((start, chunk_end))
     }
 
-    fn get(vault_id: &VaultId, db: &crate::system::DatabaseConfig) -> Result<Self, Error> {
-        Ok(db.get(vault_id, Self::KEY)?.unwrap_or_default())
+    fn get(account_id: &AccountId, db: &DatabaseConfig) -> Result<Self, Error> {
+        Ok(db.get(account_id, Self::KEY)?.unwrap_or_default())
     }
-    fn store(&self, vault_id: &VaultId, db: &crate::system::DatabaseConfig) -> Result<(), Error> {
-        db.put(vault_id, Self::KEY, self)?;
+
+    fn store(&self, account_id: &AccountId, db: &DatabaseConfig) -> Result<(), Error> {
+        db.put(account_id, Self::KEY, self)?;
         Ok(())
     }
 }
 
 pub async fn add_keys_from_past_issue_request(
     bitcoin_core: &DynBitcoinCoreApi,
     btc_parachain: &InterBtcParachain,
-    vault_id: &VaultId,
-    db: &crate::system::DatabaseConfig,
+    db: &DatabaseConfig,
 ) -> Result<(), Error> {
-    let mut scanning_status = RescanStatus::get(vault_id, db)?;
-    tracing::info!("initial status: = {scanning_status:?}");
+    let account_id = btc_parachain.get_account_id();
+    let mut scanning_status = RescanStatus::get(&account_id, db)?;
+    tracing::info!("Scanning: {scanning_status:?}");
 
-    let issue_requests: Vec<_> = btc_parachain
-        .get_vault_issue_requests(btc_parachain.get_account_id().clone())
-        .await?
-        .into_iter()
-        .filter(|(_, issue)| &issue.vault == vault_id)
-        .collect();
+    let issue_requests = btc_parachain.get_vault_issue_requests(account_id.clone()).await?;
 
     for (issue_id, request) in issue_requests.clone().into_iter() {
         if let Err(e) = add_new_deposit_key(bitcoin_core, issue_id, request.btc_public_key).await {
             tracing::error!("Failed to add deposit key #{}: {}", issue_id, e.to_string());
         }
     }
 
-    // read height only _after_ the last add_new_deposit_key.If a new block arrives
+    // read height only _after_ the last add_new_deposit_key. If a new block arrives
     // while we rescan, bitcoin core will correctly recognize addressed associated with the
     // privkey
     let btc_end_height = bitcoin_core.get_block_count().await? as usize;
@@ -197,7 +195,7 @@ pub async fn add_keys_from_past_issue_request(
     }
 
     // save progress s.t. we don't rescan pruned range again if we crash now
-    scanning_status.store(vault_id, db)?;
+    scanning_status.store(account_id, db)?;
 
     let mut chunk_size = 1;
     // rescan the blockchain in chunks, so that we can save progress. The code below
@@ -217,7 +215,7 @@ pub async fn add_keys_from_past_issue_request(
             chunk_size = (chunk_size.checked_div(2).ok_or(Error::ArithmeticUnderflow)?).max(1);
         }
 
-        scanning_status.store(vault_id, db)?;
+        scanning_status.store(account_id, db)?;
     }
 
     Ok(())
@@ -465,9 +463,9 @@ mod tests {
     use super::*;
     use runtime::{
         subxt::utils::Static,
-        AccountId,
         CurrencyId::Token,
         TokenSymbol::{DOT, IBTC, INTR},
+        VaultId,
     };
 
     fn dummy_issues(heights: Vec<(u32, usize)>) -> Vec<InterBtcIssueRequest> {