From b53c03d0500bf5b27dc58ee483e3ba780e5e7cdc Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Sun, 16 Jan 2022 08:24:04 +0100 Subject: [PATCH 01/74] feat(unit_tests): add begin of unit test for spv proof --- mm2src/coins/utxo/utxo_tests.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index b241bd6e06..6faac9c2dd 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -903,6 +903,30 @@ fn test_utxo_lock() { } } +#[test] +#[ignore] +fn test_spv_proof() { + ElectrumClient::get_transaction_bytes.mock_safe(move |_, _| { + let bytes: BytesJson = hex::decode("400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a4730440220108f40ab98a1b7cd46195814efb7f37a7a7ec8fe4953cf411fae5bd920af98bd022029e40468d3ab873c81901ae1f5b291d8bc5745c693f62bc8b19884318b0f2c0f012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9142bd37349752d1751d5cc49eed71029010a83f828870000000000000000166a14b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc612d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac5dc4e361000000000000000000000000000000").unwrap().into(); + MockResult::Return(Box::new(futures01::future::ok(bytes))) + }); + let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); + let coin = utxo_coin_for_test( + client.into(), + Some("spice describe gravity federal blast come thank unfair canal monkey style afraid"), + false, + ); + let empty_hash = H256Json::default(); + let result = coin + .as_ref() + .rpc_client + .get_transaction_bytes(&empty_hash) + .wait() + .unwrap(); + let expected: BytesJson = hex::decode("400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a4730440220108f40ab98a1b7cd46195814efb7f37a7a7ec8fe4953cf411fae5bd920af98bd022029e40468d3ab873c81901ae1f5b291d8bc5745c693f62bc8b19884318b0f2c0f012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9142bd37349752d1751d5cc49eed71029010a83f828870000000000000000166a14b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc612d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac5dc4e361000000000000000000000000000000").unwrap().into(); + assert_eq!(result, expected); +} + #[test] fn list_since_block_btc_serde() { // https://github.com/KomodoPlatform/atomicDEX-API/issues/563 From 8b1ccc5122283f825fd45fac9f0cb975e2add103 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 17 Jan 2022 09:16:15 +0100 Subject: [PATCH 02/74] feat(utxo): continue test --- mm2src/coins/utxo/utxo_tests.rs | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 6faac9c2dd..b881cebc0b 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -903,11 +903,32 @@ fn test_utxo_lock() { } } +/*#[test] +fn test_spv_fraud_proof() { + let secret = [0; 20]; + let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); + let coin = utxo_coin_for_test( + client.into(), + Some("spice describe gravity federal blast come thank unfair canal monkey style afraid"), + false, + ); + let CoinBalance { spendable, unspendable } = coin.my_balance().wait().unwrap(); + println!("{} {}", spendable, unspendable); + let timeout = (now_ms() / 1000) + 240; // timeout if test takes more than 240 seconds to run + let my_public_key = coin.my_public_key().unwrap(); + let time_lock = (now_ms() / 1000) as u32 - 3600; + + let tx = coin + .send_maker_payment(time_lock, my_public_key, &[0; 20], 1u64.into(), &None) + .wait() + .unwrap(); + println!("yay"); +}*/ + #[test] -#[ignore] fn test_spv_proof() { ElectrumClient::get_transaction_bytes.mock_safe(move |_, _| { - let bytes: BytesJson = hex::decode("400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a4730440220108f40ab98a1b7cd46195814efb7f37a7a7ec8fe4953cf411fae5bd920af98bd022029e40468d3ab873c81901ae1f5b291d8bc5745c693f62bc8b19884318b0f2c0f012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9142bd37349752d1751d5cc49eed71029010a83f828870000000000000000166a14b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc612d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac5dc4e361000000000000000000000000000000").unwrap().into(); + let bytes: BytesJson = hex::decode("0400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a473044022064318d9178eb8d5d098c1d2c9e90b7685521a9946fbbf6c4d482e6ce2e82959a0220425a675c1665f159109001176a4425e4a35a52c81b4a8a42e34ed18485e1d44d012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9147f4d6f4db8587bd878e05c83660f515800d75098870000000000000000166a14000000000000000000000000000000000000000012d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac7c10e561000000000000000000000000000000").unwrap().into(); MockResult::Return(Box::new(futures01::future::ok(bytes))) }); let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); @@ -923,8 +944,11 @@ fn test_spv_proof() { .get_transaction_bytes(&empty_hash) .wait() .unwrap(); - let expected: BytesJson = hex::decode("400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a4730440220108f40ab98a1b7cd46195814efb7f37a7a7ec8fe4953cf411fae5bd920af98bd022029e40468d3ab873c81901ae1f5b291d8bc5745c693f62bc8b19884318b0f2c0f012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9142bd37349752d1751d5cc49eed71029010a83f828870000000000000000166a14b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc612d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac5dc4e361000000000000000000000000000000").unwrap().into(); + let expected: BytesJson = hex::decode("0400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a473044022064318d9178eb8d5d098c1d2c9e90b7685521a9946fbbf6c4d482e6ce2e82959a0220425a675c1665f159109001176a4425e4a35a52c81b4a8a42e34ed18485e1d44d012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9147f4d6f4db8587bd878e05c83660f515800d75098870000000000000000166a14000000000000000000000000000000000000000012d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac7c10e561000000000000000000000000000000").unwrap().into(); assert_eq!(result, expected); + + //! get_transaction_bytes return a crafted transaction + //validate_spv() } #[test] From b175b7411e23db17230e9013c4cda92403319ac5 Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Tue, 18 Jan 2022 08:37:30 +0100 Subject: [PATCH 03/74] feat(unit_tests): very wip --- Cargo.lock | 113 +++++++++++++++++++++++++++---- mm2src/coins/Cargo.toml | 1 + mm2src/coins/utxo/utxo_common.rs | 71 ++++++++++++++++++- mm2src/coins/utxo/utxo_tests.rs | 40 ++++++++++- 4 files changed, 207 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 142e4dccd4..e352851d7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -427,6 +427,20 @@ dependencies = [ "secp256k1", ] +[[package]] +name = "bitcoin-spv" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aefc3d887fcbb15bd92c1490308bf62df200ff338a9b67e4863dc7012c052cce" +dependencies = [ + "hex 0.4.3", + "primitive-types 0.7.3", + "ripemd160 0.8.0", + "serde", + "serde_json", + "sha2 0.8.2", +] + [[package]] name = "bitcoin_hashes" version = "0.10.0" @@ -461,6 +475,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "bitvec" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41262f11d771fd4a61aa3ce019fca363b4b6c282fca9da2a31186d3965a47a5c" +dependencies = [ + "either", + "radium 0.3.0", +] + [[package]] name = "bitvec" version = "0.18.5" @@ -623,6 +647,12 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +[[package]] +name = "byte-slice-cast" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" + [[package]] name = "byte-slice-cast" version = "1.0.0" @@ -789,6 +819,7 @@ dependencies = [ "bigdecimal", "bip32", "bitcoin", + "bitcoin-spv", "bitcoin_hashes", "bitcrypto", "byteorder 1.4.3", @@ -1543,7 +1574,7 @@ dependencies = [ "fixed-hash 0.7.0", "impl-rlp", "impl-serde", - "primitive-types", + "primitive-types 0.9.1", "uint 0.9.1", ] @@ -1629,6 +1660,18 @@ dependencies = [ "rustc-hex 2.1.0", ] +[[package]] +name = "fixed-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11498d382790b7a8f2fd211780bec78619bba81cdad3a283997c0c41f836759c" +dependencies = [ + "byteorder 1.4.3", + "rand 0.7.3", + "rustc-hex 2.1.0", + "static_assertions", +] + [[package]] name = "fixed-hash" version = "0.7.0" @@ -2346,13 +2389,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "impl-codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" +dependencies = [ + "parity-scale-codec 1.3.7", +] + [[package]] name = "impl-codec" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "161ebdfec3c8e3b52bf61c4f3550a1eea4f9579d10dc1b936f3171ebdcd6c443" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 2.2.0", ] [[package]] @@ -3597,6 +3649,18 @@ dependencies = [ "group", ] +[[package]] +name = "parity-scale-codec" +version = "1.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4b26b16c7687c3075982af47719e481815df30bc544f7a6690763a25ca16e9d" +dependencies = [ + "arrayvec 0.5.1", + "bitvec 0.17.4", + "byte-slice-cast 0.3.5", + "serde", +] + [[package]] name = "parity-scale-codec" version = "2.2.0" @@ -3605,7 +3669,7 @@ checksum = "8975095a2a03bbbdc70a74ab11a4f76a6d0b84680d87c68d722531b0ac28e8a9" dependencies = [ "arrayvec 0.7.1", "bitvec 0.20.4", - "byte-slice-cast", + "byte-slice-cast 1.0.0", "impl-trait-for-tuples", "parity-scale-codec-derive", "serde", @@ -3642,7 +3706,7 @@ dependencies = [ "lru 0.6.0", "parity-util-mem-derive", "parking_lot 0.11.1", - "primitive-types", + "primitive-types 0.9.1", "smallvec 1.6.1", "winapi", ] @@ -3893,6 +3957,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" +[[package]] +name = "primitive-types" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd39dcacf71411ba488570da7bbc89b717225e46478b30ba99b92db6b149809" +dependencies = [ + "fixed-hash 0.6.1", + "impl-codec 0.4.2", + "uint 0.8.5", +] + [[package]] name = "primitive-types" version = "0.9.1" @@ -3900,7 +3975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06345ee39fbccfb06ab45f3a1a5798d9dafa04cb8921a76d227040003a234b0e" dependencies = [ "fixed-hash 0.7.0", - "impl-codec", + "impl-codec 0.5.1", "impl-rlp", "impl-serde", "uint 0.9.1", @@ -5144,9 +5219,9 @@ dependencies = [ "hash256-std-hasher", "log 0.4.11", "num-traits 0.2.12", - "parity-scale-codec", + "parity-scale-codec 2.2.0", "parity-util-mem", - "primitive-types", + "primitive-types 0.9.1", "secrecy", "sp-debug-derive", "sp-runtime-interface", @@ -5173,8 +5248,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2e5c88b4bc8d607e4e2ff767a85db58cf7101f3dd6064f06929342ea67fe8fb" dependencies = [ "impl-trait-for-tuples", - "parity-scale-codec", - "primitive-types", + "parity-scale-codec 2.2.0", + "primitive-types 0.9.1", "sp-runtime-interface-proc-macro", "sp-std", "sp-storage", @@ -5208,7 +5283,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86af458d4a0251c490cdde9dcaaccb88d398f3b97ac6694cdd49ed9337e6b961" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 2.2.0", "ref-cast", "sp-debug-derive", "sp-std", @@ -5220,7 +5295,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567382d8d4e14fb572752863b5cd57a78f9e9a6583332b590b726f061f3ea957" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 2.2.0", "sp-std", "tracing", "tracing-core", @@ -5234,7 +5309,7 @@ checksum = "b85b7f745da41ef825c6f7b93f1fdc897b03df94a4884adfbb70fbcd0aed1298" dependencies = [ "hash-db", "memory-db", - "parity-scale-codec", + "parity-scale-codec 2.2.0", "sp-core", "sp-std", "trie-db", @@ -5248,7 +5323,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b214e125666a6416cf30a70cc6a5dacd34a4e5197f8a3d479f714af7e1dc7a47" dependencies = [ "impl-trait-for-tuples", - "parity-scale-codec", + "parity-scale-codec 2.2.0", "sp-std", ] @@ -5893,6 +5968,18 @@ dependencies = [ "rustc-hex 2.1.0", ] +[[package]] +name = "uint" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9db035e67dfaf7edd9aebfe8676afcd63eed53c8a4044fed514c8cccf1835177" +dependencies = [ + "byteorder 1.4.3", + "crunchy 0.2.2", + "rustc-hex 2.1.0", + "static_assertions", +] + [[package]] name = "uint" version = "0.9.1" diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index 54c6a02099..2274cc2c5f 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -19,6 +19,7 @@ bigdecimal = { version = "0.1.0", features = ["serde"] } bip32 = { version = "0.2.2", default-features = false, features = ["alloc", "secp256k1-ffi"] } bitcoin = "0.27.1" bitcoin_hashes = "0.10.0" +bitcoin-spv = "5.0.0" bitcrypto = { path = "../mm2_bitcoin/crypto" } byteorder = "1.3" bytes = "0.4" diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index a10f2d88fd..945f3b90a4 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -5,9 +5,11 @@ use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentI use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, TradePreimageValue, TxFeeDetails, ValidateAddressResult, WithdrawResult}; use bigdecimal::{BigDecimal, Zero}; +use bitcoin_spv::std_types::{BitcoinHeader, SPVProof}; +use bitcoin_spv::types::SPVError; pub use bitcrypto::{dhash160, sha256, ChecksumType}; use chain::constants::SEQUENCE_FINAL; -use chain::{OutPoint, TransactionOutput}; +use chain::{BlockHeader, OutPoint, TransactionOutput}; use common::executor::Timer; use common::jsonrpc_client::{JsonRpcError, JsonRpcErrorType}; use common::log::{error, info, warn}; @@ -2662,6 +2664,71 @@ pub fn address_from_pubkey( } } +pub async fn validate_spv_proof(coin: T, tx: UtxoTx, script_pubkey: Bytes) -> Result<(), String> +where + T: AsRef + Send + Sync + 'static, +{ + // todo: refactor to real error type + let script_pubkey_str = hex::encode(&script_pubkey); + let client = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => return ERR!("Native not supported for spv proof."), + UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, + }; + let history = client + .scripthash_get_history(script_pubkey_str.as_str()) + .compat() + .await + .unwrap_or_default(); + if history.is_empty() { + return ERR!("Unable to get scripthash history - invalid transaction"); + } + let mut height: u64 = 0; + for item in history { + if item.tx_hash == H256Json(*tx.hash()) && item.height > 0 { + println!("height: {}", item.height); + height = item.height as u64; + break; + } + } + if height == 0 { + return ERR!("Unable to get a proper transaction height - invalid transaction"); + } + + let block_header = client.blockchain_block_header(height).compat().await; + + match block_header { + Ok(header_bytes) => { + // todo: deserialize into block_header and convert it to BitcoinHeader for the spv proof library + }, + Err(err) => return ERR!("Unable to get a block header - invalid transaction"), + }; + + // todo: implement get merkle tree tsc + + // todo: fill the SPVProof data structure correctly + let proof = SPVProof { + version: vec![], + vin: vec![], + vout: vec![], + locktime: vec![], + tx_id: Default::default(), + index: 0, + confirming_header: BitcoinHeader { + hash: Default::default(), + raw: Default::default(), + height: 0, + prevhash: Default::default(), + merkle_root: Default::default(), + }, + intermediate_nodes: vec![], + }; + match proof.validate() { + Ok(_) => {}, + Err(err) => return ERR!("{:?}", err), + }; + Ok(()) +} + #[allow(clippy::too_many_arguments)] pub fn validate_payment( coin: T, @@ -2727,7 +2794,7 @@ where expected_output ); } - return Ok(()); + return validate_spv_proof(coin, tx, expected_output.script_pubkey).await; } }; Box::new(fut.boxed().compat()) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index b881cebc0b..e3b40e862f 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -7,7 +7,9 @@ use crate::utxo::utxo_standard::{utxo_standard_coin_with_priv_key, UtxoStandardC #[cfg(not(target_arch = "wasm32"))] use crate::WithdrawFee; use crate::{CoinBalance, StakingInfosDetails, SwapOps, TradePreimageValue, TxFeeDetails}; use bigdecimal::{BigDecimal, Signed}; -use chain::OutPoint; +use bitcoin_spv::std_types::BitcoinHeader; +use bitcoin_spv::types::{Hash256Digest, RawHeader, SPVError}; +use chain::BlockHeader; use common::mm_ctx::MmCtxBuilder; use common::privkey::key_pair_from_seed; use common::{block_on, now_ms, OrdRange, DEX_FEE_ADDR_RAW_PUBKEY}; @@ -925,6 +927,34 @@ fn test_spv_fraud_proof() { println!("yay"); }*/ +#[test] +fn test_spv_proof_block_header() { + let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); + let res_block_header = block_on(client.blockchain_block_header(958318).compat()).unwrap(); + let block_header: BlockHeader = deserialize(res_block_header.0.as_slice()).unwrap(); + let tx_id: H256 = "7e9797a05abafbc1542449766ef9a41838ebbf6d24cd3223d361aa07c51981df".into(); + let height = 958318; + let merkle_root: H256 = "41f138275d13690e3c5d735e2f88eb6f1aaade1207eb09fa27a65b40711f3ae0".into(); + let merkle_branch = block_on(client.blockchain_transaction_get_merkle(tx_id.into(), height).compat()).unwrap(); + let mut vec: Vec = vec![]; + for merkle_node in merkle_branch.merkle { + vec.append(&mut merkle_node.0.as_slice().to_vec()); + } + let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice()).unwrap(); + println!("{}", nodes.len()); + println!("{}", merkle_branch.pos); + println!("{:?}", nodes); + println!( + "{}", + bitcoin_spv::validatespv::prove( + tx_id.take().into(), + merkle_root.take().into(), + &nodes, + merkle_branch.pos as u64, + ) + ); +} + #[test] fn test_spv_proof() { ElectrumClient::get_transaction_bytes.mock_safe(move |_, _| { @@ -946,9 +976,13 @@ fn test_spv_proof() { .unwrap(); let expected: BytesJson = hex::decode("0400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a473044022064318d9178eb8d5d098c1d2c9e90b7685521a9946fbbf6c4d482e6ce2e82959a0220425a675c1665f159109001176a4425e4a35a52c81b4a8a42e34ed18485e1d44d012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9147f4d6f4db8587bd878e05c83660f515800d75098870000000000000000166a14000000000000000000000000000000000000000012d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac7c10e561000000000000000000000000000000").unwrap().into(); assert_eq!(result, expected); + let tx_str = "0400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a473044022064318d9178eb8d5d098c1d2c9e90b7685521a9946fbbf6c4d482e6ce2e82959a0220425a675c1665f159109001176a4425e4a35a52c81b4a8a42e34ed18485e1d44d012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9147f4d6f4db8587bd878e05c83660f515800d75098870000000000000000166a14000000000000000000000000000000000000000012d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac7c10e561000000000000000000000000000000"; + let tx: UtxoTx = tx_str.into(); + let empty: Vec = vec![]; + let res = block_on(utxo_common::validate_spv_proof(coin.clone(), tx, empty.into())); - //! get_transaction_bytes return a crafted transaction - //validate_spv() + // empty pubkey -> empty history -> error + assert_eq!(res.is_err(), true); } #[test] From 3c122b18aa06bd6bd39f3bef68b65cf5805e0383 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 18 Jan 2022 09:35:52 +0100 Subject: [PATCH 04/74] feat(utxo): merkle tree verification now work; need full refactoring --- mm2src/coins/utxo/utxo_tests.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index e3b40e862f..ae4bb5da5e 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -930,15 +930,14 @@ fn test_spv_fraud_proof() { #[test] fn test_spv_proof_block_header() { let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); - let res_block_header = block_on(client.blockchain_block_header(958318).compat()).unwrap(); - let block_header: BlockHeader = deserialize(res_block_header.0.as_slice()).unwrap(); let tx_id: H256 = "7e9797a05abafbc1542449766ef9a41838ebbf6d24cd3223d361aa07c51981df".into(); let height = 958318; let merkle_root: H256 = "41f138275d13690e3c5d735e2f88eb6f1aaade1207eb09fa27a65b40711f3ae0".into(); let merkle_branch = block_on(client.blockchain_transaction_get_merkle(tx_id.into(), height).compat()).unwrap(); let mut vec: Vec = vec![]; for merkle_node in merkle_branch.merkle { - vec.append(&mut merkle_node.0.as_slice().to_vec()); + println!("{:?}", merkle_node.reversed()); + vec.append(&mut merkle_node.reversed().0.as_slice().to_vec()); } let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice()).unwrap(); println!("{}", nodes.len()); @@ -947,8 +946,8 @@ fn test_spv_proof_block_header() { println!( "{}", bitcoin_spv::validatespv::prove( - tx_id.take().into(), - merkle_root.take().into(), + tx_id.reversed().take().into(), + merkle_root.reversed().take().into(), &nodes, merkle_branch.pos as u64, ) From 971503acec59417897586b179af35084f7b1d46c Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Thu, 20 Jan 2022 07:46:04 +0100 Subject: [PATCH 05/74] feat(spv): start spv validation module --- Cargo.lock | 9 ++++ Cargo.toml | 1 + mm2src/mm2_bitcoin/spv_validation/Cargo.toml | 8 +++ .../spv_validation/src/helpers_validation.rs | 50 +++++++++++++++++++ mm2src/mm2_bitcoin/spv_validation/src/lib.rs | 8 +++ .../mm2_bitcoin/spv_validation/src/types.rs | 5 ++ 6 files changed, 81 insertions(+) create mode 100644 mm2src/mm2_bitcoin/spv_validation/Cargo.toml create mode 100644 mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs create mode 100644 mm2src/mm2_bitcoin/spv_validation/src/lib.rs create mode 100644 mm2src/mm2_bitcoin/spv_validation/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index e352851d7d..e369ab4229 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3359,6 +3359,7 @@ dependencies = [ "serialization_derive", "sp-runtime-interface", "sp-trie", + "spv_validation", "testcontainers", "tokio", "trie-db", @@ -5333,6 +5334,14 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spv_validation" +version = "0.1.0" +dependencies = [ + "bitcoin-spv", + "primitives", +] + [[package]] name = "sql-builder" version = "3.1.1" diff --git a/Cargo.toml b/Cargo.toml index ea51944afd..cc03b37d99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ ser_error = { path = "mm2src/derives/ser_error" } ser_error_derive = { path = "mm2src/derives/ser_error_derive" } serialization = { path = "mm2src/mm2_bitcoin/serialization" } serialization_derive = { path = "mm2src/mm2_bitcoin/serialization_derive" } +spv_validation = { path = "mm2src/mm2_bitcoin/spv_validation" } sp-runtime-interface = { version = "3.0.0", default-features = false, features = ["disable_target_static_assertions"] } sp-trie = { version = "3.0", default-features = false } diff --git a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml new file mode 100644 index 0000000000..6f56afdb65 --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "spv_validation" +version = "0.1.0" +authors = ["Roman Sztergbaum "] + +[dependencies] +bitcoin-spv = "5.0.0" +primitives = { path = "../primitives" } \ No newline at end of file diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs new file mode 100644 index 0000000000..a5b6db2274 --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -0,0 +1,50 @@ +use bitcoin_spv::validatespv::prove; +use primitives::hash::H256; +use types::SPVError; + +/// Evaluates a Bitcoin merkle inclusion proof. +/// Note that `index` is not a reliable indicator of location within a block. +/// +/// # Arguments +/// +/// * `txid` - The txid (LE) +/// * `merkle_root` - The merkle root (as in the block header) (LE) +/// * `intermediate_nodes` - The proof's intermediate nodes (digests between leaf and root) (LE) +/// * `index` - The leaf's index in the tree (0-indexed) +/// +/// # Notes +/// Wrapper around `bitcoin_spv::validatespv::prove` +pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec, index: u64) -> Result<(), SPVError> { + if txid == merkle_root && index == 0 && intermediate_nodes.is_empty() { + return Ok(()); + } + let mut vec: Vec = vec![]; + for merkle_node in intermediate_nodes { + vec.append(&mut merkle_node.as_slice().to_vec()); + } + let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice()).unwrap(); + if !prove(txid.take().into(), merkle_root.take().into(), &nodes, index) { + return Err(SPVError::BadMerkleProof); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_merkle_prove_inclusion() { + // https://rick.explorer.dexstats.info/tx/7e9797a05abafbc1542449766ef9a41838ebbf6d24cd3223d361aa07c51981df + let tx_id: H256 = H256::from_reversed_str("7e9797a05abafbc1542449766ef9a41838ebbf6d24cd3223d361aa07c51981df"); + let merkle_pos = 1; + let merkle_root: H256 = + H256::from_reversed_str("41f138275d13690e3c5d735e2f88eb6f1aaade1207eb09fa27a65b40711f3ae0").into(); + let merkle_nodes: Vec = vec![ + H256::from_reversed_str("73dfb53e6f49854b09d98500d4899d5c4e703c4fa3a2ddadc2cd7f12b72d4182"), + H256::from_reversed_str("4274d707b2308d39a04f2940024d382fa80d994152a50d4258f5a7feead2a563"), + ]; + let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); + assert_eq!(result.is_err(), false); + } +} diff --git a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs new file mode 100644 index 0000000000..d1c48b9cbd --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs @@ -0,0 +1,8 @@ +extern crate bitcoin_spv; +extern crate primitives; + +/// `types` exposes simple types for on-chain evaluation of SPV proofs +pub mod types; + +/// `helpers_validation` Override function modules from bitcoin_spv and adapt for our mm2_bitcoin library +pub mod helpers_validation; diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs new file mode 100644 index 0000000000..9ec0961f80 --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -0,0 +1,5 @@ +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum SPVError { + BadMerkleProof, + UnknownError(String), +} From 6570e074db5be931e11fb7d3ce52c19055f91996 Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Thu, 20 Jan 2022 07:56:49 +0100 Subject: [PATCH 06/74] feat(spv): add spv proof --- Cargo.lock | 1 + mm2src/mm2_bitcoin/spv_validation/Cargo.toml | 1 + mm2src/mm2_bitcoin/spv_validation/src/lib.rs | 4 ++ .../spv_validation/src/spv_proof.rs | 40 +++++++++++++++++++ 4 files changed, 46 insertions(+) create mode 100644 mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs diff --git a/Cargo.lock b/Cargo.lock index e369ab4229..fce7584868 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5339,6 +5339,7 @@ name = "spv_validation" version = "0.1.0" dependencies = [ "bitcoin-spv", + "chain", "primitives", ] diff --git a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml index 6f56afdb65..bf799a9e81 100644 --- a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml +++ b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml @@ -5,4 +5,5 @@ authors = ["Roman Sztergbaum "] [dependencies] bitcoin-spv = "5.0.0" +chain = {path = "../chain"} primitives = { path = "../primitives" } \ No newline at end of file diff --git a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs index d1c48b9cbd..ddb9c11b02 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs @@ -1,4 +1,5 @@ extern crate bitcoin_spv; +extern crate chain; extern crate primitives; /// `types` exposes simple types for on-chain evaluation of SPV proofs @@ -6,3 +7,6 @@ pub mod types; /// `helpers_validation` Override function modules from bitcoin_spv and adapt for our mm2_bitcoin library pub mod helpers_validation; + +/// `spv_proof` Contains spv proof validation logic and data structure +pub mod spv_proof; diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs new file mode 100644 index 0000000000..403ff3d3ad --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -0,0 +1,40 @@ +use chain::BlockHeader; +use helpers_validation::merkle_prove; +use primitives::hash::H256; +use types::SPVError; + +#[derive(PartialEq, Eq, Clone, Serialize, Deserialize)] +pub struct SPVProof { + /// The tx id + pub tx_id: H256, + /// The transaction index in the merkle tree + pub index: u64, + /// The confirming UTXO header + pub confirming_header: BlockHeader, + /// The intermediate nodes (digests between leaf and root) + pub intermediate_nodes: Vec, +} + +/// Checks validity of an entire SPV Proof +/// +/// # Arguments +/// +/// * `self` - The SPV Proof +/// +/// # Errors +/// +/// * Errors if any of the SPV Proof elements are invalid. +/// +/// # Notes +/// Re-write with our own types based on `bitcoin_spv::std_types::SPVProof::validate` +/// Support only merkle proof inclusion for now +impl SPVProof { + pub fn validate(&self) -> Result<(), SPVError> { + merkle_prove( + self.tx_id, + self.confirming_header.merkle_root_hash, + self.intermediate_nodes.clone(), + self.index, + ) + } +} From da81a73ff69a1587c305fab74c1bb42c8932ecbc Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Tue, 25 Jan 2022 07:22:29 +0100 Subject: [PATCH 07/74] feat(spv): add merkle proof unit test for a single element --- .../spv_validation/src/helpers_validation.rs | 16 ++++++++++++++++ .../mm2_bitcoin/spv_validation/src/spv_proof.rs | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index a5b6db2274..248a744d76 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -36,6 +36,7 @@ mod tests { #[test] fn test_merkle_prove_inclusion() { // https://rick.explorer.dexstats.info/tx/7e9797a05abafbc1542449766ef9a41838ebbf6d24cd3223d361aa07c51981df + // merkle intermediate nodes 2 element let tx_id: H256 = H256::from_reversed_str("7e9797a05abafbc1542449766ef9a41838ebbf6d24cd3223d361aa07c51981df"); let merkle_pos = 1; let merkle_root: H256 = @@ -47,4 +48,19 @@ mod tests { let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); assert_eq!(result.is_err(), false); } + + #[test] + fn test_merkle_prove_inclusion_single_element() { + // https://www.blockchain.com/btc/tx/c06fbab289f723c6261d3030ddb6be121f7d2508d77862bb1e484f5cd7f92b25 + // merkle intermediate nodes single element + let tx_id: H256 = H256::from_reversed_str("c06fbab289f723c6261d3030ddb6be121f7d2508d77862bb1e484f5cd7f92b25"); + let merkle_pos = 0; + let merkle_root: H256 = + H256::from_reversed_str("8fb300e3fdb6f30a4c67233b997f99fdd518b968b9a3fd65857bfe78b2600719").into(); + let merkle_nodes: Vec = vec![H256::from_reversed_str( + "5a4ebf66822b0b2d56bd9dc64ece0bc38ee7844a23ff1d7320a88c5fdb2ad3e2", + )]; + let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); + assert_eq!(result.is_err(), false); + } } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 403ff3d3ad..5e6e506da9 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -3,7 +3,7 @@ use helpers_validation::merkle_prove; use primitives::hash::H256; use types::SPVError; -#[derive(PartialEq, Eq, Clone, Serialize, Deserialize)] +#[derive(PartialEq, Clone)] pub struct SPVProof { /// The tx id pub tx_id: H256, From c888b42bd87f11fbbb04a5fcfe2cad4a57227b28 Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Tue, 25 Jan 2022 07:33:14 +0100 Subject: [PATCH 08/74] feat(spv): add complex merkle proof inclusion unit test --- .../spv_validation/src/helpers_validation.rs | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index 248a744d76..d904456f65 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -1,4 +1,4 @@ -use bitcoin_spv::validatespv::prove; +use bitcoin_spv::btcspv::verify_hash256_merkle; use primitives::hash::H256; use types::SPVError; @@ -23,7 +23,7 @@ pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec vec.append(&mut merkle_node.as_slice().to_vec()); } let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice()).unwrap(); - if !prove(txid.take().into(), merkle_root.take().into(), &nodes, index) { + if !verify_hash256_merkle(txid.take().into(), merkle_root.take().into(), &nodes, index) { return Err(SPVError::BadMerkleProof); } Ok(()) @@ -63,4 +63,30 @@ mod tests { let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); assert_eq!(result.is_err(), false); } + + #[test] + fn test_merkle_prove_inclusion_complex() { + // https://www.blockchain.com/btc/tx/b36bced99cc459506ad2b3af6990920b12f6dc84f9c7ed0dd2c3703f94a4b692 + // merkle intermediate nodes complex merkle proof inclusion + let tx_id: H256 = H256::from_reversed_str("b36bced99cc459506ad2b3af6990920b12f6dc84f9c7ed0dd2c3703f94a4b692"); + let merkle_pos = 680; + let merkle_root: H256 = + H256::from_reversed_str("def7a26d91789069dad448cb4b68658b7ba419f9fbd28dce7fe32ed0010e55df").into(); + let merkle_nodes: Vec = vec![ + H256::from_reversed_str("39141331f2b7133e72913460384927b421ffdef3e24b88521e7ac54d30019409"), + H256::from_reversed_str("39aeb77571ee0b0cf9feb7e121938b862f3994ff1254b34559378f6f2ed8b1fb"), + H256::from_reversed_str("5815f83f4eb2423c708127ea1f47feeabcf005d4aed18701d9692925f152d0b4"), + H256::from_reversed_str("efbb90aae6875af1b05a17e53fabe79ca1655329d6e107269a190739bf9d9038"), + H256::from_reversed_str("20eb7431ae5a185e89bd2ad89956fc660392ee9d231df58600ac675734013e82"), + H256::from_reversed_str("1f1dd980e6196ec4de9037941076a6030debe466dfc177e54447171b64ea99e5"), + H256::from_reversed_str("bbc4264359bec656298e31443034fc3ff9877752b765b9665b4da1eb8a32d1ff"), + H256::from_reversed_str("71788bf5224f228f390243a2664d41d96bae97ae1e4cfbc39095448e4cd1addd"), + H256::from_reversed_str("1b24a907c86e59eb698afeb4303c00fe3ecf8425270134ed3d0e62c6991621f2"), + H256::from_reversed_str("7776b46bb148c573d5eabe1436a428f3dae484557fea6efef1da901009ca5f8f"), + H256::from_reversed_str("623a90d6122a233b265aab497b13bb64b5d354d2e2112c3f554e51bfa4e6bbd3"), + H256::from_reversed_str("3104295d99163e16405b80321238a97d02e2448bb634017e2e027281cc4af9e8"), + ]; + let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); + assert_eq!(result.is_err(), false); + } } From a97f4d1dba64ef007044c7cfe83d40f539dad741 Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Tue, 25 Jan 2022 09:40:10 +0100 Subject: [PATCH 09/74] feat(spv): fix some cargo warnings --- Cargo.lock | 2 +- mm2src/coins/Cargo.toml | 2 +- mm2src/coins/utxo/utxo_common.rs | 56 +++++++------------ mm2src/coins/utxo/utxo_tests.rs | 52 ----------------- .../mm2_bitcoin/spv_validation/src/types.rs | 3 + 5 files changed, 26 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fce7584868..2853115c56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -819,7 +819,6 @@ dependencies = [ "bigdecimal", "bip32", "bitcoin", - "bitcoin-spv", "bitcoin_hashes", "bitcrypto", "byteorder 1.4.3", @@ -876,6 +875,7 @@ dependencies = [ "serialization_derive", "sha2 0.8.2", "sha3", + "spv_validation", "tokio", "tokio-rustls", "utxo_signer", diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index 2274cc2c5f..a4a259fca2 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -19,7 +19,6 @@ bigdecimal = { version = "0.1.0", features = ["serde"] } bip32 = { version = "0.2.2", default-features = false, features = ["alloc", "secp256k1-ffi"] } bitcoin = "0.27.1" bitcoin_hashes = "0.10.0" -bitcoin-spv = "5.0.0" bitcrypto = { path = "../mm2_bitcoin/crypto" } byteorder = "1.3" bytes = "0.4" @@ -67,6 +66,7 @@ serde_derive = "1.0" serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] } serialization = { path = "../mm2_bitcoin/serialization" } serialization_derive = { path = "../mm2_bitcoin/serialization_derive" } +spv_validation = { path = "../mm2_bitcoin/spv_validation" } sha2 = "0.8" sha3 = "0.8" utxo_signer = { path = "utxo_signer" } diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 945f3b90a4..a7bdd231fd 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -5,8 +5,6 @@ use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentI use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, TradePreimageValue, TxFeeDetails, ValidateAddressResult, WithdrawResult}; use bigdecimal::{BigDecimal, Zero}; -use bitcoin_spv::std_types::{BitcoinHeader, SPVProof}; -use bitcoin_spv::types::SPVError; pub use bitcrypto::{dhash160, sha256, ChecksumType}; use chain::constants::SEQUENCE_FINAL; use chain::{BlockHeader, OutPoint, TransactionOutput}; @@ -37,6 +35,8 @@ use utxo_signer::with_key_pair::p2sh_spend; use utxo_signer::UtxoSignerOps; pub use chain::Transaction as UtxoTx; +use spv_validation::spv_proof::SPVProof; +use spv_validation::types::SPVError; pub const DEFAULT_FEE_VOUT: usize = 0; pub const DEFAULT_SWAP_TX_SPEND_SIZE: u64 = 305; @@ -2664,14 +2664,13 @@ pub fn address_from_pubkey( } } -pub async fn validate_spv_proof(coin: T, tx: UtxoTx, script_pubkey: Bytes) -> Result<(), String> +pub async fn validate_spv_proof(coin: T, tx: UtxoTx, script_pubkey: Bytes) -> Result<(), MmError> where T: AsRef + Send + Sync + 'static, { - // todo: refactor to real error type let script_pubkey_str = hex::encode(&script_pubkey); let client = match &coin.as_ref().rpc_client { - UtxoRpcClientEnum::Native(_) => return ERR!("Native not supported for spv proof."), + UtxoRpcClientEnum::Native(_) => return MmError::err(SPVError::NonSpvClient), UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, }; let history = client @@ -2680,7 +2679,7 @@ where .await .unwrap_or_default(); if history.is_empty() { - return ERR!("Unable to get scripthash history - invalid transaction"); + return MmError::err(SPVError::TxHistoryNotAvailable); } let mut height: u64 = 0; for item in history { @@ -2691,42 +2690,26 @@ where } } if height == 0 { - return ERR!("Unable to get a proper transaction height - invalid transaction"); + return MmError::err(SPVError::TxHeightNotAvailable); } - let block_header = client.blockchain_block_header(height).compat().await; - - match block_header { - Ok(header_bytes) => { - // todo: deserialize into block_header and convert it to BitcoinHeader for the spv proof library - }, - Err(err) => return ERR!("Unable to get a block header - invalid transaction"), - }; - - // todo: implement get merkle tree tsc - - // todo: fill the SPVProof data structure correctly + let block_header = client + .blockchain_block_header(height) + .compat() + .await + .map_to_mm(|e| SPVError::UnknownError(e.to_string()))?; + let header: BlockHeader = + deserialize(block_header.0.as_slice()).map_to_mm(|e| SPVError::UnknownError(format!("{:?}", e)))?; let proof = SPVProof { - version: vec![], - vin: vec![], - vout: vec![], - locktime: vec![], tx_id: Default::default(), index: 0, - confirming_header: BitcoinHeader { - hash: Default::default(), - raw: Default::default(), - height: 0, - prevhash: Default::default(), - merkle_root: Default::default(), - }, + confirming_header: header, intermediate_nodes: vec![], }; match proof.validate() { - Ok(_) => {}, - Err(err) => return ERR!("{:?}", err), - }; - Ok(()) + Ok(_) => Ok(()), + Err(err) => MmError::err(err), + } } #[allow(clippy::too_many_arguments)] @@ -2794,7 +2777,10 @@ where expected_output ); } - return validate_spv_proof(coin, tx, expected_output.script_pubkey).await; + return match validate_spv_proof(coin, tx, expected_output.script_pubkey).await { + Ok(_) => Ok(()), + Err(err) => ERR!("{:?}", err), + }; } }; Box::new(fut.boxed().compat()) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index ae4bb5da5e..f5efb836db 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -7,9 +7,6 @@ use crate::utxo::utxo_standard::{utxo_standard_coin_with_priv_key, UtxoStandardC #[cfg(not(target_arch = "wasm32"))] use crate::WithdrawFee; use crate::{CoinBalance, StakingInfosDetails, SwapOps, TradePreimageValue, TxFeeDetails}; use bigdecimal::{BigDecimal, Signed}; -use bitcoin_spv::std_types::BitcoinHeader; -use bitcoin_spv::types::{Hash256Digest, RawHeader, SPVError}; -use chain::BlockHeader; use common::mm_ctx::MmCtxBuilder; use common::privkey::key_pair_from_seed; use common::{block_on, now_ms, OrdRange, DEX_FEE_ADDR_RAW_PUBKEY}; @@ -905,55 +902,6 @@ fn test_utxo_lock() { } } -/*#[test] -fn test_spv_fraud_proof() { - let secret = [0; 20]; - let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); - let coin = utxo_coin_for_test( - client.into(), - Some("spice describe gravity federal blast come thank unfair canal monkey style afraid"), - false, - ); - let CoinBalance { spendable, unspendable } = coin.my_balance().wait().unwrap(); - println!("{} {}", spendable, unspendable); - let timeout = (now_ms() / 1000) + 240; // timeout if test takes more than 240 seconds to run - let my_public_key = coin.my_public_key().unwrap(); - let time_lock = (now_ms() / 1000) as u32 - 3600; - - let tx = coin - .send_maker_payment(time_lock, my_public_key, &[0; 20], 1u64.into(), &None) - .wait() - .unwrap(); - println!("yay"); -}*/ - -#[test] -fn test_spv_proof_block_header() { - let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); - let tx_id: H256 = "7e9797a05abafbc1542449766ef9a41838ebbf6d24cd3223d361aa07c51981df".into(); - let height = 958318; - let merkle_root: H256 = "41f138275d13690e3c5d735e2f88eb6f1aaade1207eb09fa27a65b40711f3ae0".into(); - let merkle_branch = block_on(client.blockchain_transaction_get_merkle(tx_id.into(), height).compat()).unwrap(); - let mut vec: Vec = vec![]; - for merkle_node in merkle_branch.merkle { - println!("{:?}", merkle_node.reversed()); - vec.append(&mut merkle_node.reversed().0.as_slice().to_vec()); - } - let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice()).unwrap(); - println!("{}", nodes.len()); - println!("{}", merkle_branch.pos); - println!("{:?}", nodes); - println!( - "{}", - bitcoin_spv::validatespv::prove( - tx_id.reversed().take().into(), - merkle_root.reversed().take().into(), - &nodes, - merkle_branch.pos as u64, - ) - ); -} - #[test] fn test_spv_proof() { ElectrumClient::get_transaction_bytes.mock_safe(move |_, _| { diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index 9ec0961f80..02190c1775 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -1,5 +1,8 @@ #[derive(Debug, PartialEq, Eq, Clone)] pub enum SPVError { + NonSpvClient, + TxHistoryNotAvailable, + TxHeightNotAvailable, BadMerkleProof, UnknownError(String), } From 7f09afaf6a07988512e01061c7b3ce8806a4e6a3 Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Mon, 7 Feb 2022 07:37:53 +0100 Subject: [PATCH 10/74] feat(spv): simplify error check for validate_spv_proof - native client should not return error on validate_spv_proof since there is no verification --- mm2src/coins/utxo/utxo_common.rs | 9 ++++----- mm2src/mm2_bitcoin/spv_validation/src/types.rs | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index a7bdd231fd..9871dc337e 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2670,7 +2670,7 @@ where { let script_pubkey_str = hex::encode(&script_pubkey); let client = match &coin.as_ref().rpc_client { - UtxoRpcClientEnum::Native(_) => return MmError::err(SPVError::NonSpvClient), + UtxoRpcClientEnum::Native(_) => return Ok(()), UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, }; let history = client @@ -2777,10 +2777,9 @@ where expected_output ); } - return match validate_spv_proof(coin, tx, expected_output.script_pubkey).await { - Ok(_) => Ok(()), - Err(err) => ERR!("{:?}", err), - }; + return validate_spv_proof(coin, tx, expected_output.script_pubkey) + .await + .map_err(|e| format!("{:?}", e)); } }; Box::new(fut.boxed().compat()) diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index 02190c1775..5ee6541e4d 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -1,6 +1,5 @@ #[derive(Debug, PartialEq, Eq, Clone)] pub enum SPVError { - NonSpvClient, TxHistoryNotAvailable, TxHeightNotAvailable, BadMerkleProof, From b43aed68001e79c3beffa57f5a0797b9b3b144d9 Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Mon, 7 Feb 2022 08:46:10 +0100 Subject: [PATCH 11/74] feat(spv): complete the first step of the spv proof validation --- mm2src/coins/utxo/utxo_common.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 9871dc337e..6751fe3a7b 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2700,11 +2700,22 @@ where .map_to_mm(|e| SPVError::UnknownError(e.to_string()))?; let header: BlockHeader = deserialize(block_header.0.as_slice()).map_to_mm(|e| SPVError::UnknownError(format!("{:?}", e)))?; + + let merkle_branch = client + .blockchain_transaction_get_merkle(tx.hash().into(), height) + .compat() + .await + .map_to_mm(|e| SPVError::UnknownError(e.to_string()))?; + let intermediate_nodes: Vec = merkle_branch + .merkle + .into_iter() + .map(|hash| hash.reversed().into()) + .collect(); let proof = SPVProof { - tx_id: Default::default(), - index: 0, + tx_id: tx.hash(), + index: merkle_branch.pos as u64, confirming_header: header, - intermediate_nodes: vec![], + intermediate_nodes, }; match proof.validate() { Ok(_) => Ok(()), From 826507347264c0697ab814aa205f00fe55e6fa86 Mon Sep 17 00:00:00 2001 From: Sztergbaum Date: Mon, 7 Feb 2022 10:35:59 +0100 Subject: [PATCH 12/74] feat(spv): complete the unit test for spv proof validation in utxo module --- mm2src/coins/utxo/utxo_common.rs | 18 ++++++++++-------- mm2src/coins/utxo/utxo_tests.rs | 24 +++++------------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 6751fe3a7b..9c8cfe803e 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2664,15 +2664,20 @@ pub fn address_from_pubkey( } } -pub async fn validate_spv_proof(coin: T, tx: UtxoTx, script_pubkey: Bytes) -> Result<(), MmError> +pub async fn validate_spv_proof(coin: T, tx: UtxoTx) -> Result<(), MmError> where T: AsRef + Send + Sync + 'static, { - let script_pubkey_str = hex::encode(&script_pubkey); let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => return Ok(()), UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, }; + if tx.outputs.is_empty() { + return MmError::err(SPVError::TxHeightNotAvailable); + } + // note: all the outputs belong to the same tx, which is validated as the same height + // so accessing the history of the first element should be enough. + let script_pubkey_str = hex::encode(electrum_script_hash(&tx.outputs[0].script_pubkey)); let history = client .scripthash_get_history(script_pubkey_str.as_str()) .compat() @@ -2683,8 +2688,7 @@ where } let mut height: u64 = 0; for item in history { - if item.tx_hash == H256Json(*tx.hash()) && item.height > 0 { - println!("height: {}", item.height); + if item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0 { height = item.height as u64; break; } @@ -2702,7 +2706,7 @@ where deserialize(block_header.0.as_slice()).map_to_mm(|e| SPVError::UnknownError(format!("{:?}", e)))?; let merkle_branch = client - .blockchain_transaction_get_merkle(tx.hash().into(), height) + .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height) .compat() .await .map_to_mm(|e| SPVError::UnknownError(e.to_string()))?; @@ -2788,9 +2792,7 @@ where expected_output ); } - return validate_spv_proof(coin, tx, expected_output.script_pubkey) - .await - .map_err(|e| format!("{:?}", e)); + return validate_spv_proof(coin, tx).await.map_err(|e| format!("{:?}", e)); } }; Box::new(fut.boxed().compat()) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index f5efb836db..a64e9f1516 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -904,32 +904,18 @@ fn test_utxo_lock() { #[test] fn test_spv_proof() { - ElectrumClient::get_transaction_bytes.mock_safe(move |_, _| { - let bytes: BytesJson = hex::decode("0400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a473044022064318d9178eb8d5d098c1d2c9e90b7685521a9946fbbf6c4d482e6ce2e82959a0220425a675c1665f159109001176a4425e4a35a52c81b4a8a42e34ed18485e1d44d012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9147f4d6f4db8587bd878e05c83660f515800d75098870000000000000000166a14000000000000000000000000000000000000000012d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac7c10e561000000000000000000000000000000").unwrap().into(); - MockResult::Return(Box::new(futures01::future::ok(bytes))) - }); let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); let coin = utxo_coin_for_test( client.into(), Some("spice describe gravity federal blast come thank unfair canal monkey style afraid"), false, ); - let empty_hash = H256Json::default(); - let result = coin - .as_ref() - .rpc_client - .get_transaction_bytes(&empty_hash) - .wait() - .unwrap(); - let expected: BytesJson = hex::decode("0400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a473044022064318d9178eb8d5d098c1d2c9e90b7685521a9946fbbf6c4d482e6ce2e82959a0220425a675c1665f159109001176a4425e4a35a52c81b4a8a42e34ed18485e1d44d012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9147f4d6f4db8587bd878e05c83660f515800d75098870000000000000000166a14000000000000000000000000000000000000000012d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac7c10e561000000000000000000000000000000").unwrap().into(); - assert_eq!(result, expected); - let tx_str = "0400008085202f8901acbb02c7ef68832b3e768f8177da3dd09f0703675c4a6bbc9b0af4f4d86ff48c020000006a473044022064318d9178eb8d5d098c1d2c9e90b7685521a9946fbbf6c4d482e6ce2e82959a0220425a675c1665f159109001176a4425e4a35a52c81b4a8a42e34ed18485e1d44d012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9147f4d6f4db8587bd878e05c83660f515800d75098870000000000000000166a14000000000000000000000000000000000000000012d3d42c000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac7c10e561000000000000000000000000000000"; - let tx: UtxoTx = tx_str.into(); - let empty: Vec = vec![]; - let res = block_on(utxo_common::validate_spv_proof(coin.clone(), tx, empty.into())); - // empty pubkey -> empty history -> error - assert_eq!(res.is_err(), true); + // https://rick.explorer.dexstats.info/tx/78ea7839f6d1b0dafda2ba7e34c1d8218676a58bd1b33f03a5f76391f61b72b0 + let tx_str = "0400008085202f8902bf17bf7d1daace52e08f732a6b8771743ca4b1cb765a187e72fd091a0aabfd52000000006a47304402203eaaa3c4da101240f80f9c5e9de716a22b1ec6d66080de6a0cca32011cd77223022040d9082b6242d6acf9a1a8e658779e1c655d708379862f235e8ba7b8ca4e69c6012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffffff023ca13c0e9e085dd13f481f193e8a3e8fd609020936e98b5587342d994f4d020000006b483045022100c0ba56adb8de923975052312467347d83238bd8d480ce66e8b709a7997373994022048507bcac921fdb2302fa5224ce86e41b7efc1a2e20ae63aa738dfa99b7be826012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9141ee6d4c38a3c078eab87ad1a5e4b00f21259b10d870000000000000000166a1400000000000000000000000000000000000000001b94d736000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac2d08e35e000000000000000000000000000000"; + let tx: UtxoTx = tx_str.into(); + let res = block_on(utxo_common::validate_spv_proof(coin.clone(), tx)); + assert_eq!(res.is_err(), false); } #[test] From 9176fd14b5728a66bc9b516e1eed48c6018a6054 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 9 Feb 2022 15:03:43 +0100 Subject: [PATCH 13/74] feat(spv_validation): add vin and vout check for spv proof validation --- mm2src/coins/utxo/utxo_common.rs | 5 ++++- mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs | 11 +++++++++++ mm2src/mm2_bitcoin/spv_validation/src/types.rs | 4 ++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 9c8cfe803e..fc29d6858e 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -26,7 +26,8 @@ use rpc::v1::types::{Bytes as BytesJson, TransactionInputEnum, H256 as H256Json} use script::{Builder, Opcode, Script, ScriptAddress, TransactionInputSigner, UnsignedTransactionInput}; use secp256k1::{PublicKey, Signature}; use serde_json::{self as json}; -use serialization::{deserialize, serialize, serialize_with_flags, CoinVariant, SERIALIZE_TRANSACTION_WITNESS}; +use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, + SERIALIZE_TRANSACTION_WITNESS}; use std::cmp::Ordering; use std::collections::hash_map::{Entry, HashMap}; use std::str::FromStr; @@ -2717,6 +2718,8 @@ where .collect(); let proof = SPVProof { tx_id: tx.hash(), + vin: serialize_list(&tx.inputs).take(), + vout: serialize_list(&tx.outputs).take(), index: merkle_branch.pos as u64, confirming_header: header, intermediate_nodes, diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 5e6e506da9..1c2b6943f9 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -1,3 +1,4 @@ +use bitcoin_spv::btcspv::{validate_vin, validate_vout}; use chain::BlockHeader; use helpers_validation::merkle_prove; use primitives::hash::H256; @@ -7,6 +8,10 @@ use types::SPVError; pub struct SPVProof { /// The tx id pub tx_id: H256, + /// The vin serialized + pub vin: Vec, + /// The vout serialized + pub vout: Vec, /// The transaction index in the merkle tree pub index: u64, /// The confirming UTXO header @@ -30,6 +35,12 @@ pub struct SPVProof { /// Support only merkle proof inclusion for now impl SPVProof { pub fn validate(&self) -> Result<(), SPVError> { + if !validate_vin(self.vin.as_slice()) { + return Err(SPVError::InvalidVin); + } + if !validate_vout(self.vout.as_slice()) { + return Err(SPVError::InvalidVout); + } merkle_prove( self.tx_id, self.confirming_header.merkle_root_hash, diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index 5ee6541e4d..18f1ac026f 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -2,6 +2,10 @@ pub enum SPVError { TxHistoryNotAvailable, TxHeightNotAvailable, + /// A `vin` (transaction input vector) is malformatted. + InvalidVin, + /// A `vout` (transaction output vector) is malformatted. + InvalidVout, BadMerkleProof, UnknownError(String), } From e80eafe06454b3919b01f469246e841ac998229f Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 9 Feb 2022 16:02:54 +0100 Subject: [PATCH 14/74] feat(spv_validation): match exact error from the spv - will customize later --- mm2src/coins/utxo/utxo_common.rs | 13 ++-- .../spv_validation/src/helpers_validation.rs | 2 +- .../spv_validation/src/spv_proof.rs | 2 +- .../mm2_bitcoin/spv_validation/src/types.rs | 70 ++++++++++++++++++- 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index fc29d6858e..0e071394a6 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2674,7 +2674,7 @@ where UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, }; if tx.outputs.is_empty() { - return MmError::err(SPVError::TxHeightNotAvailable); + return MmError::err(SPVError::UnknownError); } // note: all the outputs belong to the same tx, which is validated as the same height // so accessing the history of the first element should be enough. @@ -2685,7 +2685,7 @@ where .await .unwrap_or_default(); if history.is_empty() { - return MmError::err(SPVError::TxHistoryNotAvailable); + return MmError::err(SPVError::UnknownError); } let mut height: u64 = 0; for item in history { @@ -2695,22 +2695,21 @@ where } } if height == 0 { - return MmError::err(SPVError::TxHeightNotAvailable); + return MmError::err(SPVError::UnknownError); } let block_header = client .blockchain_block_header(height) .compat() .await - .map_to_mm(|e| SPVError::UnknownError(e.to_string()))?; - let header: BlockHeader = - deserialize(block_header.0.as_slice()).map_to_mm(|e| SPVError::UnknownError(format!("{:?}", e)))?; + .map_to_mm(|_e| SPVError::UnknownError)?; + let header: BlockHeader = deserialize(block_header.0.as_slice()).map_to_mm(|_e| SPVError::UnknownError)?; let merkle_branch = client .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height) .compat() .await - .map_to_mm(|e| SPVError::UnknownError(e.to_string()))?; + .map_to_mm(|_e| SPVError::NetworkError)?; let intermediate_nodes: Vec = merkle_branch .merkle .into_iter() diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index d904456f65..de51c0e515 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -22,7 +22,7 @@ pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec for merkle_node in intermediate_nodes { vec.append(&mut merkle_node.as_slice().to_vec()); } - let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice()).unwrap(); + let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice())?; if !verify_hash256_merkle(txid.take().into(), merkle_root.take().into(), &nodes, index) { return Err(SPVError::BadMerkleProof); } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 1c2b6943f9..87400dd7fa 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -32,7 +32,7 @@ pub struct SPVProof { /// /// # Notes /// Re-write with our own types based on `bitcoin_spv::std_types::SPVProof::validate` -/// Support only merkle proof inclusion for now +/// Support only merkle proof inclusion,vin,vout for now impl SPVProof { pub fn validate(&self) -> Result<(), SPVError> { if !validate_vin(self.vin.as_slice()) { diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index 18f1ac026f..15bdf6b7f8 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -1,11 +1,75 @@ #[derive(Debug, PartialEq, Eq, Clone)] pub enum SPVError { - TxHistoryNotAvailable, - TxHeightNotAvailable, + /// Overran a checked read on a slice + ReadOverrun, + /// Attempted to parse a CompactInt without enough bytes + BadCompactInt, + /// Called `extract_op_return_data` on an output without an op_return. + MalformattedOpReturnOutput, + /// `extract_hash` identified a SH output prefix without a SH postfix. + MalformattedP2SHOutput, + /// `extract_hash` identified a PKH output prefix without a PKH postfix. + MalformattedP2PKHOutput, + /// `extract_hash` identified a Witness output with a bad length tag. + MalformattedWitnessOutput, + /// `extract_hash` could not identify the output type. + MalformattedOutput, + /// Header not exactly 80 bytes. + WrongLengthHeader, + /// Header chain changed difficulties unexpectedly + UnexpectedDifficultyChange, + /// Header does not meet its own difficulty target. + InsufficientWork, + /// Header in chain does not correctly reference parent header. + InvalidChain, + /// When validating a `BitcoinHeader`, the `hash` field is not the digest + /// of the raw header. + WrongDigest, + /// When validating a `BitcoinHeader`, the `merkle_root` field does not + /// match the root found in the raw header. + WrongMerkleRoot, + /// When validating a `BitcoinHeader`, the `prevhash` field does not + /// match the parent hash found in the raw header. + WrongPrevHash, /// A `vin` (transaction input vector) is malformatted. InvalidVin, /// A `vout` (transaction output vector) is malformatted. InvalidVout, + /// When validating an `SPVProof`, the `tx_id` field is not the digest + /// of the `version`, `vin`, `vout`, and `locktime`. + WrongTxID, + /// When validating an `SPVProof`, the `intermediate_nodes` is not a valid + /// merkle proof connecting the `tx_id_le` to the `confirming_header`. BadMerkleProof, - UnknownError(String), + /// TxOut's reported length does not match passed-in byte slice's length + OutputLengthMismatch, + /// Any other error + UnknownError, +} + +impl From for SPVError { + fn from(e: bitcoin_spv::types::SPVError) -> Self { + match e { + bitcoin_spv::types::SPVError::ReadOverrun => SPVError::ReadOverrun, + bitcoin_spv::types::SPVError::BadCompactInt => SPVError::BadCompactInt, + bitcoin_spv::types::SPVError::MalformattedOpReturnOutput => SPVError::MalformattedOpReturnOutput, + bitcoin_spv::types::SPVError::MalformattedP2SHOutput => SPVError::MalformattedP2SHOutput, + bitcoin_spv::types::SPVError::MalformattedP2PKHOutput => SPVError::MalformattedP2PKHOutput, + bitcoin_spv::types::SPVError::MalformattedWitnessOutput => SPVError::MalformattedWitnessOutput, + bitcoin_spv::types::SPVError::MalformattedOutput => SPVError::MalformattedOutput, + bitcoin_spv::types::SPVError::WrongLengthHeader => SPVError::WrongLengthHeader, + bitcoin_spv::types::SPVError::UnexpectedDifficultyChange => SPVError::UnexpectedDifficultyChange, + bitcoin_spv::types::SPVError::InsufficientWork => SPVError::InsufficientWork, + bitcoin_spv::types::SPVError::InvalidChain => SPVError::InvalidChain, + bitcoin_spv::types::SPVError::WrongDigest => SPVError::WrongDigest, + bitcoin_spv::types::SPVError::WrongMerkleRoot => SPVError::WrongMerkleRoot, + bitcoin_spv::types::SPVError::WrongPrevHash => SPVError::WrongPrevHash, + bitcoin_spv::types::SPVError::InvalidVin => SPVError::InvalidVin, + bitcoin_spv::types::SPVError::InvalidVout => SPVError::InvalidVout, + bitcoin_spv::types::SPVError::WrongTxID => SPVError::WrongTxID, + bitcoin_spv::types::SPVError::BadMerkleProof => SPVError::BadMerkleProof, + bitcoin_spv::types::SPVError::OutputLengthMismatch => SPVError::OutputLengthMismatch, + bitcoin_spv::types::SPVError::UnknownError => SPVError::UnknownError, + } + } } From db324f7680abfb464e59a101d20e42508d3f7042 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 16 Feb 2022 08:50:51 +0100 Subject: [PATCH 15/74] feat(utxo): start utxo block header storage + sync with dev --- mm2src/coins/utxo.rs | 1 + .../coins/utxo/utxo_block_header_storage.rs | 25 +++++++++++++++++++ mm2src/coins/utxo/utxo_common.rs | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 mm2src/coins/utxo/utxo_block_header_storage.rs diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index df7964e8ce..a19f730dce 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -29,6 +29,7 @@ mod bchd_pb; pub mod qtum; pub mod rpc_clients; pub mod slp; +pub mod utxo_block_header_storage; pub mod utxo_common; pub mod utxo_standard; pub mod utxo_withdraw; diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs new file mode 100644 index 0000000000..1963f7494c --- /dev/null +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -0,0 +1,25 @@ +use async_trait::async_trait; +use common::mm_ctx::MmArc; +use common::{mm_error::NotMmError, NotSame}; +use std::path::PathBuf; + +pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send {} + +#[async_trait] +pub trait BlockHeaderStorage: Send + Sync + 'static { + type Error: BlockHeaderStorageError; +} + +fn block_header_storage_dir(ctx: &MmArc, ticker: &str) -> PathBuf { ctx.dbdir().join("BLOCK_HEADERS").join(ticker) } + +mod tests { + use super::*; + use common::mm_ctx::MmCtxBuilder; + + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn test_block_header_storage_dir() { + let ctx = MmCtxBuilder::new().into_mm_arc(); + assert_eq!(block_header_storage_dir(&ctx, "BTC").as_os_str().is_empty(), false) + } +} diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 0e071394a6..b25ddc5d52 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2709,7 +2709,7 @@ where .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height) .compat() .await - .map_to_mm(|_e| SPVError::NetworkError)?; + .map_to_mm(|_e| SPVError::UnknownError)?; let intermediate_nodes: Vec = merkle_branch .merkle .into_iter() From 5fd7d6abd6e18d45b480b460ab240abbf1b740b9 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 16 Feb 2022 10:06:33 +0100 Subject: [PATCH 16/74] feat(utxo): continue block header storage interface --- .../coins/utxo/utxo_block_header_storage.rs | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 1963f7494c..12f419963a 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,6 +1,7 @@ +use crate::utxo::rpc_clients::ElectrumBlockHeader; use async_trait::async_trait; -use common::mm_ctx::MmArc; -use common::{mm_error::NotMmError, NotSame}; +use chain::BlockHeader; +use common::{mm_ctx::MmArc, mm_error::MmError, mm_error::NotMmError, NotSame}; use std::path::PathBuf; pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send {} @@ -8,12 +9,31 @@ pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send #[async_trait] pub trait BlockHeaderStorage: Send + Sync + 'static { type Error: BlockHeaderStorageError; + + /// Initializes collection/tables in storage for a specified coin + async fn init(&self, for_coin: &str) -> Result<(), MmError>; + + async fn is_initialized_for(&self, for_coin: &str) -> Result>; + + // Adds multiple block headers to the selected coin's header storage + // Should store it as `TICKER_HEIGHT=hex_string` + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: Vec, + ) -> Result<(), MmError>; + + /// Gets the block header by height from the selected coin's storage as BlockHeader + async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError>; + + /// Gets the block header by height from the selected coin's storage as hex + async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError>; } fn block_header_storage_dir(ctx: &MmArc, ticker: &str) -> PathBuf { ctx.dbdir().join("BLOCK_HEADERS").join(ticker) } mod tests { - use super::*; + use crate::utxo::utxo_block_header_storage::block_header_storage_dir; use common::mm_ctx::MmCtxBuilder; #[test] From 773b27b0dcac1329d1285b87168e9711a173f901 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 16 Feb 2022 10:40:59 +0100 Subject: [PATCH 17/74] feat(utxo): revert sled - will switch to sqllite --- Cargo.lock | 196 ++++++++++-------- .../coins/utxo/utxo_block_header_storage.rs | 17 +- 2 files changed, 106 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0556f06089..d35b2728df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -198,7 +198,7 @@ dependencies = [ "futures-io", "futures-timer", "kv-log-macro", - "log 0.4.11", + "log 0.4.14", "memchr", "num_cpus", "once_cell", @@ -274,7 +274,7 @@ dependencies = [ "libp2p-plaintext", "libp2p-swarm", "libp2p-yamux", - "log 0.4.11", + "log 0.4.14", "lru 0.4.3", "prost", "prost-build", @@ -800,15 +800,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "cloudabi" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" -dependencies = [ - "bitflags", -] - [[package]] name = "coins" version = "0.1.0" @@ -953,7 +944,7 @@ dependencies = [ "lazy_static", "libc", "lightning", - "log 0.4.11", + "log 0.4.14", "log4rs", "metrics", "metrics-core", @@ -962,7 +953,7 @@ dependencies = [ "num-bigint 0.2.6", "num-rational", "num-traits 0.2.12", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "parking_lot_core 0.6.2", "paste", "primitives", @@ -1060,11 +1051,11 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.2.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -1149,7 +1140,7 @@ dependencies = [ "crossterm_winapi", "libc", "mio", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "signal-hook", "signal-hook-mio", "winapi", @@ -1190,7 +1181,7 @@ dependencies = [ "http 0.2.1", "hw_common", "keys", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "primitives", "secp256k1", "ser_error", @@ -1255,6 +1246,16 @@ dependencies = [ "sct", ] +[[package]] +name = "ctor" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbaabec2c953050352311293be5c6aba8e141ba19d6811862b232d6fd020484" +dependencies = [ + "quote 1.0.7", + "syn 1.0.72", +] + [[package]] name = "ctr" version = "0.7.0" @@ -1308,7 +1309,7 @@ checksum = "72aa14c04dfae8dd7d8a2b1cb7ca2152618cd01336dbfe704b8dcbf8d41dbd69" name = "db_common" version = "0.1.0" dependencies = [ - "log 0.4.11", + "log 0.4.14", "rusqlite", "sql-builder", "uuid", @@ -1473,7 +1474,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime 1.3.0", - "log 0.4.11", + "log 0.4.14", "regex", "termcolor", ] @@ -2324,7 +2325,7 @@ dependencies = [ "ct-logs", "futures-util", "hyper", - "log 0.4.11", + "log 0.4.14", "rustls", "tokio", "tokio-rustls", @@ -2448,9 +2449,12 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.6" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] [[package]] name = "iovec" @@ -2585,7 +2589,7 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ff57d6d215f7ca7eb35a9a64d656ba4d9d2bef114d741dc08048e75e2f5d418" dependencies = [ - "log 0.4.11", + "log 0.4.14", ] [[package]] @@ -2609,9 +2613,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.97" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +checksum = "06e509672465a0504304aa87f9f176f2b2b716ed8fb105ebe5c02dc6dce96a94" [[package]] name = "libp2p" @@ -2635,7 +2639,7 @@ dependencies = [ "libp2p-wasm-ext", "libp2p-websocket", "multiaddr", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project 1.0.7", "smallvec 1.6.1", "wasm-timer", @@ -2655,11 +2659,11 @@ dependencies = [ "futures-timer", "lazy_static", "libsecp256k1", - "log 0.4.11", + "log 0.4.14", "multiaddr", "multihash", "multistream-select", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project 1.0.7", "prost", "prost-build", @@ -2681,7 +2685,7 @@ source = "git+https://github.com/libp2p/rust-libp2p.git#20183c1ea152f5bfe183543e dependencies = [ "futures 0.3.15", "libp2p-core", - "log 0.4.11", + "log 0.4.14", "smallvec 1.6.1", "trust-dns-resolver", ] @@ -2710,7 +2714,7 @@ dependencies = [ "futures 0.3.15", "libp2p-core", "libp2p-swarm", - "log 0.4.11", + "log 0.4.14", "prost", "prost-build", "rand 0.7.3", @@ -2726,9 +2730,9 @@ dependencies = [ "bytes 1.1.0", "futures 0.3.15", "libp2p-core", - "log 0.4.11", + "log 0.4.14", "nohash-hasher", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "smallvec 1.6.1", "unsigned-varint 0.7.0", @@ -2744,7 +2748,7 @@ dependencies = [ "futures 0.3.15", "lazy_static", "libp2p-core", - "log 0.4.11", + "log 0.4.14", "prost", "prost-build", "rand 0.8.4", @@ -2763,7 +2767,7 @@ dependencies = [ "futures 0.3.15", "libp2p-core", "libp2p-swarm", - "log 0.4.11", + "log 0.4.14", "rand 0.7.3", "void", "wasm-timer", @@ -2778,7 +2782,7 @@ dependencies = [ "bytes 1.1.0", "futures 0.3.15", "libp2p-core", - "log 0.4.11", + "log 0.4.14", "prost", "prost-build", "unsigned-varint 0.7.0", @@ -2795,7 +2799,7 @@ dependencies = [ "futures 0.3.15", "libp2p-core", "libp2p-swarm", - "log 0.4.11", + "log 0.4.14", "lru 0.6.0", "minicbor", "rand 0.7.3", @@ -2812,7 +2816,7 @@ dependencies = [ "either", "futures 0.3.15", "libp2p-core", - "log 0.4.11", + "log 0.4.14", "rand 0.7.3", "smallvec 1.6.1", "void", @@ -2839,7 +2843,7 @@ dependencies = [ "ipnet", "libc", "libp2p-core", - "log 0.4.11", + "log 0.4.14", "socket2 0.4.0", "tokio", ] @@ -2866,7 +2870,7 @@ dependencies = [ "futures 0.3.15", "futures-rustls", "libp2p-core", - "log 0.4.11", + "log 0.4.14", "quicksink", "rw-stream-sink", "soketto", @@ -2881,7 +2885,7 @@ source = "git+https://github.com/libp2p/rust-libp2p.git#20183c1ea152f5bfe183543e dependencies = [ "futures 0.3.15", "libp2p-core", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "thiserror", "yamux", ] @@ -3040,9 +3044,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" dependencies = [ "scopeguard 1.1.0", ] @@ -3053,17 +3057,18 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" dependencies = [ - "log 0.4.11", + "log 0.4.14", ] [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "serde", + "value-bag", ] [[package]] @@ -3085,9 +3090,9 @@ dependencies = [ "fnv", "humantime 2.1.0", "libc", - "log 0.4.11", + "log 0.4.14", "log-mdc", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "regex", "serde", "serde-value", @@ -3274,7 +3279,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" dependencies = [ "libc", - "log 0.4.11", + "log 0.4.14", "miow", "ntapi", "winapi", @@ -3339,7 +3344,7 @@ dependencies = [ "num-rational", "num-traits 0.2.12", "parity-util-mem", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "primitives", "rand 0.6.5", "rand 0.7.3", @@ -3389,7 +3394,7 @@ dependencies = [ "lazy_static", "libp2p", "libp2p-floodsub 0.22.0", - "log 0.4.11", + "log 0.4.14", "num-bigint 0.2.6", "num-rational", "rand 0.7.3", @@ -3484,7 +3489,7 @@ source = "git+https://github.com/libp2p/rust-libp2p.git#20183c1ea152f5bfe183543e dependencies = [ "bytes 1.1.0", "futures 0.3.15", - "log 0.4.11", + "log 0.4.14", "pin-project 1.0.7", "smallvec 1.6.1", "unsigned-varint 0.7.0", @@ -3706,7 +3711,7 @@ dependencies = [ "impl-trait-for-tuples", "lru 0.6.0", "parity-util-mem-derive", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "primitive-types 0.9.1", "smallvec 1.6.1", "winapi", @@ -3762,13 +3767,13 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api 0.4.4", - "parking_lot_core 0.8.0", + "lock_api 0.4.6", + "parking_lot_core 0.8.5", ] [[package]] @@ -3791,7 +3796,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" dependencies = [ "cfg-if 0.1.10", - "cloudabi 0.0.3", + "cloudabi", "libc", "redox_syscall 0.1.56", "rustc_version 0.2.3", @@ -3806,7 +3811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ "cfg-if 0.1.10", - "cloudabi 0.0.3", + "cloudabi", "libc", "redox_syscall 0.1.56", "smallvec 1.6.1", @@ -3815,15 +3820,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.0" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if 0.1.10", - "cloudabi 0.1.0", + "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.56", + "redox_syscall 0.2.8", "smallvec 1.6.1", "winapi", ] @@ -4074,7 +4078,7 @@ dependencies = [ "bytes 1.1.0", "heck", "itertools 0.10.1", - "log 0.4.11", + "log 0.4.14", "multimap", "petgraph", "prost", @@ -4154,7 +4158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" dependencies = [ "env_logger", - "log 0.4.11", + "log 0.4.14", "rand 0.7.3", "rand_core 0.5.1", ] @@ -4387,7 +4391,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" dependencies = [ - "cloudabi 0.0.3", + "cloudabi", "fuchsia-cprng", "libc", "rand_core 0.4.2", @@ -4617,7 +4621,7 @@ version = "0.1.0" dependencies = [ "chain", "keys", - "log 0.4.11", + "log 0.4.14", "primitives", "rustc-hex 2.1.0", "script", @@ -4725,7 +4729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.0", - "log 0.4.11", + "log 0.4.14", "ring", "sct", "webpki", @@ -4774,7 +4778,7 @@ dependencies = [ "blake2b_simd", "chain", "keys", - "log 0.4.11", + "log 0.4.14", "primitives", "serde", "serialization", @@ -5051,7 +5055,7 @@ dependencies = [ name = "shared_ref_counter" version = "0.1.0" dependencies = [ - "log 0.4.11", + "log 0.4.14", ] [[package]] @@ -5204,7 +5208,7 @@ dependencies = [ "flate2", "futures 0.3.15", "httparse", - "log 0.4.11", + "log 0.4.14", "rand 0.7.3", "sha-1", ] @@ -5218,7 +5222,7 @@ dependencies = [ "byteorder 1.4.3", "hash-db", "hash256-std-hasher", - "log 0.4.11", + "log 0.4.14", "num-traits 0.2.12", "parity-scale-codec 2.2.0", "parity-util-mem", @@ -5489,7 +5493,7 @@ name = "tc_cli_client" version = "0.2.0" source = "git+https://github.com/artemii235/testcontainers-rs.git#65e738093488f1b37185af0f1d3cf0ffd23e840d" dependencies = [ - "log 0.4.11", + "log 0.4.14", "serde", "serde_derive", "serde_json", @@ -5503,7 +5507,7 @@ source = "git+https://github.com/artemii235/testcontainers-rs.git#65e738093488f1 dependencies = [ "hex 0.3.2", "hmac 0.7.1", - "log 0.4.11", + "log 0.4.14", "rand 0.7.3", "sha2 0.8.2", "tc_core", @@ -5515,7 +5519,7 @@ version = "0.3.0" source = "git+https://github.com/artemii235/testcontainers-rs.git#65e738093488f1b37185af0f1d3cf0ffd23e840d" dependencies = [ "debug_stub_derive", - "log 0.4.11", + "log 0.4.14", ] [[package]] @@ -5523,7 +5527,7 @@ name = "tc_dynamodb_local" version = "0.2.0" source = "git+https://github.com/artemii235/testcontainers-rs.git#65e738093488f1b37185af0f1d3cf0ffd23e840d" dependencies = [ - "log 0.4.11", + "log 0.4.14", "tc_core", ] @@ -5548,7 +5552,7 @@ name = "tc_parity_parity" version = "0.5.0" source = "git+https://github.com/artemii235/testcontainers-rs.git#65e738093488f1b37185af0f1d3cf0ffd23e840d" dependencies = [ - "log 0.4.11", + "log 0.4.14", "tc_core", ] @@ -5557,7 +5561,7 @@ name = "tc_postgres" version = "0.2.0" source = "git+https://github.com/artemii235/testcontainers-rs.git#65e738093488f1b37185af0f1d3cf0ffd23e840d" dependencies = [ - "log 0.4.11", + "log 0.4.14", "tc_core", ] @@ -5803,7 +5807,7 @@ dependencies = [ "bytes 1.1.0", "futures-core", "futures-sink", - "log 0.4.11", + "log 0.4.14", "pin-project-lite 0.2.6", "tokio", ] @@ -5882,7 +5886,7 @@ checksum = "9eac131e334e81b6b3be07399482042838adcd7957aa0010231d0813e39e02fa" dependencies = [ "hash-db", "hashbrown 0.11.2", - "log 0.4.11", + "log 0.4.14", "smallvec 1.6.1", ] @@ -5911,7 +5915,7 @@ dependencies = [ "idna 0.2.0", "ipnet", "lazy_static", - "log 0.4.11", + "log 0.4.14", "rand 0.8.4", "smallvec 1.6.1", "thiserror", @@ -5930,9 +5934,9 @@ dependencies = [ "futures-util", "ipconfig", "lazy_static", - "log 0.4.11", + "log 0.4.14", "lru-cache", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "resolv-conf", "smallvec 1.6.1", "thiserror", @@ -6139,6 +6143,16 @@ dependencies = [ "serde", ] +[[package]] +name = "value-bag" +version = "1.0.0-alpha.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79923f7731dc61ebfba3633098bf3ac533bbd35ccd8c57e7088d9a5eebe0263f" +dependencies = [ + "ctor", + "version_check", +] + [[package]] name = "vcpkg" version = "0.2.10" @@ -6219,7 +6233,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.11", + "log 0.4.14", "try-lock", ] @@ -6255,7 +6269,7 @@ checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.11", + "log 0.4.14", "proc-macro2", "quote 1.0.7", "syn 1.0.72", @@ -6365,7 +6379,7 @@ dependencies = [ "ethereum-types 0.4.2", "futures 0.1.29", "jsonrpc-core", - "log 0.4.11", + "log 0.4.14", "parking_lot 0.7.1", "rustc-hex 1.0.0", "serde", @@ -6501,9 +6515,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d9028f208dd5e63c614be69f115c1b53cacc1111437d4c765185856666c107" dependencies = [ "futures 0.3.15", - "log 0.4.11", + "log 0.4.14", "nohash-hasher", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.8.4", "static_assertions", ] @@ -6566,7 +6580,7 @@ dependencies = [ "hex 0.4.3", "jubjub", "lazy_static", - "log 0.4.11", + "log 0.4.14", "rand 0.7.3", "rand_core 0.5.1", "ripemd160 0.9.1", diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 12f419963a..1685fe8af5 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,8 +1,7 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; use async_trait::async_trait; use chain::BlockHeader; -use common::{mm_ctx::MmArc, mm_error::MmError, mm_error::NotMmError, NotSame}; -use std::path::PathBuf; +use common::{mm_error::MmError, mm_error::NotMmError, NotSame}; pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send {} @@ -29,17 +28,3 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { /// Gets the block header by height from the selected coin's storage as hex async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError>; } - -fn block_header_storage_dir(ctx: &MmArc, ticker: &str) -> PathBuf { ctx.dbdir().join("BLOCK_HEADERS").join(ticker) } - -mod tests { - use crate::utxo::utxo_block_header_storage::block_header_storage_dir; - use common::mm_ctx::MmCtxBuilder; - - #[test] - #[cfg(not(target_arch = "wasm32"))] - fn test_block_header_storage_dir() { - let ctx = MmCtxBuilder::new().into_mm_arc(); - assert_eq!(block_header_storage_dir(&ctx, "BTC").as_os_str().is_empty(), false) - } -} From b3b8750d6c15b243c89434de681795b489ec40d1 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 16 Feb 2022 11:30:28 +0100 Subject: [PATCH 18/74] feat(utxo): start utxo sql block header storage trait implementation --- mm2src/coins/utxo.rs | 2 + .../utxo/utxo_sql_block_header_storage.rs | 143 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 mm2src/coins/utxo/utxo_sql_block_header_storage.rs diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index a19f730dce..8b0ed778e4 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -35,6 +35,8 @@ pub mod utxo_standard; pub mod utxo_withdraw; #[cfg(not(target_arch = "wasm32"))] pub mod tx_cache; +#[cfg(not(target_arch = "wasm32"))] +pub mod utxo_sql_block_header_storage; use async_trait::async_trait; use bigdecimal::BigDecimal; diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs new file mode 100644 index 0000000000..dbd155ab3b --- /dev/null +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -0,0 +1,143 @@ +use crate::utxo::rpc_clients::ElectrumBlockHeader; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderStorageError}; +use async_trait::async_trait; +use chain::BlockHeader; +use common::async_blocking; +use common::mm_error::MmError; +use db_common::sqlite::rusqlite::Error as SqlError; +use db_common::sqlite::rusqlite::{Connection, Row, ToSql, NO_PARAMS}; +use db_common::sqlite::validate_table_name; +use std::sync::{Arc, Mutex}; + +const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; + +fn block_headers_cache_table(ticker: &str) -> String { ticker.to_owned() + "_block_headers_cache" } +fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { + let table_name = block_headers_cache_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() + + &table_name + + " ( + block_height INTEGER NOT NULL UNIQUE, + hex TEXT NOT NULL + );"; + + Ok(sql) +} + +fn insert_block_header_in_cache_sql(for_coin: &str) -> Result> { + let table_name = block_headers_cache_table(for_coin); + validate_table_name(&table_name)?; + + // We can simply ignore the repetitive attempt to insert the same block_height + let sql = "INSERT OR IGNORE INTO ".to_owned() + &table_name + " (block_height, hex) VALUES (?1, ?2);"; + + Ok(sql) +} + +#[derive(Clone)] +pub struct SqliteBlockHeadersStorage(pub Arc>); + +fn query_single_row( + conn: &Connection, + query: &str, + params: P, + map_fn: F, +) -> Result, MmError> +where + P: IntoIterator, + P::Item: ToSql, + F: FnOnce(&Row<'_>) -> Result, +{ + let maybe_result = conn.query_row(query, params, map_fn); + if let Err(SqlError::QueryReturnedNoRows) = maybe_result { + return Ok(None); + } + + let result = maybe_result?; + Ok(Some(result)) +} + +fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } + +impl BlockHeaderStorageError for SqlError {} + +#[async_trait] +impl BlockHeaderStorage for SqliteBlockHeadersStorage { + type Error = SqlError; + + async fn init(&self, for_coin: &str) -> Result<(), MmError> { + let selfi = self.clone(); + let sql_cache = create_block_header_cache_table_sql(for_coin)?; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + conn.execute(&sql_cache, NO_PARAMS).map(|_| ())?; + Ok(()) + }) + .await + } + + async fn is_initialized_for(&self, for_coin: &str) -> Result> { + let block_headers_cache_table = block_headers_cache_table(for_coin); + validate_table_name(&block_headers_cache_table)?; + + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let cache_initialized = query_single_row( + &conn, + CHECK_TABLE_EXISTS_SQL, + [block_headers_cache_table], + string_from_row, + )?; + Ok(cache_initialized.is_some()) + }) + .await + } + + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: Vec, + ) -> Result<(), MmError> { + todo!() + } + + async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError> { + todo!() + } + + async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError> { + todo!() + } +} + +#[cfg(test)] +impl SqliteBlockHeadersStorage { + pub fn in_memory() -> Self { + SqliteBlockHeadersStorage(Arc::new(Mutex::new(Connection::open_in_memory().unwrap()))) + } +} + +#[cfg(test)] +mod sql_block_headers_storage_tests { + use super::*; + use common::block_on; + use std::num::NonZeroUsize; + + #[test] + fn test_init_collection() { + let for_coin = "init_collection"; + let storage = SqliteBlockHeadersStorage::in_memory(); + let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); + assert!(!initialized); + + block_on(storage.init(for_coin)).unwrap(); + // repetitive init must not fail + block_on(storage.init(for_coin)).unwrap(); + + let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); + assert!(initialized); + } +} From f3728f828ee67486602170b0187690a5b8482b15 Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 17 Feb 2022 08:44:18 +0100 Subject: [PATCH 19/74] feat(sql): implement partially add_block_headers_to_storage and the unit test --- .../utxo/utxo_sql_block_header_storage.rs | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index dbd155ab3b..2317f4a5fc 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -12,6 +12,7 @@ use std::sync::{Arc, Mutex}; const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; fn block_headers_cache_table(ticker: &str) -> String { ticker.to_owned() + "_block_headers_cache" } + fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { let table_name = block_headers_cache_table(for_coin); validate_table_name(&table_name)?; @@ -32,7 +33,6 @@ fn insert_block_header_in_cache_sql(for_coin: &str) -> Result, ) -> Result<(), MmError> { - todo!() + let for_coin = for_coin.to_owned(); + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + for header in headers { + match header { + ElectrumBlockHeader::V12(_) => {}, + ElectrumBlockHeader::V14(h) => { + let block_hex = format!("{:02x}", h.hex); + let block_cache_params = [&h.height.to_string(), &block_hex]; + sql_transaction.execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params)?; + }, + } + } + sql_transaction.commit()?; + Ok(()) + }) + .await } async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError> { @@ -118,13 +136,21 @@ impl SqliteBlockHeadersStorage { pub fn in_memory() -> Self { SqliteBlockHeadersStorage(Arc::new(Mutex::new(Connection::open_in_memory().unwrap()))) } + + fn is_table_empty(&self, table_name: &str) -> bool { + validate_table_name(table_name).unwrap(); + let sql = "SELECT COUNT(block_height) FROM ".to_owned() + table_name + ";"; + let conn = self.0.lock().unwrap(); + let rows_count: u32 = conn.query_row(&sql, NO_PARAMS, |row| row.get(0)).unwrap(); + rows_count == 0 + } } #[cfg(test)] mod sql_block_headers_storage_tests { use super::*; + use crate::utxo::rpc_clients::ElectrumBlockHeaderV14; use common::block_on; - use std::num::NonZeroUsize; #[test] fn test_init_collection() { @@ -140,4 +166,23 @@ mod sql_block_headers_storage_tests { let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); assert!(initialized); } + + #[test] + fn test_add_block_headers() { + let for_coin = "insert"; + let storage = SqliteBlockHeadersStorage::in_memory(); + let table = block_headers_cache_table(for_coin); + block_on(storage.init(for_coin)).unwrap(); + + let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); + assert!(initialized); + + let block_header = ElectrumBlockHeaderV14 { + height: 520481, + hex: "00000020890208a0ae3a3892aa047c5468725846577cfcd9b512b50000000000000000005dc2b02f2d297a9064ee103036c14d678f9afc7e3d9409cf53fd58b82e938e8ecbeca05a2d2103188ce804c4".into(), + }.into(); + let headers = vec![ElectrumBlockHeader::V14(block_header)]; + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); + assert!(!storage.is_table_empty(&table)); + } } From 5c408a89cfae7bce1d8f1813004023c1703ce3a2 Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 17 Feb 2022 10:00:44 +0100 Subject: [PATCH 20/74] feat(sql): add get_block header functions --- .../utxo/utxo_sql_block_header_storage.rs | 64 ++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index 2317f4a5fc..076a11bca0 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -4,9 +4,11 @@ use async_trait::async_trait; use chain::BlockHeader; use common::async_blocking; use common::mm_error::MmError; +use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::Error as SqlError; use db_common::sqlite::rusqlite::{Connection, Row, ToSql, NO_PARAMS}; use db_common::sqlite::validate_table_name; +use serialization::deserialize; use std::sync::{Arc, Mutex}; const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; @@ -36,6 +38,15 @@ fn insert_block_header_in_cache_sql(for_coin: &str) -> Result Result> { + let table_name = block_headers_cache_table(for_coin); + validate_table_name(&table_name)?; + + let sql = "SELECT hex FROM ".to_owned() + &table_name + " WHERE block_height=?1;"; + + Ok(sql) +} + #[derive(Clone)] pub struct SqliteBlockHeadersStorage(pub Arc>); @@ -123,11 +134,26 @@ impl BlockHeaderStorage for SqliteBlockHeadersStorage { } async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError> { - todo!() + if let Some(header_raw) = self.get_block_header_raw(for_coin, height).await? { + let header_bytes = + hex::decode(header_raw).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e)))?; + let header: BlockHeader = deserialize(header_bytes.as_slice()) + .map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e)))?; + return Ok(Some(header)); + } + Ok(None) } async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError> { - todo!() + let params = [height.to_string()]; + let sql = get_block_header_by_height(for_coin)?; + let selfi = self.clone(); + + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, params, string_from_row) + }) + .await } } @@ -151,6 +177,8 @@ mod sql_block_headers_storage_tests { use super::*; use crate::utxo::rpc_clients::ElectrumBlockHeaderV14; use common::block_on; + use hex::FromHex; + use primitives::hash::H256; #[test] fn test_init_collection() { @@ -179,10 +207,40 @@ mod sql_block_headers_storage_tests { let block_header = ElectrumBlockHeaderV14 { height: 520481, - hex: "00000020890208a0ae3a3892aa047c5468725846577cfcd9b512b50000000000000000005dc2b02f2d297a9064ee103036c14d678f9afc7e3d9409cf53fd58b82e938e8ecbeca05a2d2103188ce804c4".into(), + hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), }.into(); let headers = vec![ElectrumBlockHeader::V14(block_header)]; block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); assert!(!storage.is_table_empty(&table)); } + + #[test] + fn test_get_block_header() { + let for_coin = "get"; + let storage = SqliteBlockHeadersStorage::in_memory(); + let table = block_headers_cache_table(for_coin); + block_on(storage.init(for_coin)).unwrap(); + + let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); + assert!(initialized); + + let block_header = ElectrumBlockHeaderV14 { + height: 520481, + hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), + }.into(); + let headers = vec![ElectrumBlockHeader::V14(block_header)]; + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); + assert!(!storage.is_table_empty(&table)); + + let hex = block_on(storage.get_block_header_raw(for_coin, 520481)) + .unwrap() + .unwrap(); + assert_eq!(hex, "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".to_string()); + + let block_header = block_on(storage.get_block_header(for_coin, 520481)).unwrap().unwrap(); + assert_eq!( + block_header.hash(), + H256::from_reversed_str("0000000000000000002e31d0714a5ab23100945ff87ba2d856cd566a3c9344ec") + ) + } } From d880ff87a06e5aa6fb6c7af18e58f8bfbb75d1ac Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 18 Feb 2022 10:42:16 +0100 Subject: [PATCH 21/74] feat(sql): add extra function for block insertion --- mm2src/coins/utxo/rpc_clients.rs | 16 ++++++-- .../coins/utxo/utxo_block_header_storage.rs | 13 ++++++- .../utxo/utxo_sql_block_header_storage.rs | 38 ++++++++++++++++--- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 77bf7776d3..6c42f38553 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1022,8 +1022,8 @@ pub struct ElectrumBlockHeaderV12 { } impl ElectrumBlockHeaderV12 { - pub fn hash(&self) -> H256Json { - let block_header = BlockHeader { + fn as_block_header(&self) -> BlockHeader { + BlockHeader { version: self.version as u32, previous_header_hash: self.prev_block_hash.clone().into(), merkle_root_hash: self.merkle_root.clone().into(), @@ -1042,7 +1042,17 @@ impl ElectrumBlockHeaderV12 { n_height: None, n_nonce_u64: None, mix_hash: None, - }; + } + } + + pub fn as_hex(&self) -> String { + let block_header = self.as_block_header(); + let serialized = serialize(&block_header); + hex::encode(serialized) + } + + pub fn hash(&self) -> H256Json { + let block_header = self.as_block_header(); BlockHeader::hash(&block_header).into() } } diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 1685fe8af5..31b16873fd 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -2,6 +2,7 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; use async_trait::async_trait; use chain::BlockHeader; use common::{mm_error::MmError, mm_error::NotMmError, NotSame}; +use std::collections::HashMap; pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send {} @@ -16,12 +17,22 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { // Adds multiple block headers to the selected coin's header storage // Should store it as `TICKER_HEIGHT=hex_string` - async fn add_block_headers_to_storage( + // use this function for headers that comes from `blockchain_headers_subscribe` + async fn add_electrum_block_headers_to_storage( &self, for_coin: &str, headers: Vec, ) -> Result<(), MmError>; + // Adds multiple block headers to the selected coin's header storage + // Should store it as `TICKER_HEIGHT=hex_string` + // use this function for headers that comes from `blockchain_block_headers` + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: HashMap, + ) -> Result<(), MmError>; + /// Gets the block header by height from the selected coin's storage as BlockHeader async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError>; diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index 076a11bca0..de04012fdd 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -8,7 +8,8 @@ use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::Error as SqlError; use db_common::sqlite::rusqlite::{Connection, Row, ToSql, NO_PARAMS}; use db_common::sqlite::validate_table_name; -use serialization::deserialize; +use serialization::{deserialize, serialize}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; @@ -107,7 +108,7 @@ impl BlockHeaderStorage for SqliteBlockHeadersStorage { .await } - async fn add_block_headers_to_storage( + async fn add_electrum_block_headers_to_storage( &self, for_coin: &str, headers: Vec, @@ -119,7 +120,11 @@ impl BlockHeaderStorage for SqliteBlockHeadersStorage { let sql_transaction = conn.transaction()?; for header in headers { match header { - ElectrumBlockHeader::V12(_) => {}, + ElectrumBlockHeader::V12(h) => { + let block_hex = h.as_hex(); + let block_cache_params = [&h.block_height.to_string(), &block_hex]; + sql_transaction.execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params)?; + }, ElectrumBlockHeader::V14(h) => { let block_hex = format!("{:02x}", h.hex); let block_cache_params = [&h.height.to_string(), &block_hex]; @@ -133,6 +138,28 @@ impl BlockHeaderStorage for SqliteBlockHeadersStorage { .await } + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: HashMap, + ) -> Result<(), MmError> { + let for_coin = for_coin.to_owned(); + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + for (height, header) in headers { + let serialized = serialize(&header); + let block_hex = hex::encode(&serialized); + let block_cache_params = [&height.to_string(), &block_hex]; + sql_transaction.execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params)?; + } + sql_transaction.commit()?; + Ok(()) + }) + .await + } + async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError> { if let Some(header_raw) = self.get_block_header_raw(for_coin, height).await? { let header_bytes = @@ -177,7 +204,6 @@ mod sql_block_headers_storage_tests { use super::*; use crate::utxo::rpc_clients::ElectrumBlockHeaderV14; use common::block_on; - use hex::FromHex; use primitives::hash::H256; #[test] @@ -210,7 +236,7 @@ mod sql_block_headers_storage_tests { hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), }.into(); let headers = vec![ElectrumBlockHeader::V14(block_header)]; - block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); + block_on(storage.add_electrum_block_headers_to_storage(for_coin, headers)).unwrap(); assert!(!storage.is_table_empty(&table)); } @@ -229,7 +255,7 @@ mod sql_block_headers_storage_tests { hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), }.into(); let headers = vec![ElectrumBlockHeader::V14(block_header)]; - block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); + block_on(storage.add_electrum_block_headers_to_storage(for_coin, headers)).unwrap(); assert!(!storage.is_table_empty(&table)); let hex = block_on(storage.get_block_header_raw(for_coin, 520481)) From b62ab5a8d0a8db55b89101f630fd3b88467d5198 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 09:34:32 +0100 Subject: [PATCH 22/74] feat(lib_bitcoin): add raw header type --- mm2src/mm2_bitcoin/chain/src/lib.rs | 1 + mm2src/mm2_bitcoin/chain/src/raw_block.rs | 67 +++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 mm2src/mm2_bitcoin/chain/src/raw_block.rs diff --git a/mm2src/mm2_bitcoin/chain/src/lib.rs b/mm2src/mm2_bitcoin/chain/src/lib.rs index 8761e7c434..a81fe27bab 100644 --- a/mm2src/mm2_bitcoin/chain/src/lib.rs +++ b/mm2src/mm2_bitcoin/chain/src/lib.rs @@ -10,6 +10,7 @@ pub mod constants; mod block; mod block_header; mod merkle_root; +mod raw_block; mod transaction; /// `IndexedBlock` extension diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs new file mode 100644 index 0000000000..996e6f7d59 --- /dev/null +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -0,0 +1,67 @@ +use crypto::dhash256; +use primitives::bytes::Bytes; +use primitives::hash::H256; + +/// Hex-encoded block +#[derive(Default, PartialEq, Clone, Eq, Hash)] +pub struct RawHeader(Bytes); + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum RawHeaderError { + WrongLengthHeader, +} + +impl RawHeader { + pub fn new(buf: Vec) -> Result { + if buf.len() < 80 { + return Err(RawHeaderError::WrongLengthHeader); + } + let header = RawHeader(buf.into()); + Ok(header) + } + + /// Calculate the LE header digest + pub fn digest(&self) -> H256 { dhash256(&self.0.as_slice()) } + + /// Extract the LE tx merkle root from the header + pub fn extract_merkle_root(&self) -> H256 { + let mut root = H256::default(); + root.as_mut().copy_from_slice(&self.0.as_ref()[36..68]); + root + } + + /// Extract the LE parent digest from the header + pub fn parent(&self) -> H256 { + let mut root = H256::default(); + root.as_mut().copy_from_slice(&self.0.as_ref()[4..36]); + root + } +} + +#[cfg(test)] +mod raw_block_header_tests { + use hex::FromHex; + use primitives::hash::H256; + use raw_block::RawHeader; + + #[test] + fn test_raw_header() { + // block header of https://kmdexplorer.io/block/01ad31e22fea912a974c3e1eea11dc26348676528a586f77199ac3cfe29e271f + let header_hex = "040000008e4e7283b71dd1572d220935db0a1654d1042e92378579f8abab67b143f93a02fa026610d2634b72ff729b9ea7850c0d2c25eeaf7a82878ca42a8e9912028863a2d8a734eb73a4dc734072dbfd12406f1e7121bfe0e3d6c10922495c44e5cc1c91185d5ee519011d0400b9caaf41d4b63a6ab55bb4e6925d46fc3adea7be37b713d3a615e7cf0000fd40050001a80fa65b9a46fdb1506a7a4d26f43e7995d69902489b9f6c4599c88f9c169605cc135258953da0d6299ada4ff81a76ad63c943261078d5dd1918f91cea68b65b7fc362e9df49ba57c2ea5c6dba91591c85eb0d59a1905ac66e2295b7a291a1695301489a3cc7310fd45f2b94e3b8d94f3051e9bbaada1e0641fcec6e0d6230e76753aa9574a3f3e28eaa085959beffd3231dbe1aeea3955328f3a973650a38e31632a4ffc7ec007a3345124c0b99114e2444b3ef0ada75adbd077b247bbf3229adcffbe95bc62daac88f96317d5768540b5db636f8c39a8529a736465ed830ab2c1bbddf523587abe14397a6f1835d248092c4b5b691a955572607093177a5911e317739187b41f4aa662aa6bca0401f1a0a77915ebb6947db686cff549c5f4e7b9dd93123b00a1ae8d411cfb13fa7674de21cbee8e9fc74e12aa6753b261eab3d9256c7c32cc9b16219dad73c61014e7d88d74d5e218f12e11bc47557347ff49a9ab4490647418d2a5c2da1df24d16dfb611173608fe4b10a357b0fa7a1918b9f2d7836c84bf05f384e1e678b2fdd47af0d8e66e739fe45209ede151a180aba1188058a0db093e30bc9851980cf6fbfa5adb612d1146905da662c3347d7e7e569a1041641049d951ab867bc0c6a3863c7667d43f596a849434958cee2b63dc8fa11bd0f38aa96df86ed66461993f64736345313053508c4e939506c08a766f5b6ed0950759f3901bbc4db3dc97e05bf20b9dda4ff242083db304a4e487ac2101b823998371542354e5d534b5b6ae6420cc19b11512108b61208f4d9a5a97263d2c060da893544dea6251bcadc682d2238af35f2b1c2f65a73b89a4e194f9e1eef6f0e5948ef8d0d2862f48fd3356126b00c6a2d3770ecd0d1a78fa34974b454f270b23d461e357c9356c19496522b59ff9d5b4608c542ff89e558798324021704b2cfe9f6c1a70906c43c7a690f16615f198d29fa647d84ce8461fa570b33e3eada2ed7d77e1f280a0d2e9f03c2e1db535d922b1759a191b417595f3c15d8e8b7f810527ff942e18443a3860e67ccba356809ecedc31c5d8db59c7e039dae4b53d126679e8ffa20cc26e8b9d229c8f6ee434ad053f5f4f5a94e249a13afb995aad82b4d90890187e516e114b168fc7c7e291b9738ea578a7bab0ba31030b14ba90b772b577806ea2d17856b0cb9e74254ba582a9f2638ea7ed2ca23be898c6108ff8f466b443537ed9ec56b8771bfbf0f2f6e1092a28a7fd182f111e1dbdd155ea82c6cb72d5f9e6518cc667b8226b5f5c6646125fc851e97cf125f48949f988ed37c4283072fc03dd1da3e35161e17f44c0e22c76f708bb66405737ef24176e291b4fc2eadab876115dc62d48e053a85f0ad132ef07ad5175b036fe39e1ad14fcdcdc6ac5b3daabe05161a72a50545dd812e0f9af133d061b726f491e904d89ee57811ef58d3bda151f577aed381963a30d91fb98dc49413300d132a7021a5e834e266b4ac982d76e00f43f5336b8e8028a0cacfa11813b01e50f71236a73a4c0d0757c1832b0680ada56c80edf070f438ab2bc587542f926ff8d3644b8b8a56c78576f127dec7aed9cb3e1bc2442f978a9df1dc3056a63e653132d0f419213d3cb86e7b61720de1aa3af4b3757a58156970da27560c6629257158452b9d5e4283dc6fe7df42d2fda3352d5b62ce5a984d912777c3b01837df8968a4d494db1b663e0e68197dbf196f21ea11a77095263dec548e2010460840231329d83978885ee2423e8b327785970e27c6c6d436157fb5b56119b19239edbb730ebae013d82c35df4a6e70818a74d1ef7a2e87c090ff90e32939f58ed24e85b492b5750fd2cd14b9b8517136b76b1cc6ccc6f6f027f65f1967a0eb4f32cd6e5d5315"; + let header_bytes: Vec = header_hex.from_hex().unwrap(); + let raw_header = RawHeader::new(header_bytes).unwrap(); + assert_eq!( + raw_header.digest(), + H256::from_reversed_str("01ad31e22fea912a974c3e1eea11dc26348676528a586f77199ac3cfe29e271f") + ); + assert_eq!( + raw_header.extract_merkle_root(), + H256::from_reversed_str("63880212998e2aa48c87827aafee252c0d0c85a79e9b72ff724b63d2106602fa") + ); + + assert_eq!( + raw_header.parent(), + H256::from_reversed_str("023af943b167ababf8798537922e04d154160adb3509222d57d11db783724e8e") + ); + } +} From 3710ea7345c44c219e2048ece87d9c93a7447bbe Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 09:53:35 +0100 Subject: [PATCH 23/74] feat(spv): Integrate raw block header into spv validation --- Cargo.lock | 2 + mm2src/mm2_bitcoin/chain/src/lib.rs | 1 + mm2src/mm2_bitcoin/chain/src/raw_block.rs | 10 ++--- mm2src/mm2_bitcoin/spv_validation/Cargo.toml | 4 +- mm2src/mm2_bitcoin/spv_validation/src/lib.rs | 2 + .../spv_validation/src/spv_proof.rs | 43 +++++++++++++++++++ 6 files changed, 56 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d35b2728df..f4efd4cf75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5345,6 +5345,8 @@ dependencies = [ "bitcoin-spv", "chain", "primitives", + "rustc-hex 2.1.0", + "serialization", ] [[package]] diff --git a/mm2src/mm2_bitcoin/chain/src/lib.rs b/mm2src/mm2_bitcoin/chain/src/lib.rs index a81fe27bab..47f5e537e9 100644 --- a/mm2src/mm2_bitcoin/chain/src/lib.rs +++ b/mm2src/mm2_bitcoin/chain/src/lib.rs @@ -11,6 +11,7 @@ mod block; mod block_header; mod merkle_root; mod raw_block; +pub use raw_block::RawBlockHeader; mod transaction; /// `IndexedBlock` extension diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index 996e6f7d59..00bdb617b0 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -4,19 +4,19 @@ use primitives::hash::H256; /// Hex-encoded block #[derive(Default, PartialEq, Clone, Eq, Hash)] -pub struct RawHeader(Bytes); +pub struct RawBlockHeader(Bytes); #[derive(Debug, PartialEq, Eq, Clone)] pub enum RawHeaderError { WrongLengthHeader, } -impl RawHeader { +impl RawBlockHeader { pub fn new(buf: Vec) -> Result { if buf.len() < 80 { return Err(RawHeaderError::WrongLengthHeader); } - let header = RawHeader(buf.into()); + let header = RawBlockHeader(buf.into()); Ok(header) } @@ -42,14 +42,14 @@ impl RawHeader { mod raw_block_header_tests { use hex::FromHex; use primitives::hash::H256; - use raw_block::RawHeader; + use raw_block::RawBlockHeader; #[test] fn test_raw_header() { // block header of https://kmdexplorer.io/block/01ad31e22fea912a974c3e1eea11dc26348676528a586f77199ac3cfe29e271f let header_hex = "040000008e4e7283b71dd1572d220935db0a1654d1042e92378579f8abab67b143f93a02fa026610d2634b72ff729b9ea7850c0d2c25eeaf7a82878ca42a8e9912028863a2d8a734eb73a4dc734072dbfd12406f1e7121bfe0e3d6c10922495c44e5cc1c91185d5ee519011d0400b9caaf41d4b63a6ab55bb4e6925d46fc3adea7be37b713d3a615e7cf0000fd40050001a80fa65b9a46fdb1506a7a4d26f43e7995d69902489b9f6c4599c88f9c169605cc135258953da0d6299ada4ff81a76ad63c943261078d5dd1918f91cea68b65b7fc362e9df49ba57c2ea5c6dba91591c85eb0d59a1905ac66e2295b7a291a1695301489a3cc7310fd45f2b94e3b8d94f3051e9bbaada1e0641fcec6e0d6230e76753aa9574a3f3e28eaa085959beffd3231dbe1aeea3955328f3a973650a38e31632a4ffc7ec007a3345124c0b99114e2444b3ef0ada75adbd077b247bbf3229adcffbe95bc62daac88f96317d5768540b5db636f8c39a8529a736465ed830ab2c1bbddf523587abe14397a6f1835d248092c4b5b691a955572607093177a5911e317739187b41f4aa662aa6bca0401f1a0a77915ebb6947db686cff549c5f4e7b9dd93123b00a1ae8d411cfb13fa7674de21cbee8e9fc74e12aa6753b261eab3d9256c7c32cc9b16219dad73c61014e7d88d74d5e218f12e11bc47557347ff49a9ab4490647418d2a5c2da1df24d16dfb611173608fe4b10a357b0fa7a1918b9f2d7836c84bf05f384e1e678b2fdd47af0d8e66e739fe45209ede151a180aba1188058a0db093e30bc9851980cf6fbfa5adb612d1146905da662c3347d7e7e569a1041641049d951ab867bc0c6a3863c7667d43f596a849434958cee2b63dc8fa11bd0f38aa96df86ed66461993f64736345313053508c4e939506c08a766f5b6ed0950759f3901bbc4db3dc97e05bf20b9dda4ff242083db304a4e487ac2101b823998371542354e5d534b5b6ae6420cc19b11512108b61208f4d9a5a97263d2c060da893544dea6251bcadc682d2238af35f2b1c2f65a73b89a4e194f9e1eef6f0e5948ef8d0d2862f48fd3356126b00c6a2d3770ecd0d1a78fa34974b454f270b23d461e357c9356c19496522b59ff9d5b4608c542ff89e558798324021704b2cfe9f6c1a70906c43c7a690f16615f198d29fa647d84ce8461fa570b33e3eada2ed7d77e1f280a0d2e9f03c2e1db535d922b1759a191b417595f3c15d8e8b7f810527ff942e18443a3860e67ccba356809ecedc31c5d8db59c7e039dae4b53d126679e8ffa20cc26e8b9d229c8f6ee434ad053f5f4f5a94e249a13afb995aad82b4d90890187e516e114b168fc7c7e291b9738ea578a7bab0ba31030b14ba90b772b577806ea2d17856b0cb9e74254ba582a9f2638ea7ed2ca23be898c6108ff8f466b443537ed9ec56b8771bfbf0f2f6e1092a28a7fd182f111e1dbdd155ea82c6cb72d5f9e6518cc667b8226b5f5c6646125fc851e97cf125f48949f988ed37c4283072fc03dd1da3e35161e17f44c0e22c76f708bb66405737ef24176e291b4fc2eadab876115dc62d48e053a85f0ad132ef07ad5175b036fe39e1ad14fcdcdc6ac5b3daabe05161a72a50545dd812e0f9af133d061b726f491e904d89ee57811ef58d3bda151f577aed381963a30d91fb98dc49413300d132a7021a5e834e266b4ac982d76e00f43f5336b8e8028a0cacfa11813b01e50f71236a73a4c0d0757c1832b0680ada56c80edf070f438ab2bc587542f926ff8d3644b8b8a56c78576f127dec7aed9cb3e1bc2442f978a9df1dc3056a63e653132d0f419213d3cb86e7b61720de1aa3af4b3757a58156970da27560c6629257158452b9d5e4283dc6fe7df42d2fda3352d5b62ce5a984d912777c3b01837df8968a4d494db1b663e0e68197dbf196f21ea11a77095263dec548e2010460840231329d83978885ee2423e8b327785970e27c6c6d436157fb5b56119b19239edbb730ebae013d82c35df4a6e70818a74d1ef7a2e87c090ff90e32939f58ed24e85b492b5750fd2cd14b9b8517136b76b1cc6ccc6f6f027f65f1967a0eb4f32cd6e5d5315"; let header_bytes: Vec = header_hex.from_hex().unwrap(); - let raw_header = RawHeader::new(header_bytes).unwrap(); + let raw_header = RawBlockHeader::new(header_bytes).unwrap(); assert_eq!( raw_header.digest(), H256::from_reversed_str("01ad31e22fea912a974c3e1eea11dc26348676528a586f77199ac3cfe29e271f") diff --git a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml index bf799a9e81..c8af6d79bb 100644 --- a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml +++ b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml @@ -6,4 +6,6 @@ authors = ["Roman Sztergbaum "] [dependencies] bitcoin-spv = "5.0.0" chain = {path = "../chain"} -primitives = { path = "../primitives" } \ No newline at end of file +primitives = { path = "../primitives" } +serialization = { path = "../serialization" } +rustc-hex = "2" \ No newline at end of file diff --git a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs index ddb9c11b02..c0d52eb202 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs @@ -1,6 +1,8 @@ extern crate bitcoin_spv; extern crate chain; extern crate primitives; +extern crate rustc_hex as hex; +extern crate serialization; /// `types` exposes simple types for on-chain evaluation of SPV proofs pub mod types; diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 87400dd7fa..969cf35223 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -1,5 +1,6 @@ use bitcoin_spv::btcspv::{validate_vin, validate_vout}; use chain::BlockHeader; +use chain::RawBlockHeader; use helpers_validation::merkle_prove; use primitives::hash::H256; use types::SPVError; @@ -16,6 +17,8 @@ pub struct SPVProof { pub index: u64, /// The confirming UTXO header pub confirming_header: BlockHeader, + /// The Raw confirming UTXO Header + pub raw_header: RawBlockHeader, /// The intermediate nodes (digests between leaf and root) pub intermediate_nodes: Vec, } @@ -34,6 +37,19 @@ pub struct SPVProof { /// Re-write with our own types based on `bitcoin_spv::std_types::SPVProof::validate` /// Support only merkle proof inclusion,vin,vout for now impl SPVProof { + pub fn validate_block_header(&self) -> Result<(), SPVError> { + if self.confirming_header.hash() != self.raw_header.digest() { + return Err(SPVError::WrongDigest); + } + if self.confirming_header.merkle_root_hash != self.raw_header.extract_merkle_root() { + return Err(SPVError::WrongMerkleRoot); + } + if self.confirming_header.previous_header_hash != self.raw_header.parent() { + return Err(SPVError::WrongPrevHash); + } + Ok(()) + } + pub fn validate(&self) -> Result<(), SPVError> { if !validate_vin(self.vin.as_slice()) { return Err(SPVError::InvalidVin); @@ -41,6 +57,7 @@ impl SPVProof { if !validate_vout(self.vout.as_slice()) { return Err(SPVError::InvalidVout); } + self.validate_block_header()?; merkle_prove( self.tx_id, self.confirming_header.merkle_root_hash, @@ -49,3 +66,29 @@ impl SPVProof { ) } } + +#[cfg(test)] +mod spv_proof_tests { + use chain::BlockHeader; + use chain::RawBlockHeader; + use hex::FromHex; + use serialization::deserialize; + use spv_proof::SPVProof; + + #[test] + fn test_block_header() { + let header_hex = "040000008e4e7283b71dd1572d220935db0a1654d1042e92378579f8abab67b143f93a02fa026610d2634b72ff729b9ea7850c0d2c25eeaf7a82878ca42a8e9912028863a2d8a734eb73a4dc734072dbfd12406f1e7121bfe0e3d6c10922495c44e5cc1c91185d5ee519011d0400b9caaf41d4b63a6ab55bb4e6925d46fc3adea7be37b713d3a615e7cf0000fd40050001a80fa65b9a46fdb1506a7a4d26f43e7995d69902489b9f6c4599c88f9c169605cc135258953da0d6299ada4ff81a76ad63c943261078d5dd1918f91cea68b65b7fc362e9df49ba57c2ea5c6dba91591c85eb0d59a1905ac66e2295b7a291a1695301489a3cc7310fd45f2b94e3b8d94f3051e9bbaada1e0641fcec6e0d6230e76753aa9574a3f3e28eaa085959beffd3231dbe1aeea3955328f3a973650a38e31632a4ffc7ec007a3345124c0b99114e2444b3ef0ada75adbd077b247bbf3229adcffbe95bc62daac88f96317d5768540b5db636f8c39a8529a736465ed830ab2c1bbddf523587abe14397a6f1835d248092c4b5b691a955572607093177a5911e317739187b41f4aa662aa6bca0401f1a0a77915ebb6947db686cff549c5f4e7b9dd93123b00a1ae8d411cfb13fa7674de21cbee8e9fc74e12aa6753b261eab3d9256c7c32cc9b16219dad73c61014e7d88d74d5e218f12e11bc47557347ff49a9ab4490647418d2a5c2da1df24d16dfb611173608fe4b10a357b0fa7a1918b9f2d7836c84bf05f384e1e678b2fdd47af0d8e66e739fe45209ede151a180aba1188058a0db093e30bc9851980cf6fbfa5adb612d1146905da662c3347d7e7e569a1041641049d951ab867bc0c6a3863c7667d43f596a849434958cee2b63dc8fa11bd0f38aa96df86ed66461993f64736345313053508c4e939506c08a766f5b6ed0950759f3901bbc4db3dc97e05bf20b9dda4ff242083db304a4e487ac2101b823998371542354e5d534b5b6ae6420cc19b11512108b61208f4d9a5a97263d2c060da893544dea6251bcadc682d2238af35f2b1c2f65a73b89a4e194f9e1eef6f0e5948ef8d0d2862f48fd3356126b00c6a2d3770ecd0d1a78fa34974b454f270b23d461e357c9356c19496522b59ff9d5b4608c542ff89e558798324021704b2cfe9f6c1a70906c43c7a690f16615f198d29fa647d84ce8461fa570b33e3eada2ed7d77e1f280a0d2e9f03c2e1db535d922b1759a191b417595f3c15d8e8b7f810527ff942e18443a3860e67ccba356809ecedc31c5d8db59c7e039dae4b53d126679e8ffa20cc26e8b9d229c8f6ee434ad053f5f4f5a94e249a13afb995aad82b4d90890187e516e114b168fc7c7e291b9738ea578a7bab0ba31030b14ba90b772b577806ea2d17856b0cb9e74254ba582a9f2638ea7ed2ca23be898c6108ff8f466b443537ed9ec56b8771bfbf0f2f6e1092a28a7fd182f111e1dbdd155ea82c6cb72d5f9e6518cc667b8226b5f5c6646125fc851e97cf125f48949f988ed37c4283072fc03dd1da3e35161e17f44c0e22c76f708bb66405737ef24176e291b4fc2eadab876115dc62d48e053a85f0ad132ef07ad5175b036fe39e1ad14fcdcdc6ac5b3daabe05161a72a50545dd812e0f9af133d061b726f491e904d89ee57811ef58d3bda151f577aed381963a30d91fb98dc49413300d132a7021a5e834e266b4ac982d76e00f43f5336b8e8028a0cacfa11813b01e50f71236a73a4c0d0757c1832b0680ada56c80edf070f438ab2bc587542f926ff8d3644b8b8a56c78576f127dec7aed9cb3e1bc2442f978a9df1dc3056a63e653132d0f419213d3cb86e7b61720de1aa3af4b3757a58156970da27560c6629257158452b9d5e4283dc6fe7df42d2fda3352d5b62ce5a984d912777c3b01837df8968a4d494db1b663e0e68197dbf196f21ea11a77095263dec548e2010460840231329d83978885ee2423e8b327785970e27c6c6d436157fb5b56119b19239edbb730ebae013d82c35df4a6e70818a74d1ef7a2e87c090ff90e32939f58ed24e85b492b5750fd2cd14b9b8517136b76b1cc6ccc6f6f027f65f1967a0eb4f32cd6e5d5315"; + let header_bytes: Vec = header_hex.from_hex().unwrap(); + let header: BlockHeader = deserialize(header_bytes.as_slice()).unwrap(); + let spv_proof = SPVProof { + tx_id: Default::default(), + vin: vec![], + vout: vec![], + index: 0, + confirming_header: header, + raw_header: RawBlockHeader::new(header_bytes).unwrap(), + intermediate_nodes: vec![], + }; + assert!(spv_proof.validate_block_header().is_ok()) + } +} From 0a0eb479059dd7e17dc4efcc96544b5fd5af769b Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 10:27:17 +0100 Subject: [PATCH 24/74] feat(spv): remove old comments --- mm2src/mm2_bitcoin/chain/src/raw_block.rs | 2 +- mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index 00bdb617b0..3ce0a4e92e 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -21,7 +21,7 @@ impl RawBlockHeader { } /// Calculate the LE header digest - pub fn digest(&self) -> H256 { dhash256(&self.0.as_slice()) } + pub fn digest(&self) -> H256 { dhash256(self.0.as_slice()) } /// Extract the LE tx merkle root from the header pub fn extract_merkle_root(&self) -> H256 { diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 969cf35223..0aa378388d 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -35,7 +35,6 @@ pub struct SPVProof { /// /// # Notes /// Re-write with our own types based on `bitcoin_spv::std_types::SPVProof::validate` -/// Support only merkle proof inclusion,vin,vout for now impl SPVProof { pub fn validate_block_header(&self) -> Result<(), SPVError> { if self.confirming_header.hash() != self.raw_header.digest() { From deba4d0509eefa3918dd767393d2f31319b9b49d Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 10:40:14 +0100 Subject: [PATCH 25/74] feat(spv): Adding raw header into utxo_common spv validation --- mm2src/coins/utxo/utxo_common.rs | 4 +++- mm2src/mm2_bitcoin/chain/src/lib.rs | 2 +- mm2src/mm2_bitcoin/spv_validation/src/types.rs | 10 ++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index b25ddc5d52..c65bd61c27 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -7,7 +7,7 @@ use crate::{CanRefundHtlc, CoinBalance, TradePreimageValue, TxFeeDetails, Valida use bigdecimal::{BigDecimal, Zero}; pub use bitcrypto::{dhash160, sha256, ChecksumType}; use chain::constants::SEQUENCE_FINAL; -use chain::{BlockHeader, OutPoint, TransactionOutput}; +use chain::{BlockHeader, OutPoint, RawBlockHeader, TransactionOutput}; use common::executor::Timer; use common::jsonrpc_client::{JsonRpcError, JsonRpcErrorType}; use common::log::{error, info, warn}; @@ -2703,6 +2703,7 @@ where .compat() .await .map_to_mm(|_e| SPVError::UnknownError)?; + let raw_header = RawBlockHeader::new(block_header.0.clone())?; let header: BlockHeader = deserialize(block_header.0.as_slice()).map_to_mm(|_e| SPVError::UnknownError)?; let merkle_branch = client @@ -2721,6 +2722,7 @@ where vout: serialize_list(&tx.outputs).take(), index: merkle_branch.pos as u64, confirming_header: header, + raw_header, intermediate_nodes, }; match proof.validate() { diff --git a/mm2src/mm2_bitcoin/chain/src/lib.rs b/mm2src/mm2_bitcoin/chain/src/lib.rs index 47f5e537e9..b3f474fd49 100644 --- a/mm2src/mm2_bitcoin/chain/src/lib.rs +++ b/mm2src/mm2_bitcoin/chain/src/lib.rs @@ -11,7 +11,7 @@ mod block; mod block_header; mod merkle_root; mod raw_block; -pub use raw_block::RawBlockHeader; +pub use raw_block::{RawBlockHeader, RawHeaderError}; mod transaction; /// `IndexedBlock` extension diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index 15bdf6b7f8..2e0d8f2e6b 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -1,3 +1,5 @@ +use chain::RawHeaderError; + #[derive(Debug, PartialEq, Eq, Clone)] pub enum SPVError { /// Overran a checked read on a slice @@ -73,3 +75,11 @@ impl From for SPVError { } } } + +impl From for SPVError { + fn from(e: RawHeaderError) -> Self { + match e { + RawHeaderError::WrongLengthHeader => SPVError::WrongLengthHeader, + } + } +} From 6bd7c368d5d31aaae6a5ee237f388b6199196352 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 14:24:11 +0100 Subject: [PATCH 26/74] feat(spv): make slp great again --- mm2src/coins/utxo/utxo_common.rs | 31 ++++++++++--------- .../mm2_bitcoin/spv_validation/src/types.rs | 6 ++++ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index c65bd61c27..b23fa15b89 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2676,21 +2676,24 @@ where if tx.outputs.is_empty() { return MmError::err(SPVError::UnknownError); } - // note: all the outputs belong to the same tx, which is validated as the same height - // so accessing the history of the first element should be enough. - let script_pubkey_str = hex::encode(electrum_script_hash(&tx.outputs[0].script_pubkey)); - let history = client - .scripthash_get_history(script_pubkey_str.as_str()) - .compat() - .await - .unwrap_or_default(); - if history.is_empty() { - return MmError::err(SPVError::UnknownError); - } let mut height: u64 = 0; - for item in history { - if item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0 { - height = item.height as u64; + for output in tx.outputs.clone() { + let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); + let history = client + .scripthash_get_history(script_pubkey_str.as_str()) + .compat() + .await + .unwrap_or_default(); + if history.is_empty() { + continue; + } + for item in history { + if item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0 { + height = item.height as u64; + break; + } + } + if height != 0 { break; } } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index 2e0d8f2e6b..ce99c27ffd 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -16,6 +16,10 @@ pub enum SPVError { MalformattedWitnessOutput, /// `extract_hash` could not identify the output type. MalformattedOutput, + /// Unable to get block header from network or storage + UnableToGetHeader, + /// Unable to deserialize raw block header from electrum to concrete type + MalformattedHeader, /// Header not exactly 80 bytes. WrongLengthHeader, /// Header chain changed difficulties unexpectedly @@ -45,6 +49,8 @@ pub enum SPVError { BadMerkleProof, /// TxOut's reported length does not match passed-in byte slice's length OutputLengthMismatch, + /// scripthash_get_history return empty history + UnableToAccessElectrumHistoryItem, /// Any other error UnknownError, } From 3f9dfb6aba2aecd4c4d212b84f702d151f79a61e Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 15:12:30 +0100 Subject: [PATCH 27/74] feat(spv): returning more concrete error types --- mm2src/coins/utxo/utxo_common.rs | 10 +++++----- mm2src/mm2_bitcoin/spv_validation/src/types.rs | 8 +++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index b23fa15b89..aba1fe7508 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2674,7 +2674,7 @@ where UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, }; if tx.outputs.is_empty() { - return MmError::err(SPVError::UnknownError); + return MmError::err(SPVError::InvalidVout); } let mut height: u64 = 0; for output in tx.outputs.clone() { @@ -2698,22 +2698,22 @@ where } } if height == 0 { - return MmError::err(SPVError::UnknownError); + return MmError::err(SPVError::InvalidHeight); } let block_header = client .blockchain_block_header(height) .compat() .await - .map_to_mm(|_e| SPVError::UnknownError)?; + .map_to_mm(|_e| SPVError::UnableToGetHeader)?; let raw_header = RawBlockHeader::new(block_header.0.clone())?; - let header: BlockHeader = deserialize(block_header.0.as_slice()).map_to_mm(|_e| SPVError::UnknownError)?; + let header: BlockHeader = deserialize(block_header.0.as_slice()).map_to_mm(|_e| SPVError::MalformattedHeader)?; let merkle_branch = client .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height) .compat() .await - .map_to_mm(|_e| SPVError::UnknownError)?; + .map_to_mm(|_e| SPVError::UnableToGetMerkle)?; let intermediate_nodes: Vec = merkle_branch .merkle .into_iter() diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index ce99c27ffd..2fb1c293f1 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -39,7 +39,7 @@ pub enum SPVError { WrongPrevHash, /// A `vin` (transaction input vector) is malformatted. InvalidVin, - /// A `vout` (transaction output vector) is malformatted. + /// A `vout` (transaction output vector) is malformatted or empty. InvalidVout, /// When validating an `SPVProof`, the `tx_id` field is not the digest /// of the `version`, `vin`, `vout`, and `locktime`. @@ -47,10 +47,12 @@ pub enum SPVError { /// When validating an `SPVProof`, the `intermediate_nodes` is not a valid /// merkle proof connecting the `tx_id_le` to the `confirming_header`. BadMerkleProof, + /// Unable to get merkle tree from network or storage + UnableToGetMerkle, /// TxOut's reported length does not match passed-in byte slice's length OutputLengthMismatch, - /// scripthash_get_history return empty history - UnableToAccessElectrumHistoryItem, + /// Unable to retrieve block height / block height is zero. + InvalidHeight, /// Any other error UnknownError, } From 4c88900de4941c99fe43bcf3e33cb476731e4053 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 21:05:25 +0100 Subject: [PATCH 28/74] feat(toolchain): update to `nightly-2022-02-01` to fix osx compilation - https://github.com/rust-lang/rust/pull/93414 --- Cargo.lock | 72 ++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 +- README.md | 4 +-- mm2src/common/Cargo.toml | 2 +- mm2src/crypto/Cargo.toml | 2 +- rust-toolchain.toml | 2 +- 6 files changed, 75 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4efd4cf75..1317c41251 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -953,7 +953,7 @@ dependencies = [ "num-bigint 0.2.6", "num-rational", "num-traits 0.2.12", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "parking_lot_core 0.6.2", "paste", "primitives", @@ -1181,7 +1181,7 @@ dependencies = [ "http 0.2.1", "hw_common", "keys", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "primitives", "secp256k1", "ser_error", @@ -3344,7 +3344,7 @@ dependencies = [ "num-rational", "num-traits 0.2.12", "parity-util-mem", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "primitives", "rand 0.6.5", "rand 0.7.3", @@ -3776,6 +3776,16 @@ dependencies = [ "parking_lot_core 0.8.5", ] +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api 0.4.6", + "parking_lot_core 0.9.1", +] + [[package]] name = "parking_lot_core" version = "0.4.0" @@ -3832,6 +3842,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.2.8", + "smallvec 1.6.1", + "windows-sys", +] + [[package]] name = "paste" version = "1.0.4" @@ -6475,6 +6498,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + [[package]] name = "winreg" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index cc03b37d99..ddb70737a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,7 +88,7 @@ num-rational = { version = "0.2", features = ["serde", "bigint", "bigint-std"] } num-traits = "0.2" rpc = { path = "mm2src/mm2_bitcoin/rpc" } rpc_task = { path = "mm2src/rpc_task" } -parking_lot = { version = "0.11", features = ["nightly"] } +parking_lot = { version = "0.12.0", features = ["nightly"] } parity-util-mem = "0.9" # AP: portfolio RPCs are not documented and not used as of now # so the crate is disabled to speed up the entire removal of C code diff --git a/README.md b/README.md index 45f0c5c010..bbec952b81 100755 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ The current state can be considered as very early alpha. 1. (Optional) OSX: run `LIBRARY_PATH=/usr/local/opt/openssl/lib` 1. Run ``` - rustup install nightly-2021-12-16 - rustup default nightly-2021-12-16 + rustup install nightly-2022-02-01 + rustup default nightly-2022-02-01 rustup component add rustfmt-preview ``` 1. Run `cargo build` (or `cargo build -vv` to get verbose build output). diff --git a/mm2src/common/Cargo.toml b/mm2src/common/Cargo.toml index 0fc5b3be08..243e916ef5 100644 --- a/mm2src/common/Cargo.toml +++ b/mm2src/common/Cargo.toml @@ -42,7 +42,7 @@ log = "0.4.8" num-bigint = { version = "0.2", features = ["serde", "std"] } num-rational = { version = "0.2", features = ["serde", "bigint", "bigint-std"] } num-traits = "0.2" -parking_lot = { version = "0.11", features = ["nightly"] } +parking_lot = { version = "0.12.0", features = ["nightly"] } parking_lot_core = { version = "0.6", features = ["nightly"] } paste = "1.0" primitives = { path = "../mm2_bitcoin/primitives" } diff --git a/mm2src/crypto/Cargo.toml b/mm2src/crypto/Cargo.toml index 7254ac3f82..f8bb2bd160 100644 --- a/mm2src/crypto/Cargo.toml +++ b/mm2src/crypto/Cargo.toml @@ -15,7 +15,7 @@ futures = "0.3" hex = "0.4.2" http = "0.2" hw_common = { path = "../hw_common" } -parking_lot = { version = "0.11", features = ["nightly"] } +parking_lot = { version = "0.12.0", features = ["nightly"] } primitives = { path = "../mm2_bitcoin/primitives" } keys = { path = "../mm2_bitcoin/keys" } secp256k1 = "0.20" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4566e0b2cb..6c54b7e3e3 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-12-16" +channel = "nightly-2022-02-01" components = ["rustfmt"] From 2f25426d858d2b645a3ae2bbffa8f2a12061ab49 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 21:18:46 +0100 Subject: [PATCH 29/74] feat(cargo): fix warnings --- mm2src/coins/utxo/qtum_delegation.rs | 30 ++++++++++----------- mm2src/coins/utxo/utxo_withdraw.rs | 2 ++ mm2src/coins/utxo_signer/src/sign_common.rs | 2 +- mm2src/common/common.rs | 2 +- mm2src/gossipsub/src/behaviour.rs | 6 ++--- mm2src/lp_ordermatch.rs | 4 +++ 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/mm2src/coins/utxo/qtum_delegation.rs b/mm2src/coins/utxo/qtum_delegation.rs index c4c689de2b..20a85c5682 100644 --- a/mm2src/coins/utxo/qtum_delegation.rs +++ b/mm2src/coins/utxo/qtum_delegation.rs @@ -124,14 +124,13 @@ impl QtumCoin { let delegation_output = self.remove_delegation_output(QRC20_GAS_LIMIT_DEFAULT, QRC20_GAS_PRICE_DEFAULT)?; let outputs = vec![delegation_output]; let my_address = self.my_address().map_to_mm(DelegationError::InternalError)?; - Ok(self - .generate_delegation_transaction( - outputs, - my_address, - QRC20_GAS_LIMIT_DEFAULT, - TransactionType::RemoveDelegation, - ) - .await?) + self.generate_delegation_transaction( + outputs, + my_address, + QRC20_GAS_LIMIT_DEFAULT, + TransactionType::RemoveDelegation, + ) + .await } async fn am_i_currently_staking(&self) -> Result, MmError> { @@ -252,14 +251,13 @@ impl QtumCoin { let outputs = vec![delegation_output]; let my_address = self.my_address().map_to_mm(DelegationError::InternalError)?; - Ok(self - .generate_delegation_transaction( - outputs, - my_address, - QRC20_GAS_LIMIT_DELEGATION, - TransactionType::StakingDelegation, - ) - .await?) + self.generate_delegation_transaction( + outputs, + my_address, + QRC20_GAS_LIMIT_DELEGATION, + TransactionType::StakingDelegation, + ) + .await } async fn generate_delegation_transaction( diff --git a/mm2src/coins/utxo/utxo_withdraw.rs b/mm2src/coins/utxo/utxo_withdraw.rs index c2c045b4df..07d46ab445 100644 --- a/mm2src/coins/utxo/utxo_withdraw.rs +++ b/mm2src/coins/utxo/utxo_withdraw.rs @@ -109,8 +109,10 @@ where { fn coin(&self) -> &Coin; + #[allow(clippy::wrong_self_convention)] fn from_address(&self) -> Address; + #[allow(clippy::wrong_self_convention)] fn from_address_string(&self) -> String; fn request(&self) -> &WithdrawRequest; diff --git a/mm2src/coins/utxo_signer/src/sign_common.rs b/mm2src/coins/utxo_signer/src/sign_common.rs index 74a7c8906f..fff677f8fa 100644 --- a/mm2src/coins/utxo_signer/src/sign_common.rs +++ b/mm2src/coins/utxo_signer/src/sign_common.rs @@ -105,7 +105,7 @@ pub(crate) fn script_sig_with_pub(public_key: &PublicKey, fork_id: u32, signatur let builder = Builder::default(); builder .push_data(&script_sig) - .push_data(&public_key.to_vec()) + .push_data(public_key.to_vec().as_slice()) .into_bytes() } diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index 584f930420..ca4ef21506 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -1321,7 +1321,7 @@ pub fn round_to(bd: &BigDecimal, places: u8) -> String { if pos < dot { //println! ("{}, pos < dot, stopping at pos {}", bds, pos); - let mut integer: i64 = (&bds[0..=pos]).parse().unwrap(); + let mut integer: i64 = (bds[0..=pos]).parse().unwrap(); if prev_digit > 5 { if bda[0] == b'-' { integer = integer.checked_sub(1).unwrap() diff --git a/mm2src/gossipsub/src/behaviour.rs b/mm2src/gossipsub/src/behaviour.rs index c702d907f9..991a29c187 100644 --- a/mm2src/gossipsub/src/behaviour.rs +++ b/mm2src/gossipsub/src/behaviour.rs @@ -849,7 +849,7 @@ impl Gossipsub { .collect(); let mut prunes: Vec = to_prune .remove(peer) - .unwrap_or_else(Vec::new) + .unwrap_or_default() .iter() .map(|topic_hash| GossipsubControlAction::Prune { topic_hash: topic_hash.clone(), @@ -1043,11 +1043,11 @@ impl Gossipsub { } pub fn get_mesh_peers(&self, topic: &TopicHash) -> Vec { - self.mesh.get(topic).cloned().unwrap_or_else(Vec::new) + self.mesh.get(topic).cloned().unwrap_or_default() } pub fn get_topic_peers(&self, topic: &TopicHash) -> Vec { - self.topic_peers.get(topic).cloned().unwrap_or_else(Vec::new) + self.topic_peers.get(topic).cloned().unwrap_or_default() } pub fn get_num_peers(&self) -> usize { self.peer_topics.len() } diff --git a/mm2src/lp_ordermatch.rs b/mm2src/lp_ordermatch.rs index 1a0d909ece..7e6a09ad23 100644 --- a/mm2src/lp_ordermatch.rs +++ b/mm2src/lp_ordermatch.rs @@ -3435,6 +3435,7 @@ pub struct TakerRequestForRpc<'a> { conf_settings: &'a Option, } +#[allow(clippy::needless_borrow)] pub async fn lp_auto_buy( ctx: &MmArc, base_coin: &MmCoinEnum, @@ -3927,6 +3928,7 @@ struct MakerMatchForRpc<'a> { last_updated: u64, } +#[allow(clippy::needless_borrow)] impl<'a> From<&'a MakerMatch> for MakerMatchForRpc<'a> { fn from(maker_match: &'a MakerMatch) -> MakerMatchForRpc { MakerMatchForRpc { @@ -4654,6 +4656,7 @@ struct TakerMatchForRpc<'a> { last_updated: u64, } +#[allow(clippy::needless_borrow)] impl<'a> From<&'a TakerMatch> for TakerMatchForRpc<'a> { fn from(taker_match: &'a TakerMatch) -> TakerMatchForRpc { TakerMatchForRpc { @@ -4676,6 +4679,7 @@ struct TakerOrderForRpc<'a> { rel_orderbook_ticker: &'a Option, } +#[allow(clippy::needless_borrow)] impl<'a> From<&'a TakerOrder> for TakerOrderForRpc<'a> { fn from(order: &'a TakerOrder) -> TakerOrderForRpc { TakerOrderForRpc { From 2368209a59f891c41e19f0fd40d531febbe5dd85 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Feb 2022 21:26:34 +0100 Subject: [PATCH 30/74] feat(fmt): rust fmt --- mm2src/gossipsub/src/behaviour.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mm2src/gossipsub/src/behaviour.rs b/mm2src/gossipsub/src/behaviour.rs index 991a29c187..b308c37ba8 100644 --- a/mm2src/gossipsub/src/behaviour.rs +++ b/mm2src/gossipsub/src/behaviour.rs @@ -1042,9 +1042,7 @@ impl Gossipsub { } } - pub fn get_mesh_peers(&self, topic: &TopicHash) -> Vec { - self.mesh.get(topic).cloned().unwrap_or_default() - } + pub fn get_mesh_peers(&self, topic: &TopicHash) -> Vec { self.mesh.get(topic).cloned().unwrap_or_default() } pub fn get_topic_peers(&self, topic: &TopicHash) -> Vec { self.topic_peers.get(topic).cloned().unwrap_or_default() From 8914b7667b3ea053e0a0b9071bcd4478585ca5e3 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 09:32:54 +0100 Subject: [PATCH 31/74] feat(utxo_block_header): add a box dyn for block header storage to utxo common fields --- mm2src/coins/utxo.rs | 4 ++++ mm2src/coins/utxo/utxo_block_header_storage.rs | 5 +++++ mm2src/coins/utxo/utxo_tests.rs | 1 + 3 files changed, 10 insertions(+) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 8b0ed778e4..18e7a1ac8e 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -97,6 +97,7 @@ use super::{BalanceError, BalanceFut, BalanceResult, CoinTransportMetrics, Coins RpcTransportEventHandler, RpcTransportEventHandlerShared, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, Transaction, TransactionDetails, TransactionEnum, TransactionFut, WithdrawError, WithdrawRequest}; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderStorageError}; use std::array::TryFromSliceError; #[cfg(test)] pub mod utxo_tests; @@ -553,6 +554,7 @@ pub struct UtxoCoinFields { pub tx_fee: TxFee, /// Minimum transaction value at which the value is not less than fee pub dust_amount: u64, + pub block_header_storage: Option>>, /// RPC client pub rpc_client: UtxoRpcClientEnum, /// Either ECDSA key pair or a Hardware Wallet info. @@ -1475,6 +1477,7 @@ pub trait UtxoCoinBuilder { conf, decimals, dust_amount, + block_header_storage: None, rpc_client, priv_key_policy, derivation_method, @@ -1508,6 +1511,7 @@ pub trait UtxoCoinBuilder { conf, decimals, dust_amount, + block_header_storage: None, rpc_client, priv_key_policy: PrivKeyPolicy::HardwareWallet, derivation_method: DerivationMethod::HDWallet(HDWalletInfo::new(addr_format)), diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 31b16873fd..0219bceab4 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use chain::BlockHeader; use common::{mm_error::MmError, mm_error::NotMmError, NotSame}; use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send {} @@ -39,3 +40,7 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { /// Gets the block header by height from the selected coin's storage as hex async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError>; } + +impl Debug for dyn BlockHeaderStorage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "") } +} diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index a64e9f1516..224534188e 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -131,6 +131,7 @@ fn utxo_coin_fields_for_test( tx_cache_directory: None, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_hash_algo: TxHashAlgo::DSHA256, + block_header_storage: None, } } From 1def8071009ef74cd582dab2fe7bc3ce974d4e01 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 09:56:28 +0100 Subject: [PATCH 32/74] Revert "feat(utxo_block_header): add a box dyn for block header storage to utxo common fields" This reverts commit 8914b7667b3ea053e0a0b9071bcd4478585ca5e3. --- mm2src/coins/utxo.rs | 4 ---- mm2src/coins/utxo/utxo_block_header_storage.rs | 5 ----- mm2src/coins/utxo/utxo_tests.rs | 1 - 3 files changed, 10 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 18e7a1ac8e..8b0ed778e4 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -97,7 +97,6 @@ use super::{BalanceError, BalanceFut, BalanceResult, CoinTransportMetrics, Coins RpcTransportEventHandler, RpcTransportEventHandlerShared, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, Transaction, TransactionDetails, TransactionEnum, TransactionFut, WithdrawError, WithdrawRequest}; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderStorageError}; use std::array::TryFromSliceError; #[cfg(test)] pub mod utxo_tests; @@ -554,7 +553,6 @@ pub struct UtxoCoinFields { pub tx_fee: TxFee, /// Minimum transaction value at which the value is not less than fee pub dust_amount: u64, - pub block_header_storage: Option>>, /// RPC client pub rpc_client: UtxoRpcClientEnum, /// Either ECDSA key pair or a Hardware Wallet info. @@ -1477,7 +1475,6 @@ pub trait UtxoCoinBuilder { conf, decimals, dust_amount, - block_header_storage: None, rpc_client, priv_key_policy, derivation_method, @@ -1511,7 +1508,6 @@ pub trait UtxoCoinBuilder { conf, decimals, dust_amount, - block_header_storage: None, rpc_client, priv_key_policy: PrivKeyPolicy::HardwareWallet, derivation_method: DerivationMethod::HDWallet(HDWalletInfo::new(addr_format)), diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 0219bceab4..31b16873fd 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -3,7 +3,6 @@ use async_trait::async_trait; use chain::BlockHeader; use common::{mm_error::MmError, mm_error::NotMmError, NotSame}; use std::collections::HashMap; -use std::fmt::{Debug, Formatter}; pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send {} @@ -40,7 +39,3 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { /// Gets the block header by height from the selected coin's storage as hex async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError>; } - -impl Debug for dyn BlockHeaderStorage { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "") } -} diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 224534188e..a64e9f1516 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -131,7 +131,6 @@ fn utxo_coin_fields_for_test( tx_cache_directory: None, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_hash_algo: TxHashAlgo::DSHA256, - block_header_storage: None, } } From f75aee9341c2cf6383eaa0fb94038b19bfe1aa43 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 10:38:28 +0100 Subject: [PATCH 33/74] feat(utxo): add empty utxo_indexedb_block_header_storage.rs --- mm2src/coins/utxo.rs | 2 + .../utxo_indexedb_block_header_storage.rs | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 8b0ed778e4..e1034e5dcc 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -35,6 +35,8 @@ pub mod utxo_standard; pub mod utxo_withdraw; #[cfg(not(target_arch = "wasm32"))] pub mod tx_cache; +#[cfg(target_arch = "wasm32")] +pub mod utxo_indexedb_block_header_storage; #[cfg(not(target_arch = "wasm32"))] pub mod utxo_sql_block_header_storage; diff --git a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs new file mode 100644 index 0000000000..c746626f56 --- /dev/null +++ b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs @@ -0,0 +1,53 @@ +use crate::utxo::rpc_clients::ElectrumBlockHeader; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; +use async_trait::async_trait; +use chain::BlockHeader; +use common::indexed_db::DbTransactionError; +use common::mm_error::MmError; +use std::collections::HashMap; + +impl BlockHeaderStorageError for DbTransactionError {} + +pub struct IndexedDBBlockHeadersStorage {} + +#[async_trait] +impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { + type Error = DbTransactionError; + + async fn init(&self, _for_coin: &str) -> Result<(), MmError> { todo!() } + + async fn is_initialized_for(&self, _for_coin: &str) -> Result> { todo!() } + + async fn add_electrum_block_headers_to_storage( + &self, + _for_coin: &str, + _headers: Vec, + ) -> Result<(), MmError> { + todo!() + } + + async fn add_block_headers_to_storage( + &self, + _for_coin: &str, + _headers: HashMap, + ) -> Result<(), MmError> { + todo!() + } + + async fn get_block_header( + &self, + _for_coin: &str, + _height: u64, + ) -> Result, MmError> { + todo!() + } + + async fn get_block_header_raw( + &self, + _for_coin: &str, + _height: u64, + ) -> Result, MmError> { + todo!() + } +} From 40e3cf7109af70be9cb5b0fa8822459f26c520b7 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 11:28:03 +0100 Subject: [PATCH 34/74] feat(utxo): start an empty loop for downloading headers --- mm2src/coins/utxo/utxo_common.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index aba1fe7508..13d66dbbc0 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -150,6 +150,10 @@ where info!("Starting UTXO merge loop for coin {}", ticker); spawn(merge_loop); } + + let block_header_loop = block_header_utxo_loop(ctx.clone(), 60.0, utxo_arc.clone()); + info!("Starting UTXO download header loop for coin {}", ticker); + spawn(block_header_loop); Ok(coin) } @@ -3027,6 +3031,13 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } +async fn block_header_utxo_loop(_ctx: MmArc, check_every: f64, coin: UtxoArc) { + loop { + info!("tick block_header_utxo_loop for {}", coin.conf.ticker); + Timer::sleep(check_every).await; + } +} + async fn merge_utxo_loop( weak: UtxoWeak, merge_at: usize, From 4dedf0ba118fcad0cc69d96a3949c491493b8526 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 11:39:26 +0100 Subject: [PATCH 35/74] feat(indexed_db_block_header): do not use todo to prevent crash at runtime --- mm2src/coins/utxo/utxo_common.rs | 2 +- .../coins/utxo/utxo_indexedb_block_header_storage.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 13d66dbbc0..b4ebf8d5a3 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -151,7 +151,7 @@ where spawn(merge_loop); } - let block_header_loop = block_header_utxo_loop(ctx.clone(), 60.0, utxo_arc.clone()); + let block_header_loop = block_header_utxo_loop(ctx.clone(), 60.0, utxo_arc); info!("Starting UTXO download header loop for coin {}", ticker); spawn(block_header_loop); Ok(coin) diff --git a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs index c746626f56..3d405ba7be 100644 --- a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs @@ -15,16 +15,16 @@ pub struct IndexedDBBlockHeadersStorage {} impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { type Error = DbTransactionError; - async fn init(&self, _for_coin: &str) -> Result<(), MmError> { todo!() } + async fn init(&self, _for_coin: &str) -> Result<(), MmError> { Ok(()) } - async fn is_initialized_for(&self, _for_coin: &str) -> Result> { todo!() } + async fn is_initialized_for(&self, _for_coin: &str) -> Result> { Ok(true) } async fn add_electrum_block_headers_to_storage( &self, _for_coin: &str, _headers: Vec, ) -> Result<(), MmError> { - todo!() + Ok(()) } async fn add_block_headers_to_storage( @@ -32,7 +32,7 @@ impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { _for_coin: &str, _headers: HashMap, ) -> Result<(), MmError> { - todo!() + Ok(()) } async fn get_block_header( @@ -40,7 +40,7 @@ impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { _for_coin: &str, _height: u64, ) -> Result, MmError> { - todo!() + Ok(None) } async fn get_block_header_raw( @@ -48,6 +48,6 @@ impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { _for_coin: &str, _height: u64, ) -> Result, MmError> { - todo!() + Ok(None) } } From aab5c80551c32da58d5b0f438732b474441a96f9 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 14:15:34 +0100 Subject: [PATCH 36/74] feat(block_header_loop): add block header loop in arc builder --- .../utxo/utxo_builder/utxo_arc_builder.rs | 36 +++++++++++++++++-- mm2src/coins/utxo/utxo_common.rs | 14 ++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 756df4d813..67b49e0c36 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -2,6 +2,7 @@ use crate::hd_pubkey::HDXPubExtractor; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; +use crate::utxo::utxo_common::block_header_utxo_loop; use crate::utxo::utxo_common::merge_utxo_loop; use crate::utxo::{UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoWeak}; use crate::{PrivKeyBuildPolicy, UtxoActivationParams}; @@ -102,7 +103,8 @@ where let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); - self.spawn_merge_utxo_loop_if_required(utxo_weak, self.constructor.clone()); + self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); + self.spawn_block_header_utxo_loop(self.ctx.clone(), utxo_weak, self.constructor.clone()); Ok(result_coin) } } @@ -115,6 +117,14 @@ where { } +impl<'a, F, T, XPubExtractor> BlockHeaderUtxoArcOps for UtxoArcBuilder<'a, F, T, XPubExtractor> +where + F: Fn(UtxoArc) -> T + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + Send + Sync + 'static, + XPubExtractor: HDXPubExtractor + Send + Sync, +{ +} + pub struct UtxoArcWithIguanaPrivKeyBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Send + Sync + 'static, @@ -152,6 +162,13 @@ where { } +impl<'a, F, T> BlockHeaderUtxoArcOps for UtxoArcWithIguanaPrivKeyBuilder<'a, F, T> +where + F: Fn(UtxoArc) -> T + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + Send + Sync + 'static, +{ +} + #[async_trait] impl<'a, F, T> UtxoCoinWithIguanaPrivKeyBuilder for UtxoArcWithIguanaPrivKeyBuilder<'a, F, T> where @@ -169,7 +186,8 @@ where let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); - self.spawn_merge_utxo_loop_if_required(utxo_weak, self.constructor.clone()); + self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); + self.spawn_block_header_utxo_loop(self.ctx.clone(), utxo_weak, self.constructor.clone()); Ok(result_coin) } } @@ -219,3 +237,17 @@ where } } } + +pub trait BlockHeaderUtxoArcOps: UtxoCoinBuilderCommonOps +where + T: AsRef + UtxoCommonOps + Send + Sync + 'static, +{ + fn spawn_block_header_utxo_loop(&self, ctx: MmArc, weak: UtxoWeak, constructor: F) + where + F: Fn(UtxoArc) -> T + Send + Sync + 'static, + { + let fut = block_header_utxo_loop(ctx, weak, 60.0, constructor); + info!("Starting UTXO block header loop for coin {}", self.ticker()); + spawn(fut); + } +} diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 1ac1d50acf..8631d45b56 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3271,9 +3271,17 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } -async fn block_header_utxo_loop(_ctx: MmArc, check_every: f64, coin: UtxoArc) { - loop { - info!("tick block_header_utxo_loop for {}", coin.conf.ticker); +pub async fn block_header_utxo_loop( + _ctx: MmArc, + weak: UtxoWeak, + check_every: f64, + constructor: impl Fn(UtxoArc) -> T, +) where + T: AsRef + UtxoCommonOps, +{ + while let Some(arc) = weak.upgrade() { + let coin = constructor(arc); + info!("tick block_header_utxo_loop for {}", coin.as_ref().conf.ticker); Timer::sleep(check_every).await; } } From 1db867d2a8cc4a1d4dc4a121851d8afe3cbaa331 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 14:34:59 +0100 Subject: [PATCH 37/74] feat(utxo): add a function to retrieve a storage from the ctx for block header --- mm2src/coins/utxo/utxo_common.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 8631d45b56..f83211a971 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -41,6 +41,11 @@ use std::sync::atomic::Ordering as AtomicOrdering; use utxo_signer::with_key_pair::p2sh_spend; use utxo_signer::UtxoSignerOps; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; +#[cfg(target_arch = "wasm32")] +use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; +#[cfg(not(target_arch = "wasm32"))] +use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; pub use chain::Transaction as UtxoTx; use spv_validation::spv_proof::SPVProof; use spv_validation::types::SPVError; @@ -3271,14 +3276,19 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } -pub async fn block_header_utxo_loop( - _ctx: MmArc, - weak: UtxoWeak, - check_every: f64, - constructor: impl Fn(UtxoArc) -> T, -) where +#[cfg(target_arch = "wasm32")] +fn retrieve_header_storage_from_ctx(_ctx: &MmArc) -> impl BlockHeaderStorage { IndexedDBBlockHeadersStorage {} } + +#[cfg(not(target_arch = "wasm32"))] +fn retrieve_header_storage_from_ctx(ctx: &MmArc) -> impl BlockHeaderStorage { + SqliteBlockHeadersStorage(ctx.sqlite_connection.as_option().unwrap().clone()) +} + +pub async fn block_header_utxo_loop(ctx: MmArc, weak: UtxoWeak, check_every: f64, constructor: impl Fn(UtxoArc) -> T) +where T: AsRef + UtxoCommonOps, { + let _storage = retrieve_header_storage_from_ctx(&ctx); while let Some(arc) = weak.upgrade() { let coin = constructor(arc); info!("tick block_header_utxo_loop for {}", coin.as_ref().conf.ticker); From 03e937b631c41fabd5049a8294670d64363c4823 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Feb 2022 16:37:16 +0100 Subject: [PATCH 38/74] feat(utxo): start logic of block header downloading, very WIP --- mm2src/coins/utxo/rpc_clients.rs | 2 +- mm2src/coins/utxo/utxo_common.rs | 99 ++++++++++++++++++- .../spv_validation/src/helpers_validation.rs | 3 + 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index c891280116..8bc8934c8e 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1023,7 +1023,7 @@ impl Into for ElectrumNonce { #[derive(Debug, Deserialize)] pub struct ElectrumBlockHeadersRes { - count: u64, + pub count: u64, pub hex: BytesJson, #[allow(dead_code)] max: u64, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index f83211a971..7db6b5d691 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3,8 +3,8 @@ use crate::coin_balance::{AddressBalanceStatus, HDAddressBalance, HDWalletBalanc use crate::hd_pubkey::{ExtractExtendedPubkey, HDExtractPubkeyError, HDXPubExtractor}; use crate::hd_wallet::{AddressDerivingError, HDAccountMut, NewAccountCreatingError}; use crate::init_withdraw::WithdrawTaskHandle; -use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UtxoRpcClientEnum, - UtxoRpcClientOps, UtxoRpcResult}; +use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, ElectrumBlockHeader, UnspentInfo, + UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, TradePreimageValue, TxFeeDetails, ValidateAddressResult, WithdrawFrom, WithdrawResult, @@ -32,7 +32,7 @@ use rpc::v1::types::{Bytes as BytesJson, TransactionInputEnum, H256 as H256Json} use script::{Builder, Opcode, Script, ScriptAddress, TransactionInputSigner, UnsignedTransactionInput}; use secp256k1::{PublicKey, Signature}; use serde_json::{self as json}; -use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, +use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, CompactInteger, Reader, SERIALIZE_TRANSACTION_WITNESS}; use std::cmp::Ordering; use std::collections::hash_map::{Entry, HashMap}; @@ -47,6 +47,7 @@ use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorag #[cfg(not(target_arch = "wasm32"))] use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; pub use chain::Transaction as UtxoTx; +use spv_validation::helpers_validation::validate_headers; use spv_validation::spv_proof::SPVProof; use spv_validation::types::SPVError; @@ -3284,11 +3285,101 @@ fn retrieve_header_storage_from_ctx(ctx: &MmArc) -> impl BlockHeaderStorage { SqliteBlockHeadersStorage(ctx.sqlite_connection.as_option().unwrap().clone()) } +async fn get_best_block(rpc: &ElectrumClient) -> Result> { + rpc.blockchain_headers_subscribe() + .compat() + .await + .map_err(|_e| MmError::new(SPVError::UnableToGetHeader)) +} + +pub async fn retrieve_last_headers( + coin: &T, +) -> Result<(HashMap, Vec), MmError> +where + T: AsRef + UtxoCommonOps, +{ + let electrum_rpc_client = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => return MmError::err(SPVError::UnknownError), + UtxoRpcClientEnum::Electrum(electrum) => electrum, + }; + let best_block = get_best_block(electrum_rpc_client).await; + let (from, count) = match best_block { + Ok(block) => { + let block_height = block.block_height(); + let from = if block_height < 100 { 0 } else { block_height - 100 }; + (from, NonZeroU64::new(100).unwrap()) + }, + Err(_) => { + error!("Unable to retrieve last block height - skipping"); + return MmError::err(SPVError::UnableToGetHeader); + }, + }; + let headers_resp = electrum_rpc_client.blockchain_block_headers(from, count).compat().await; + let (block_registry, block_headers) = match headers_resp { + Ok(headers) => { + if headers.count == 0 { + return MmError::err(SPVError::UnableToGetHeader); + } + let len = CompactInteger::from(headers.count); + let mut serialized = serialize(&len).take(); + serialized.extend(headers.hex.0.into_iter()); + let mut reader = Reader::new_with_coin_variant(serialized.as_slice(), CoinVariant::Standard); + let maybe_block_headers = reader.read_list::(); + let block_headers = match maybe_block_headers { + Ok(headers) => headers, + Err(_) => return MmError::err(SPVError::MalformattedHeader), + }; + let mut block_registry: HashMap = HashMap::new(); + let mut starting_height = from; + for block_header in &block_headers { + block_registry.insert(starting_height, block_header.clone()); + starting_height += 1; + } + (block_registry, block_headers) + }, + Err(_) => return MmError::err(SPVError::UnableToGetHeader), + }; + Ok((block_registry, block_headers)) +} + pub async fn block_header_utxo_loop(ctx: MmArc, weak: UtxoWeak, check_every: f64, constructor: impl Fn(UtxoArc) -> T) where T: AsRef + UtxoCommonOps, { - let _storage = retrieve_header_storage_from_ctx(&ctx); + let coin = match weak.upgrade() { + Some(arc) => constructor(arc), + None => return, + }; + match retrieve_last_headers(&coin).await { + Ok((block_registry, block_headers)) => { + if validate_headers(block_headers) { + let ticker = coin.as_ref().conf.ticker.as_str(); + let storage = retrieve_header_storage_from_ctx(&ctx); + match storage.is_initialized_for(ticker).await { + Ok(is_init) => { + if !is_init { + match storage.init(ticker).await { + Ok(()) => {}, + Err(e) => { + error!("{:?}", e); + return; + }, + } + } + }, + Err(_e) => return, + } + let _ = storage + .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), block_registry) + .await; + } + }, + Err(err) => { + error!("error: {:?}", err); + return; + }, + } + drop(coin); while let Some(arc) = weak.upgrade() { let coin = constructor(arc); info!("tick block_header_utxo_loop for {}", coin.as_ref().conf.ticker); diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index de51c0e515..e355b8e9af 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -1,4 +1,5 @@ use bitcoin_spv::btcspv::verify_hash256_merkle; +use chain::BlockHeader; use primitives::hash::H256; use types::SPVError; @@ -29,6 +30,8 @@ pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec Ok(()) } +pub fn validate_headers(_headers: Vec) -> bool { true } + #[cfg(test)] mod tests { use super::*; From 93a6505cca8c95ad2a4c79ec0401ac00c98c61ed Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 23 Feb 2022 10:13:58 +0100 Subject: [PATCH 39/74] feat(header_validation): add partial header validation + params for enabling --- mm2src/coins/utxo.rs | 14 +++ .../utxo/utxo_builder/utxo_arc_builder.rs | 35 ++++-- mm2src/coins/utxo/utxo_common.rs | 115 ++++++++++-------- .../spv_validation/src/helpers_validation.rs | 16 ++- 4 files changed, 118 insertions(+), 62 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 9cf5f2a81c..05ad433ea9 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -1021,10 +1021,18 @@ pub struct UtxoMergeParams { pub max_merge_at_once: usize, } +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct UtxoBlockHeaderVerificationParams { + pub difficulty_check: bool, + pub blocks_limit_to_check: u64, + pub check_every: f64, +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UtxoActivationParams { pub mode: UtxoRpcMode, pub utxo_merge_params: Option, + pub utxo_block_header_verification_params: Option, #[serde(default)] pub tx_history: bool, pub required_confirmations: Option, @@ -1041,6 +1049,7 @@ pub enum UtxoFromLegacyReqErr { UnexpectedMethod, InvalidElectrumServers(json::Error), InvalidMergeParams(json::Error), + InvalidBlockHeaderVerificationParams(json::Error), InvalidRequiredConfs(json::Error), InvalidRequiresNota(json::Error), InvalidAddressFormat(json::Error), @@ -1061,6 +1070,9 @@ impl UtxoActivationParams { let utxo_merge_params = json::from_value(req["utxo_merge_params"].clone()).map_to_mm(UtxoFromLegacyReqErr::InvalidMergeParams)?; + let utxo_block_header_verification_params = json::from_value(req["block_header_params"].clone()) + .map_to_mm(UtxoFromLegacyReqErr::InvalidBlockHeaderVerificationParams)?; + let tx_history = req["tx_history"].as_bool().unwrap_or_default(); let required_confirmations = json::from_value(req["required_confirmations"].clone()) .map_to_mm(UtxoFromLegacyReqErr::InvalidRequiredConfs)?; @@ -1073,6 +1085,7 @@ impl UtxoActivationParams { Ok(UtxoActivationParams { mode, + utxo_block_header_verification_params, utxo_merge_params, tx_history, required_confirmations, @@ -1437,6 +1450,7 @@ pub fn address_by_conf_and_pubkey_str( let params = UtxoActivationParams { mode: UtxoRpcMode::Native, utxo_merge_params: None, + utxo_block_header_verification_params: None, tx_history: false, required_confirmations: None, requires_notarization: None, diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 67b49e0c36..c2c94e782a 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -5,7 +5,7 @@ use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBui use crate::utxo::utxo_common::block_header_utxo_loop; use crate::utxo::utxo_common::merge_utxo_loop; use crate::utxo::{UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoWeak}; -use crate::{PrivKeyBuildPolicy, UtxoActivationParams}; +use crate::{MarketCoinOps, PrivKeyBuildPolicy, UtxoActivationParams}; use async_trait::async_trait; use common::executor::spawn; use common::log::info; @@ -87,7 +87,7 @@ where impl<'a, F, T, XPubExtractor> UtxoCoinBuilder for UtxoArcBuilder<'a, F, T, XPubExtractor> where F: Fn(UtxoArc) -> T + Clone + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, XPubExtractor: HDXPubExtractor + Send + Sync, { type ResultCoin = T; @@ -104,7 +104,7 @@ where let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop(self.ctx.clone(), utxo_weak, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required(self.ctx.clone(), utxo_weak, self.constructor.clone()); Ok(result_coin) } } @@ -120,7 +120,7 @@ where impl<'a, F, T, XPubExtractor> BlockHeaderUtxoArcOps for UtxoArcBuilder<'a, F, T, XPubExtractor> where F: Fn(UtxoArc) -> T + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, XPubExtractor: HDXPubExtractor + Send + Sync, { } @@ -165,7 +165,7 @@ where impl<'a, F, T> BlockHeaderUtxoArcOps for UtxoArcWithIguanaPrivKeyBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, { } @@ -173,7 +173,7 @@ where impl<'a, F, T> UtxoCoinWithIguanaPrivKeyBuilder for UtxoArcWithIguanaPrivKeyBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Clone + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, { type ResultCoin = T; type Error = UtxoCoinBuildError; @@ -187,7 +187,7 @@ where let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop(self.ctx.clone(), utxo_weak, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required(self.ctx.clone(), utxo_weak, self.constructor.clone()); Ok(result_coin) } } @@ -240,14 +240,25 @@ where pub trait BlockHeaderUtxoArcOps: UtxoCoinBuilderCommonOps where - T: AsRef + UtxoCommonOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, { - fn spawn_block_header_utxo_loop(&self, ctx: MmArc, weak: UtxoWeak, constructor: F) + fn spawn_block_header_utxo_loop_if_required(&self, ctx: MmArc, weak: UtxoWeak, constructor: F) where F: Fn(UtxoArc) -> T + Send + Sync + 'static, { - let fut = block_header_utxo_loop(ctx, weak, 60.0, constructor); - info!("Starting UTXO block header loop for coin {}", self.ticker()); - spawn(fut); + if let Some(ref block_header_verification_params) = + self.activation_params().utxo_block_header_verification_params + { + let fut = block_header_utxo_loop( + ctx, + weak, + block_header_verification_params.check_every, + block_header_verification_params.difficulty_check, + block_header_verification_params.blocks_limit_to_check, + constructor, + ); + info!("Starting UTXO block header loop for coin {}", self.ticker()); + spawn(fut); + } } } diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 7db6b5d691..8cd535f4ed 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3,8 +3,8 @@ use crate::coin_balance::{AddressBalanceStatus, HDAddressBalance, HDWalletBalanc use crate::hd_pubkey::{ExtractExtendedPubkey, HDExtractPubkeyError, HDXPubExtractor}; use crate::hd_wallet::{AddressDerivingError, HDAccountMut, NewAccountCreatingError}; use crate::init_withdraw::WithdrawTaskHandle; -use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, ElectrumBlockHeader, UnspentInfo, - UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; +use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UtxoRpcClientEnum, + UtxoRpcClientOps, UtxoRpcResult}; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, TradePreimageValue, TxFeeDetails, ValidateAddressResult, WithdrawFrom, WithdrawResult, @@ -3285,29 +3285,28 @@ fn retrieve_header_storage_from_ctx(ctx: &MmArc) -> impl BlockHeaderStorage { SqliteBlockHeadersStorage(ctx.sqlite_connection.as_option().unwrap().clone()) } -async fn get_best_block(rpc: &ElectrumClient) -> Result> { - rpc.blockchain_headers_subscribe() - .compat() - .await - .map_err(|_e| MmError::new(SPVError::UnableToGetHeader)) -} - pub async fn retrieve_last_headers( coin: &T, + blocks_limit_to_check: u64, ) -> Result<(HashMap, Vec), MmError> where - T: AsRef + UtxoCommonOps, + T: AsRef + UtxoCommonOps + MarketCoinOps, { let electrum_rpc_client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => return MmError::err(SPVError::UnknownError), UtxoRpcClientEnum::Electrum(electrum) => electrum, }; - let best_block = get_best_block(electrum_rpc_client).await; + + let best_block = coin.current_block().compat().await; let (from, count) = match best_block { Ok(block) => { - let block_height = block.block_height(); - let from = if block_height < 100 { 0 } else { block_height - 100 }; - (from, NonZeroU64::new(100).unwrap()) + let block_height = block; + let from = if block_height < blocks_limit_to_check { + 0 + } else { + block_height - blocks_limit_to_check + }; + (from, NonZeroU64::new(blocks_limit_to_check).unwrap()) }, Err(_) => { error!("Unable to retrieve last block height - skipping"); @@ -3342,46 +3341,64 @@ where Ok((block_registry, block_headers)) } -pub async fn block_header_utxo_loop(ctx: MmArc, weak: UtxoWeak, check_every: f64, constructor: impl Fn(UtxoArc) -> T) -where - T: AsRef + UtxoCommonOps, +pub async fn block_header_utxo_loop( + ctx: MmArc, + weak: UtxoWeak, + check_every: f64, + difficulty_check: bool, + blocks_limit_to_check: u64, + constructor: impl Fn(UtxoArc) -> T, +) where + T: AsRef + UtxoCommonOps + MarketCoinOps, { - let coin = match weak.upgrade() { - Some(arc) => constructor(arc), - None => return, - }; - match retrieve_last_headers(&coin).await { - Ok((block_registry, block_headers)) => { - if validate_headers(block_headers) { - let ticker = coin.as_ref().conf.ticker.as_str(); - let storage = retrieve_header_storage_from_ctx(&ctx); - match storage.is_initialized_for(ticker).await { - Ok(is_init) => { - if !is_init { - match storage.init(ticker).await { - Ok(()) => {}, - Err(e) => { - error!("{:?}", e); - return; - }, - } - } - }, - Err(_e) => return, + { + let coin = match weak.upgrade() { + Some(arc) => constructor(arc), + None => return, + }; + let ticker = coin.ticker(); + let storage = retrieve_header_storage_from_ctx(&ctx); + match storage.is_initialized_for(ticker).await { + Ok(is_init) => { + if !is_init { + match storage.init(ticker).await { + Ok(()) => { + info!("Block Header Storage successfully initialized for {}", ticker); + }, + Err(e) => { + error!( + "Couldn't initiate storage - aborting the block_header_utxo_loop: {:?}", + e + ); + return; + }, + } + } else { + info!("Block Header Storage already initialized for {}", ticker); } - let _ = storage - .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), block_registry) - .await; - } - }, - Err(err) => { - error!("error: {:?}", err); - return; - }, + }, + Err(_e) => return, + } } - drop(coin); while let Some(arc) = weak.upgrade() { let coin = constructor(arc); + match retrieve_last_headers(&coin, blocks_limit_to_check).await { + Ok((block_registry, block_headers)) => match validate_headers(block_headers, difficulty_check) { + Ok(_) => { + let storage = retrieve_header_storage_from_ctx(&ctx); + let ticker = coin.as_ref().conf.ticker.as_str(); + match storage.add_block_headers_to_storage(ticker, block_registry).await { + Ok(_) => info!( + "Successfully add block header to storage after validation for {}", + ticker + ), + Err(err) => error!("error: {:?}", err), + } + }, + Err(err) => error!("error: {:?}", err), + }, + Err(err) => error!("error: {:?}", err), + } info!("tick block_header_utxo_loop for {}", coin.as_ref().conf.ticker); Timer::sleep(check_every).await; } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index e355b8e9af..033b711cdd 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -30,7 +30,21 @@ pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec Ok(()) } -pub fn validate_headers(_headers: Vec) -> bool { true } +fn validate_header_prev_hash(actual: &H256, to_compare_with: &H256) -> bool { actual == to_compare_with } + +pub fn validate_headers(headers: Vec, difficulty_check: bool) -> Result<(), SPVError> { + let mut previous_hash = H256::default(); + for (i, header) in headers.into_iter().enumerate() { + if i != 0 && !validate_header_prev_hash(&header.previous_header_hash, &previous_hash) { + return Err(SPVError::InvalidChain); + } + if difficulty_check { + todo!() + } + previous_hash = header.hash(); + } + Ok(()) +} #[cfg(test)] mod tests { From de9478c011959ce97a19cf6740a5e8a4d5bd20ed Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 23 Feb 2022 11:37:14 +0100 Subject: [PATCH 40/74] feat(header_validation): add unexpected difficulty change check --- mm2src/coins/utxo.rs | 1 + .../utxo/utxo_builder/utxo_arc_builder.rs | 1 + mm2src/coins/utxo/utxo_common.rs | 29 ++++++++++--------- mm2src/mm2_bitcoin/chain/src/raw_block.rs | 21 ++++++++++++++ .../spv_validation/src/helpers_validation.rs | 21 +++++++++++--- 5 files changed, 56 insertions(+), 17 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 05ad433ea9..fa0eec0eef 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -1024,6 +1024,7 @@ pub struct UtxoMergeParams { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UtxoBlockHeaderVerificationParams { pub difficulty_check: bool, + pub constant_difficulty: bool, pub blocks_limit_to_check: u64, pub check_every: f64, } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index c2c94e782a..8ee9395ebc 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -254,6 +254,7 @@ where weak, block_header_verification_params.check_every, block_header_verification_params.difficulty_check, + block_header_verification_params.constant_difficulty, block_header_verification_params.blocks_limit_to_check, constructor, ); diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 8cd535f4ed..34a27b7a68 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3346,6 +3346,7 @@ pub async fn block_header_utxo_loop( weak: UtxoWeak, check_every: f64, difficulty_check: bool, + constant_difficulty: bool, blocks_limit_to_check: u64, constructor: impl Fn(UtxoArc) -> T, ) where @@ -3383,19 +3384,21 @@ pub async fn block_header_utxo_loop( while let Some(arc) = weak.upgrade() { let coin = constructor(arc); match retrieve_last_headers(&coin, blocks_limit_to_check).await { - Ok((block_registry, block_headers)) => match validate_headers(block_headers, difficulty_check) { - Ok(_) => { - let storage = retrieve_header_storage_from_ctx(&ctx); - let ticker = coin.as_ref().conf.ticker.as_str(); - match storage.add_block_headers_to_storage(ticker, block_registry).await { - Ok(_) => info!( - "Successfully add block header to storage after validation for {}", - ticker - ), - Err(err) => error!("error: {:?}", err), - } - }, - Err(err) => error!("error: {:?}", err), + Ok((block_registry, block_headers)) => { + match validate_headers(block_headers, difficulty_check, constant_difficulty) { + Ok(_) => { + let storage = retrieve_header_storage_from_ctx(&ctx); + let ticker = coin.as_ref().conf.ticker.as_str(); + match storage.add_block_headers_to_storage(ticker, block_registry).await { + Ok(_) => info!( + "Successfully add block header to storage after validation for {}", + ticker + ), + Err(err) => error!("error: {:?}", err), + } + }, + Err(err) => error!("error: {:?}", err), + } }, Err(err) => error!("error: {:?}", err), } diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index 3ce0a4e92e..2f1b8e8e4a 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -1,6 +1,9 @@ use crypto::dhash256; use primitives::bytes::Bytes; use primitives::hash::H256; +use primitives::U256; +use ser::serialize; +use BlockHeader; /// Hex-encoded block #[derive(Default, PartialEq, Clone, Eq, Hash)] @@ -36,6 +39,24 @@ impl RawBlockHeader { root.as_mut().copy_from_slice(&self.0.as_ref()[4..36]); root } + + /// Extract the target from the header + pub fn target(&self) -> U256 { + let mantissa = U256::from_little_endian(&self.0.as_ref()[72..75]); + // We use saturating here to avoid panicking. + // This is safe because it saturates at `0`, which gives an unreachable target of `1` + let exponent = self.0.as_ref()[75].saturating_sub(3); + let offset = U256::from(256_u64).pow(exponent.into()); + + mantissa * offset + } +} + +impl From for RawBlockHeader { + fn from(header: BlockHeader) -> Self { + let bytes = serialize(&header); + RawBlockHeader(bytes) + } } #[cfg(test)] diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index 033b711cdd..ed554b1806 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -1,6 +1,7 @@ use bitcoin_spv::btcspv::verify_hash256_merkle; -use chain::BlockHeader; +use chain::{BlockHeader, RawBlockHeader}; use primitives::hash::H256; +use primitives::U256; use types::SPVError; /// Evaluates a Bitcoin merkle inclusion proof. @@ -32,16 +33,28 @@ pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec fn validate_header_prev_hash(actual: &H256, to_compare_with: &H256) -> bool { actual == to_compare_with } -pub fn validate_headers(headers: Vec, difficulty_check: bool) -> Result<(), SPVError> { +pub fn validate_headers( + headers: Vec, + difficulty_check: bool, + constant_difficulty: bool, +) -> Result<(), SPVError> { let mut previous_hash = H256::default(); + let mut target = U256::default(); for (i, header) in headers.into_iter().enumerate() { - if i != 0 && !validate_header_prev_hash(&header.previous_header_hash, &previous_hash) { + let raw_header = RawBlockHeader::from(header.clone()); + if i == 0 { + target = raw_header.target(); + } + if !constant_difficulty && raw_header.target() != target { + return Err(SPVError::UnexpectedDifficultyChange); + } + if i != 0 && !validate_header_prev_hash(&raw_header.parent(), &previous_hash) { return Err(SPVError::InvalidChain); } if difficulty_check { todo!() } - previous_hash = header.hash(); + previous_hash = raw_header.digest(); } Ok(()) } From c36d9b0183d9bfce835509bad6c7d3a84775e5cc Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 23 Feb 2022 14:52:43 +0100 Subject: [PATCH 41/74] feat(header_validation): add unit test for validate headers --- mm2src/mm2_bitcoin/chain/src/block_header.rs | 7 +++ mm2src/mm2_bitcoin/chain/src/raw_block.rs | 1 - .../spv_validation/src/helpers_validation.rs | 45 +++++++++++++++++-- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/mm2src/mm2_bitcoin/chain/src/block_header.rs b/mm2src/mm2_bitcoin/chain/src/block_header.rs index 95a713accc..0e3b9ad446 100644 --- a/mm2src/mm2_bitcoin/chain/src/block_header.rs +++ b/mm2src/mm2_bitcoin/chain/src/block_header.rs @@ -2,6 +2,7 @@ use compact::Compact; use crypto::dhash256; use hash::H256; use hex::FromHex; +use primitives::U256; use ser::{deserialize, serialize, Deserializable, Reader, Serializable, Stream}; use std::io; use transaction::{deserialize_tx, TxType}; @@ -264,6 +265,12 @@ impl Deserializable for BlockHeader { impl BlockHeader { pub fn hash(&self) -> H256 { dhash256(&serialize(self)) } + pub fn target(&self) -> Result { + match self.bits { + BlockHeaderBits::Compact(compact) => compact.to_u256(), + BlockHeaderBits::U32(nb) => Ok(U256::from(nb)), + } + } } impl From<&'static str> for BlockHeader { diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index 2f1b8e8e4a..f56ad735f4 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -47,7 +47,6 @@ impl RawBlockHeader { // This is safe because it saturates at `0`, which gives an unreachable target of `1` let exponent = self.0.as_ref()[75].saturating_sub(3); let offset = U256::from(256_u64).pow(exponent.into()); - mantissa * offset } } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index ed554b1806..d693b51e0d 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -33,6 +33,16 @@ pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec fn validate_header_prev_hash(actual: &H256, to_compare_with: &H256) -> bool { actual == to_compare_with } +pub fn validate_header_work(digest: H256, target: &U256) -> bool { + let empty = H256::default(); + + if digest == empty { + return false; + } + + U256::from_little_endian(digest.as_slice()) < *target +} + pub fn validate_headers( headers: Vec, difficulty_check: bool, @@ -43,16 +53,23 @@ pub fn validate_headers( for (i, header) in headers.into_iter().enumerate() { let raw_header = RawBlockHeader::from(header.clone()); if i == 0 { - target = raw_header.target(); + target = match header.target() { + Ok(target) => target, + Err(_) => return Err(SPVError::MalformattedHeader), + }; } - if !constant_difficulty && raw_header.target() != target { + let cur_target = match header.target() { + Ok(target) => target, + Err(_) => return Err(SPVError::MalformattedHeader), + }; + if (!constant_difficulty && difficulty_check) && cur_target != target { return Err(SPVError::UnexpectedDifficultyChange); } if i != 0 && !validate_header_prev_hash(&raw_header.parent(), &previous_hash) { return Err(SPVError::InvalidChain); } - if difficulty_check { - todo!() + if difficulty_check && !validate_header_work(raw_header.digest(), &target) { + return Err(SPVError::InsufficientWork); } previous_hash = raw_header.digest(); } @@ -119,4 +136,24 @@ mod tests { let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); assert_eq!(result.is_err(), false); } + + #[test] + fn test_block_headers_no_difficulty_check() { + // morty: 1330480, 1330481, 1330482 + let headers: Vec = vec![ + "04000000bb496ba8d09f8f98b15cdaf5798163bdd70676eb1c8b538f53ab4f83da4a27000db352177c6b5ad2499a906cec33b843fb17fc1ec298cd06c7e7ceb7b62e144232d719d14c15e565c05e84ead95a2f101a1b658ee2f36eb7ca65206e27cfca473de614625be6071f09006c286bc5ec73dd27a09bf687700c06fb04d0b9a063c0aa0746c9db170000fd40050053b27dad1f5a858b78f3154039759e985ed57db10ecb772810d7f158c55083a14b9f2ba26ae9fcb82012186e2528f67c45b7b216a69fe26232ad2d179a141b1b10e4d5f108c7b920b49348f6eef2d70b7f02cb01d8d9992f8f2d7b6608806b10ff329846b188de200aa37c73ac03f6c9b79cf5613c71b7969b4abafdbc1165ad955a049269584c83b36f36a3e9becf2fe81f3b1917475eb13ecfed3813ecc32206078d8c1e2797013dfc6f6a55e06f1c06a07959ef94d53ca0fc81d03cb6f614761156ed4ff1a8e5c9f0b96f3c8c3eeb9a0720cf4ed10397330f49b83439c5083eea1d1785a10d86ca2866d0da4ca746c49118b780c55aa6cd5b4c0491cefa258ecf129307d15e001415b203e89c008f4444b236aa556dbf4f6d05e0c57642cfa142df2f8546f1d37a6b2feaf98496892b41caefbe7dc7bcbb2755752df3dbf00ac1fc558896f14541aea4cc78ec5d00bbe5398fac4a658b1ae3399777f15117c0f3de3c63bc5b3edf6543d172cfc66907f9cf8706e97b14281daeb427801dfb0910743873265ae6bae71dbf22353c321f726e68f747965858f488dd507b7e6adee42509e5720373dce5b111b420c906b0f2cb391cfb9d581e2509da3829d6718469f383e07043694db87db0ce1196449a6c9cd941a8bde507e553c0ca534238dcc93633631926102c87cd0f83720ccff60de8b05b103e086a2c2cb7943f21033a5658235fc52708907e1ea722e726808db0270bf898c51e9dd0745614857783dc11a6dcd7760d4a07ddbd83a2e02b23fa789b79eed22dc411b9b48f71c54f12387065e3ff0638701e0f6a0dd56d0ce395d150b237b60c166352e69b92173b884446d7660f5857458b97c6d4ee54f8a1f60113aff30e54c1f7c572b85dcb7a2419d2f736a9b0a6d99ea549bd74e546251c0b8be7975e9a6d96aa3467b1dc6b024745fdef43b37cf21a657a3247d9adf8c252ef210d9a4e9c7191f698ccc9b10103b8bb811cdcf1a62903786476db8195ffb3cd004c57ad07a7a3c41eee391f66a7697e69409d7a78558720f6a1b9804d72de820b7b6165b8e14a2b1316576022423f22bb82fab16127be7173ddcd43fa7ea5c4474f79321a8c4b792caf12320c3047d026b7d63216a022e83655c2d811d2bd2a559970e9155b979953f9801ce918f690f43f5e3f07f7ce27a6837bf33b2490d9add8549f1e603a750c114bb92740cc3987cb9f948a6229f175a7b577b0b60d885a0a7ef05debe921376a7acdb25eaa8bb72e120e529cd775175012efb454cf41d240a946bf140af20d9a5dbed2e196d91a7ff33c2769f140fa0bb968111e1602221deae8d162e7a471354c2051acb43ec31015aaefa0b08bf1bddbb282e86a1caf45f3b63e4c6427ba9e99aed28ef79711794511511c52daf13b735e02b9833d3467bfd16886606d5555b7cc95ff2fea3b03c82cfe60e8602d9f70a3870f5b755573b955bb300bd3733b5ddf9a61fd3cd281af39520d6dfd8b7e2b165ec91749614a3b5241e2ea12470f91b58cf6163e02dfe79392db70cd17db9497cf59c89ac8377dbd02042f6ed270c8c2bc717623b203b74676890f5f4cd905b25772a25292d76b6f42a094c27eed13793d189e395ed3f28c5731976a7b45184acee45b3cf05a9c62045644dfe39f79cd331e282edae99cea652eb82819415ac2a5c21539cdd636fb835063ace3b6befffaf50bf6866e9b1a2b35037a330faeb18ca1696693dafd26b5f5da8dcd3e50ff09249bdda695f576d25024560b643d873d07293a80fe71998ef6ccd88c0cf9f69326b463c26fe4906faaf454ae68accd7ef3edffefdd2ede23a822a2267332f0791f1c4e6d5ab4661f279f5039b36a4476e56fd5b0461e585ff30a7c661b93f1".into(), + "04000000001f22e1bc88c53b1554f8fdcf261fdb09f4cae6ef5e5032b788515f4a60d30d67d1b35fda68abc05f5af39e5ade224a5312b8dcd1f3629a7ff33355bb7ca93e32d719d14c15e565c05e84ead95a2f101a1b658ee2f36eb7ca65206e27cfca478be6146220bb071f49000b055b22a7a4bbafd6b52efb90f963d5f80126c27e437005fb47720e0000fd4005004d9875d71c540f558813142e263f597243bdd8d8105ff3d1ffd62ae51ccf22729debe510f97ab0631701dbd34b73e570597dc8825be6bd669e693037fb701040c273b44745f4e850c2d8aeca7ccab6ef7f462206a16d75358f2e8fddf9d0dbc6333ff55b1813a37f0ba240bd2d897fbd6cfdb1989ac8f3ec93b15ae4360edf84088ac9a4ea7d3d71290532bb51675e7310be1210aa33c184d693f6f7c15c5be1e89356ae3d663d0c548fceac0974fe4cb6c6559f50643280df9508460fd04f9cde55521b4c6d61c644c6c7b7473f9e39b412e3776f5e47b6c466aaf1dc76ff2114e716eb6b9614d0c93cdc229ec13b07057a7f7446c1aac51ef0950d4361fa2d20f22f29ff490bf6d6a2a267c45d88d3152d9f5291695f2f4fba65ca9763cb4176506c73b8162611b6004af7ec8d1ea55a225cca2576e4ac84ac333b663693a2f19f7786340ad9d2212d576a0b4e7700bd7d60de88940dce1f01481f9c41350eefd7b496218bcf70c4c8922dfd18d666d37d10cb0f14dd38e1225ec179dcab5501a4434674d6f9ff9f23c4df5f445cc2accf43189fc99ac56693df373a4207b0dc991009fae4796fd7e49cea4dd139ee72264dfd47f4e1ad2420d635c7a1f37950d022ffdcccc7651b645db0ba0ce94c18dcc902279b4601806beefe05016f1f85411e6562b584da0854db2e36f602d8c4974d385aee4a01d1132082c8cd7c71443162f7d7487c73d8a46f830f72a0d352d957bef5afc33c4447ef33b2491e28000d1f4687e95ffc2b9532d28ae4c48f8551bf527dbe18c672204495f2bd546566fd5770189e28c2de0974130a492ccd8737a8c6e971d02a23c4f9f27410348d1f666f93385bdc81bad8e9a9d1dbffdfa2609ebae52740b457ecd67a3bf0db02a14f5bdf3e25b35b2d3d303094e46e0e3daef559d9f0e074e512bcaf9fcc9d035083eec16806af8a93d27b4ad46754a425b6a02b1ac22f682e48f214d66b379d7042aa39f2c5f3448d05ca4b6360e162f31f197225f4ad579d69207c666711fb3f6ca814efcf430899360cced1168cd69ec0e809a89cf2cf2015f9f895a3dadd4ced6d94793e98201b1da6a0a5d90be5d06925e3ad60b9227f84b9c3060a6db6e7857d8731f975d4a993abf10d84590da02b114625109d864de070813179b651d528f66036c30a0700ee84fc5e59757a509745b64e76fa3396f3c8b01a7724cd434e6d774dad36be8a73ad29f6859352aa15236e7825947396cb98e26b912b19ddc127590e59200c4334d1d96d7585a0e349b920f2e4e59cdedac911214c42c0894f72c8a7423d7aef3ea5ef9a5b650821f46537c65509ad8dcf6558c16c04f9877c737ff81875d9fbe01d23d37e937444cf257b0b57bc1c2a774f2e2bf5f3b0881be0e2282ba97ef6aad797f8fdb4053da4e478575805c7a93076c09847544a8e89f1cb3838df7870bcf61deb2144c6f6349c966b67545703058f9227965b97835b049538fb428431a8461586b022368626d20e9b6bfdd7232a5cc6a0aa214319cb440c45443a2446d1e17713c0e1049f0fd759d1dbff493302140376cfb153330ed455a043189260cb7d2d90333a37d3584f2d907d0a73dccee299ad14141d60d1409cda688464a13b5dab37476641741717d599a60c0ac84d85869ed449f83933ad30e2591157fd1f07b73ecf26f34e91bc00f1ca86ae34ca8231b372cdc2ed18d463ac42f92859d6f0e2c483dbb23d785f1233db2033458af9d7c1e7029ac5cc33ca7d25b2b49fd71b1ae5f5ce969b6e77333bf5fbb5e6645dd0a4d0c6e82eb534ac264ddbe28513e4b82b3578c1a6cbfaa2522aa50985fe2cce43cf3363eaacca0e09c721fd603d43c3a4fdf8dde0c9ff2c054910b16aeef7c4d86b31".into(), + "04000000fcead9a1b425124f11aa97e0614120ce87bdddcad655672916f9c4564dc057002bd3df07a4602620282b276359529114ba89b59b16bec235d584c3cf5cc6b2d132d719d14c15e565c05e84ead95a2f101a1b658ee2f36eb7ca65206e27cfca47bfe61462d5b9071f1a001daf299c51afbd74fd75a98ba49a6e40ae8ad92b3afdc1cf215fd6190000fd40050044b5e035b02d138a9704f9513c0865f2733b7c09294ee504c155c283f4895559b6ac39828eac98ad393a642330589e8849040f55ce44f8f2197529d0b0ed57ccdda41f1971e153ec28ac5b4eba968741db374104d65ee234580a83bea1c0cdb67b8bc207057486eb1d90e21ba0cd4f5e9fd834821fafc1517c5d1fceb50ba6f6b102a9b4edac46f2359aec795a4e2458f51114a41289634b3b1cf250e3e38f3689f951278dfa7202a7dfe311cc098fd4a8d02c8f8a74e4a5010b18ee2e60578d5e9f1c094433a73f26e6546e20a574fc261baaa79e9910ab86ed607786a1cc88e7de51ff928d434e26eaef1437f7068c743f26d7c0eea6791e869b101fee8ab41b50af6174c5e6b731a1719f31ee3e6529efef49f31665baedc9382e9665278a84467d479f139fc7a8ef66fef9bd2fd17f7779ee315d458f691a290fa7c2179de8bb91a78458c5290d4aa45b163254006800ba2fce7479511f744fd7de96495c39be93413d8b0b187fe092537e1a7646a66a125b33333f6ecd10085e23ad168b24ee7be69d01ea021a39401e4bd41d818499e7174dd9b85542076c78cb89eeec1c190301b4709dbc963d47926e31bb0235ba6a7029d49458150f6491ac9c973b8a2c893258f907baf4bcb7c39f12b900ba2b2382cd5dd84314ee504ade835ad9a1cb13a7f5928a483ebc9415429810fd99893f2f8f83970b8b47143d617e6f9853e4d86ff378be664218f1c32531143e209f171590dd48216fec879a6b9cbf04432bf4f1a3734b69b6a9f1a358a259a0f9082cfb6c1f3d9d2d9e4522ad651ccce565f06b30c1c0b27252270c2f6608cf4f3288a7e7d4b174e646de05341f7db62b00b5ccb295f058d34b87201148828e9b3f7e08f60e100f810be27eb7f4c471cda7621106fe78bc69ec2bd27acabd55dc094b8626913b7d24d9b60939754700f32574a733a195f8b0220d56f6797de0bcd7b80d561896b816586593409f76e85a7a1035f821dee32a02fdbc26bc4cca375bed418b9d678ac589249a1a5a5b24447ee9b42e33f817066caf3d4e17d0347f6acf0cbf426d4df49413b3d12350edec2681ab9cfecd0825ccfb2649a57391d3f153050dfb4350d60e5e464229ddd6e49ece95557b8ef48c18cbffbe9fc8d7700f611a4b33a2a254afcec638c485e36daf0364da7d4302e488db7b6c41297571048cfea5452e324abb9f9e1043e625fd0853b7e03063d1c3a43aa1ee62d45d890b5e4d10640e775cff6852b6d1acd4a503b3ece3b319cbcf33ff9fdf17b8f852d748db1e05af80507f5d0e1bc44444b155d7da20f7f0b4d6d83368c3bb9e1321b39472a8677ea1d3aca43b453d35edca37b7536d19c26b764958b3c7c30f3211d7b7bb7f6a6d7fd7bf2dda6e7d7b1e533556863549bbe1394a3828596f25029b7e30495e1235f084e5edd133bc29fce4f1e5e514eb1d1cb19fd8dfbb0d130fbec4e288f23dae86311ffd6f4afbaacc2ffe1cc8811a455ba6f5659f82515b56c6ac84277bff5bef98fefc74e002e4a11866a417a429541f8a62df4108e4730d3045f92984bcf1ab2f7d03f8bb1767e91791530cd8eec412919e1f2e341e66a1588a8f485f7aa005787af946b9cb10f6685420b7e1663f66374fddc5e70720507ee2134f3b02df042fcf6db4a5bdd74cc5010793634816fe447cc68e076b225cc1ca872929ef246ce356dc8d8964ff6d7119d071eccb6dc37f75b932c44cdc30723b8357a2761c6de6ab2713e6f6a782538cb731b07950d3f459760a00cc0af406d6848014746b02653636f479d952b46fdeff976e1d159ba46ae7363d5b0042d3905a0bda12aaa6eaae1a5a0d55d4c1930aa1c004cd610866853a247239366aa20f8968ea9ca3d5d6d7321a5d0f2c".into() + ]; + assert_eq!(true, validate_headers(headers, false, false).is_ok()); + } + + #[test] + fn test_block_headers_difficulty_check() { + // BTC: 724609, 724610, 724611 + let headers: Vec = vec!["00200020eab6fa183da8f9e4c761b31a67a76fa6a7658eb84c760200000000000000000063cd9585d434ec0db25894ec4b1f03735f10e31709c4395ea67c50c8378f134b972f166278100a17bfd87203".into(), + "0000402045c698413fbe8b5bf10635658d2a1cec72062798e51200000000000000000000869617420a4c95b1d3d6d012419d2b6c199cff9b68dd9a790892a4da8466fb056033166278100a1743ac4d5b".into(), + "0400e02019d733c1fd76a1fa5950de7bee9d80f107276b93a67204000000000000000000a0d1dee718f5f732c041800e9aa2c25e92be3f6de28278545388db8a6ae27df64c37166278100a170a970c19".into()]; + assert_eq!(true, validate_headers(headers, true, true).is_ok()); + } } From 78fd499531047acf92909abe5146a541efe27959 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 23 Feb 2022 14:57:05 +0100 Subject: [PATCH 42/74] feat(header_validation): document the validate_headers function --- .../spv_validation/src/helpers_validation.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index d693b51e0d..d3476bd9f6 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -43,6 +43,23 @@ pub fn validate_header_work(digest: H256, target: &U256) -> bool { U256::from_little_endian(digest.as_slice()) < *target } +/// Checks validity of header chain. +/// Compares the hash of each header to the prevHash in the next header. +/// +/// # Arguments +/// +/// * `headers` - Raw byte array of header chain +/// * `difficulty_check`: Rather the difficulty need to check or not, usefull for chain like Qtum (Pos) +/// or KMD/SmartChain (Difficulty change NN) +/// * `constant_difficulty`: If we do not expect difficulty change (BTC difficulty change every 2016 blocks) +/// use this variable to false when you do not have a chance to use a checkpoint +/// +/// # Errors +/// +/// * Errors if header chain is the wrong length, chain is invalid or insufficient work +/// +/// # Notes +/// Wrapper inspired by `bitcoin_spv::validatespv::validate_header_chain` pub fn validate_headers( headers: Vec, difficulty_check: bool, From db57ce14ecc0db86e2867a560a0603bf21d44304 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 23 Feb 2022 15:01:05 +0100 Subject: [PATCH 43/74] feat(header_validation): use appropriate error for validate_headers --- mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs | 6 +++--- mm2src/mm2_bitcoin/spv_validation/src/types.rs | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index d3476bd9f6..ff2323b445 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -56,7 +56,7 @@ pub fn validate_header_work(digest: H256, target: &U256) -> bool { /// /// # Errors /// -/// * Errors if header chain is the wrong length, chain is invalid or insufficient work +/// * Errors if header chain is invalid, insufficient work, unexpected difficulty change or unable to get a target /// /// # Notes /// Wrapper inspired by `bitcoin_spv::validatespv::validate_header_chain` @@ -72,12 +72,12 @@ pub fn validate_headers( if i == 0 { target = match header.target() { Ok(target) => target, - Err(_) => return Err(SPVError::MalformattedHeader), + Err(_) => return Err(SPVError::UnableToGetTarget), }; } let cur_target = match header.target() { Ok(target) => target, - Err(_) => return Err(SPVError::MalformattedHeader), + Err(_) => return Err(SPVError::UnableToGetTarget), }; if (!constant_difficulty && difficulty_check) && cur_target != target { return Err(SPVError::UnexpectedDifficultyChange); diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index 2fb1c293f1..a7d2631355 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -16,6 +16,8 @@ pub enum SPVError { MalformattedWitnessOutput, /// `extract_hash` could not identify the output type. MalformattedOutput, + /// Unable to get target from block header + UnableToGetTarget, /// Unable to get block header from network or storage UnableToGetHeader, /// Unable to deserialize raw block header from electrum to concrete type From 43b272af2b3ad2c4864948f48be5f4024550db07 Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 24 Feb 2022 09:51:11 +0100 Subject: [PATCH 44/74] feat(storage): rework storage to make it persistent --- mm2src/coins/utxo.rs | 3 + .../coins/utxo/utxo_block_header_storage.rs | 58 +++++-- .../utxo/utxo_builder/utxo_arc_builder.rs | 7 +- .../utxo/utxo_builder/utxo_coin_builder.rs | 3 + mm2src/coins/utxo/utxo_common.rs | 28 +--- .../utxo_indexedb_block_header_storage.rs | 22 ++- .../utxo/utxo_sql_block_header_storage.rs | 142 +++++++++++++----- .../utxo/utxo_wrapper_block_header_storage.rs | 80 ++++++++++ 8 files changed, 259 insertions(+), 84 deletions(-) create mode 100644 mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index fa0eec0eef..32c0846499 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -40,6 +40,8 @@ pub mod utxo_withdraw; pub mod utxo_indexedb_block_header_storage; #[cfg(not(target_arch = "wasm32"))] pub mod utxo_sql_block_header_storage; +pub mod utxo_wrapper_block_header_storage; +use utxo_wrapper_block_header_storage::BlockHeaderStorage; use async_trait::async_trait; use bigdecimal::BigDecimal; @@ -500,6 +502,7 @@ pub struct UtxoCoinFields { pub history_sync_state: Mutex, /// Path to the TX cache directory pub tx_cache_directory: Option, + pub block_headers_storage: BlockHeaderStorage, /// The cache of recently send transactions used to track the spent UTXOs and replace them with new outputs /// The daemon needs some time to update the listunspent list for address which makes it return already spent UTXOs /// This cache helps to prevent UTXO reuse in such cases diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 31b16873fd..f274912765 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,19 +1,47 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; use async_trait::async_trait; use chain::BlockHeader; -use common::{mm_error::MmError, mm_error::NotMmError, NotSame}; +use common::mm_error::MmError; +use derive_more::Display; use std::collections::HashMap; -pub trait BlockHeaderStorageError: std::fmt::Debug + NotMmError + NotSame + Send {} +#[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum BlockHeaderStorageError { + #[display(fmt = "The storage cannot be initialized for {}", _0)] + InitializationError(String), + #[display(fmt = "The storage is not initialized for {} - reason: {}", ticker, reason)] + NotInitializedError { ticker: String, reason: String }, + #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] + AddToStorageError { ticker: String, reason: String }, + #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] + GetFromStorageError { ticker: String, reason: String }, + #[display( + fmt = "Can't retrieve the table from the storage for {} - reason: {}", + ticker, + reason + )] + CantRetrieveTableError { ticker: String, reason: String }, + #[display(fmt = "Can't query from the storage - query: {} - reason: {}", query, reason)] + QueryError { query: String, reason: String }, + #[display(fmt = "Can't retrieve string from row - reason: {}", reason)] + StringRowError { reason: String }, + #[display(fmt = "Can't execute query from storage for {} - reason: {}", ticker, reason)] + ExecutionError { ticker: String, reason: String }, + #[display(fmt = "Can't start a transaction from storage for {} - reason: {}", ticker, reason)] + TransactionError { ticker: String, reason: String }, + #[display(fmt = "Can't commit a transaction from storage for {} - reason: {}", ticker, reason)] + CommitError { ticker: String, reason: String }, + #[display(fmt = "Can't decode/deserialize from storage for {} - reason: {}", ticker, reason)] + DecodeError { ticker: String, reason: String }, +} #[async_trait] -pub trait BlockHeaderStorage: Send + Sync + 'static { - type Error: BlockHeaderStorageError; - +pub trait BlockHeaderStorageOps: Send + Sync + 'static { /// Initializes collection/tables in storage for a specified coin - async fn init(&self, for_coin: &str) -> Result<(), MmError>; + async fn init(&self, for_coin: &str) -> Result<(), MmError>; - async fn is_initialized_for(&self, for_coin: &str) -> Result>; + async fn is_initialized_for(&self, for_coin: &str) -> Result>; // Adds multiple block headers to the selected coin's header storage // Should store it as `TICKER_HEIGHT=hex_string` @@ -22,7 +50,7 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { &self, for_coin: &str, headers: Vec, - ) -> Result<(), MmError>; + ) -> Result<(), MmError>; // Adds multiple block headers to the selected coin's header storage // Should store it as `TICKER_HEIGHT=hex_string` @@ -31,11 +59,19 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { &self, for_coin: &str, headers: HashMap, - ) -> Result<(), MmError>; + ) -> Result<(), MmError>; /// Gets the block header by height from the selected coin's storage as BlockHeader - async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError>; + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError>; /// Gets the block header by height from the selected coin's storage as hex - async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError>; + async fn get_block_header_raw( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError>; } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 8ee9395ebc..37e8d2ac4b 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -104,7 +104,7 @@ where let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required(self.ctx.clone(), utxo_weak, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required(utxo_weak, self.constructor.clone()); Ok(result_coin) } } @@ -187,7 +187,7 @@ where let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required(self.ctx.clone(), utxo_weak, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required(utxo_weak, self.constructor.clone()); Ok(result_coin) } } @@ -242,7 +242,7 @@ pub trait BlockHeaderUtxoArcOps: UtxoCoinBuilderCommonOps where T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, { - fn spawn_block_header_utxo_loop_if_required(&self, ctx: MmArc, weak: UtxoWeak, constructor: F) + fn spawn_block_header_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) where F: Fn(UtxoArc) -> T + Send + Sync + 'static, { @@ -250,7 +250,6 @@ where self.activation_params().utxo_block_header_verification_params { let fut = block_header_utxo_loop( - ctx, weak, block_header_verification_params.check_every, block_header_verification_params.difficulty_check, diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index e3f82ed42b..db575b5a61 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -3,6 +3,7 @@ use crate::hd_wallet::{HDAccountsMap, HDAccountsMutex}; use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumRpcRequest, EstimateFeeMethod, UtxoRpcClientEnum}; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError, UtxoConfResult}; +use crate::utxo::utxo_wrapper_block_header_storage::BlockHeaderStorage; use crate::utxo::{output_script, utxo_common, ElectrumBuilderArgs, ElectrumProtoVerifier, RecentlySpentOutPoints, TxFee, UtxoCoinConf, UtxoCoinFields, UtxoHDAccount, UtxoHDWallet, UtxoRpcMode, DEFAULT_GAP_LIMIT, UTXO_DUST_AMOUNT}; @@ -190,6 +191,7 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { derivation_method, history_sync_state: Mutex::new(initial_history_state), tx_cache_directory, + block_headers_storage: BlockHeaderStorage::new_from_ctx(self.ctx().clone()), recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_fee, tx_hash_algo, @@ -243,6 +245,7 @@ where priv_key_policy: PrivKeyPolicy::HardwareWallet, derivation_method: DerivationMethod::HDWallet(hd_wallet), history_sync_state: Mutex::new(initial_history_state), + block_headers_storage: BlockHeaderStorage::new_from_ctx(self.ctx().clone()), tx_cache_directory, recently_spent_outpoints, tx_fee, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 34a27b7a68..df5ba1d077 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -12,6 +12,7 @@ use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSen use bigdecimal::{BigDecimal, Zero}; pub use bitcrypto::{dhash160, sha256, ChecksumType}; use chain::constants::SEQUENCE_FINAL; +pub use chain::Transaction as UtxoTx; use chain::{BlockHeader, OutPoint, RawBlockHeader, TransactionOutput}; use common::executor::Timer; use common::jsonrpc_client::{JsonRpcError, JsonRpcErrorType}; @@ -34,23 +35,17 @@ use secp256k1::{PublicKey, Signature}; use serde_json::{self as json}; use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, CompactInteger, Reader, SERIALIZE_TRANSACTION_WITNESS}; +use spv_validation::helpers_validation::validate_headers; +use spv_validation::spv_proof::SPVProof; +use spv_validation::types::SPVError; use std::cmp::Ordering; use std::collections::hash_map::{Entry, HashMap}; use std::str::FromStr; use std::sync::atomic::Ordering as AtomicOrdering; +use utxo_block_header_storage::BlockHeaderStorageOps; use utxo_signer::with_key_pair::p2sh_spend; use utxo_signer::UtxoSignerOps; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; -#[cfg(target_arch = "wasm32")] -use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; -#[cfg(not(target_arch = "wasm32"))] -use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; -pub use chain::Transaction as UtxoTx; -use spv_validation::helpers_validation::validate_headers; -use spv_validation::spv_proof::SPVProof; -use spv_validation::types::SPVError; - pub const DEFAULT_FEE_VOUT: usize = 0; pub const DEFAULT_SWAP_TX_SPEND_SIZE: u64 = 305; pub const DEFAULT_SWAP_VOUT: usize = 0; @@ -3277,14 +3272,6 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } -#[cfg(target_arch = "wasm32")] -fn retrieve_header_storage_from_ctx(_ctx: &MmArc) -> impl BlockHeaderStorage { IndexedDBBlockHeadersStorage {} } - -#[cfg(not(target_arch = "wasm32"))] -fn retrieve_header_storage_from_ctx(ctx: &MmArc) -> impl BlockHeaderStorage { - SqliteBlockHeadersStorage(ctx.sqlite_connection.as_option().unwrap().clone()) -} - pub async fn retrieve_last_headers( coin: &T, blocks_limit_to_check: u64, @@ -3342,7 +3329,6 @@ where } pub async fn block_header_utxo_loop( - ctx: MmArc, weak: UtxoWeak, check_every: f64, difficulty_check: bool, @@ -3358,7 +3344,7 @@ pub async fn block_header_utxo_loop( None => return, }; let ticker = coin.ticker(); - let storage = retrieve_header_storage_from_ctx(&ctx); + let storage = &coin.as_ref().block_headers_storage; match storage.is_initialized_for(ticker).await { Ok(is_init) => { if !is_init { @@ -3387,7 +3373,7 @@ pub async fn block_header_utxo_loop( Ok((block_registry, block_headers)) => { match validate_headers(block_headers, difficulty_check, constant_difficulty) { Ok(_) => { - let storage = retrieve_header_storage_from_ctx(&ctx); + let storage = &coin.as_ref().block_headers_storage; let ticker = coin.as_ref().conf.ticker.as_str(); match storage.add_block_headers_to_storage(ticker, block_registry).await { Ok(_) => info!( diff --git a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs index 3d405ba7be..4814aaeb6f 100644 --- a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs @@ -1,29 +1,25 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorageOps; use async_trait::async_trait; use chain::BlockHeader; -use common::indexed_db::DbTransactionError; use common::mm_error::MmError; use std::collections::HashMap; -impl BlockHeaderStorageError for DbTransactionError {} - +#[derive(Debug)] pub struct IndexedDBBlockHeadersStorage {} #[async_trait] -impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { - type Error = DbTransactionError; - - async fn init(&self, _for_coin: &str) -> Result<(), MmError> { Ok(()) } +impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage { + async fn init(&self, _for_coin: &str) -> Result<(), MmError> { Ok(()) } - async fn is_initialized_for(&self, _for_coin: &str) -> Result> { Ok(true) } + async fn is_initialized_for(&self, _for_coin: &str) -> Result> { Ok(true) } async fn add_electrum_block_headers_to_storage( &self, _for_coin: &str, _headers: Vec, - ) -> Result<(), MmError> { + ) -> Result<(), MmError> { Ok(()) } @@ -31,7 +27,7 @@ impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { &self, _for_coin: &str, _headers: HashMap, - ) -> Result<(), MmError> { + ) -> Result<(), MmError> { Ok(()) } @@ -39,7 +35,7 @@ impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { &self, _for_coin: &str, _height: u64, - ) -> Result, MmError> { + ) -> Result, MmError> { Ok(None) } @@ -47,7 +43,7 @@ impl BlockHeaderStorage for IndexedDBBlockHeadersStorage { &self, _for_coin: &str, _height: u64, - ) -> Result, MmError> { + ) -> Result, MmError> { Ok(None) } } diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index de04012fdd..19013d65d6 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -1,10 +1,9 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderStorageError}; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use async_trait::async_trait; use chain::BlockHeader; use common::async_blocking; use common::mm_error::MmError; -use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::Error as SqlError; use db_common::sqlite::rusqlite::{Connection, Row, ToSql, NO_PARAMS}; use db_common::sqlite::validate_table_name; @@ -16,9 +15,12 @@ const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type= fn block_headers_cache_table(ticker: &str) -> String { ticker.to_owned() + "_block_headers_cache" } -fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { +fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { let table_name = block_headers_cache_table(for_coin); - validate_table_name(&table_name)?; + validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() + &table_name @@ -30,25 +32,31 @@ fn create_block_header_cache_table_sql(for_coin: &str) -> Result Result> { +fn insert_block_header_in_cache_sql(for_coin: &str) -> Result> { let table_name = block_headers_cache_table(for_coin); - validate_table_name(&table_name)?; + validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; // We can simply ignore the repetitive attempt to insert the same block_height let sql = "INSERT OR IGNORE INTO ".to_owned() + &table_name + " (block_height, hex) VALUES (?1, ?2);"; Ok(sql) } -fn get_block_header_by_height(for_coin: &str) -> Result> { +fn get_block_header_by_height(for_coin: &str) -> Result> { let table_name = block_headers_cache_table(for_coin); - validate_table_name(&table_name)?; + validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; let sql = "SELECT hex FROM ".to_owned() + &table_name + " WHERE block_height=?1;"; Ok(sql) } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct SqliteBlockHeadersStorage(pub Arc>); fn query_single_row( @@ -56,7 +64,7 @@ fn query_single_row( query: &str, params: P, map_fn: F, -) -> Result, MmError> +) -> Result, MmError> where P: IntoIterator, P::Item: ToSql, @@ -67,32 +75,42 @@ where return Ok(None); } - let result = maybe_result?; + let result = maybe_result.map_err(|e| BlockHeaderStorageError::QueryError { + query: query.to_string(), + reason: e.to_string(), + })?; Ok(Some(result)) } fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } -impl BlockHeaderStorageError for SqlError {} - #[async_trait] -impl BlockHeaderStorage for SqliteBlockHeadersStorage { - type Error = SqlError; - - async fn init(&self, for_coin: &str) -> Result<(), MmError> { +impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { + async fn init(&self, for_coin: &str) -> Result<(), MmError> { let selfi = self.clone(); let sql_cache = create_block_header_cache_table_sql(for_coin)?; + let ticker = for_coin.to_owned(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - conn.execute(&sql_cache, NO_PARAMS).map(|_| ())?; + conn.execute(&sql_cache, NO_PARAMS) + .map(|_| ()) + .map_err(|e| BlockHeaderStorageError::ExecutionError { + ticker, + reason: e.to_string(), + })?; Ok(()) }) .await } - async fn is_initialized_for(&self, for_coin: &str) -> Result> { + async fn is_initialized_for(&self, for_coin: &str) -> Result> { let block_headers_cache_table = block_headers_cache_table(for_coin); - validate_table_name(&block_headers_cache_table)?; + validate_table_name(&block_headers_cache_table).map_err(|e| { + BlockHeaderStorageError::CantRetrieveTableError { + ticker: for_coin.to_string(), + reason: e.to_string(), + } + })?; let selfi = self.clone(); async_blocking(move || { @@ -112,27 +130,47 @@ impl BlockHeaderStorage for SqliteBlockHeadersStorage { &self, for_coin: &str, headers: Vec, - ) -> Result<(), MmError> { + ) -> Result<(), MmError> { let for_coin = for_coin.to_owned(); let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); - let sql_transaction = conn.transaction()?; + let sql_transaction = conn + .transaction() + .map_err(|e| BlockHeaderStorageError::TransactionError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; for header in headers { match header { ElectrumBlockHeader::V12(h) => { let block_hex = h.as_hex(); let block_cache_params = [&h.block_height.to_string(), &block_hex]; - sql_transaction.execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params)?; + sql_transaction + .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) + .map_err(|e| BlockHeaderStorageError::ExecutionError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; }, ElectrumBlockHeader::V14(h) => { let block_hex = format!("{:02x}", h.hex); let block_cache_params = [&h.height.to_string(), &block_hex]; - sql_transaction.execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params)?; + sql_transaction + .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) + .map_err(|e| BlockHeaderStorageError::ExecutionError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; }, } } - sql_transaction.commit()?; + sql_transaction + .commit() + .map_err(|e| BlockHeaderStorageError::CommitError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; Ok(()) }) .await @@ -142,36 +180,64 @@ impl BlockHeaderStorage for SqliteBlockHeadersStorage { &self, for_coin: &str, headers: HashMap, - ) -> Result<(), MmError> { + ) -> Result<(), MmError> { let for_coin = for_coin.to_owned(); let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); - let sql_transaction = conn.transaction()?; + let sql_transaction = conn + .transaction() + .map_err(|e| BlockHeaderStorageError::TransactionError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; for (height, header) in headers { let serialized = serialize(&header); let block_hex = hex::encode(&serialized); let block_cache_params = [&height.to_string(), &block_hex]; - sql_transaction.execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params)?; + sql_transaction + .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) + .map_err(|e| BlockHeaderStorageError::ExecutionError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; } - sql_transaction.commit()?; + sql_transaction + .commit() + .map_err(|e| BlockHeaderStorageError::CommitError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; Ok(()) }) .await } - async fn get_block_header(&self, for_coin: &str, height: u64) -> Result, MmError> { + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError> { if let Some(header_raw) = self.get_block_header_raw(for_coin, height).await? { - let header_bytes = - hex::decode(header_raw).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e)))?; - let header: BlockHeader = deserialize(header_bytes.as_slice()) - .map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e)))?; + let header_bytes = hex::decode(header_raw).map_err(|e| BlockHeaderStorageError::DecodeError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; + let header: BlockHeader = + deserialize(header_bytes.as_slice()).map_err(|e| BlockHeaderStorageError::DecodeError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; return Ok(Some(header)); } Ok(None) } - async fn get_block_header_raw(&self, for_coin: &str, height: u64) -> Result, MmError> { + async fn get_block_header_raw( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError> { let params = [height.to_string()]; let sql = get_block_header_by_height(for_coin)?; let selfi = self.clone(); @@ -181,6 +247,12 @@ impl BlockHeaderStorage for SqliteBlockHeadersStorage { query_single_row(&conn, &sql, params, string_from_row) }) .await + .map_err(|e| { + MmError::new(BlockHeaderStorageError::GetFromStorageError { + ticker: for_coin.to_string(), + reason: e.into_inner().to_string(), + }) + }) } } diff --git a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs new file mode 100644 index 0000000000..616577a091 --- /dev/null +++ b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs @@ -0,0 +1,80 @@ +use crate::utxo::rpc_clients::ElectrumBlockHeader; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; +#[cfg(target_arch = "wasm32")] +use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; +#[cfg(not(target_arch = "wasm32"))] +use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; +use async_trait::async_trait; +use chain::BlockHeader; +use common::mm_ctx::MmArc; +use common::mm_error::MmError; +use std::collections::HashMap; + +#[derive(Debug)] +pub struct BlockHeaderStorage { + #[cfg(not(target_arch = "wasm32"))] + inner: SqliteBlockHeadersStorage, + #[cfg(target_arch = "wasm32")] + inner: IndexedDBBlockHeadersStorage, +} + +impl BlockHeaderStorage { + #[cfg(not(target_arch = "wasm32"))] + pub fn new_from_ctx(ctx: MmArc) -> BlockHeaderStorage { + BlockHeaderStorage { + inner: SqliteBlockHeadersStorage(ctx.sqlite_connection.as_option().unwrap().clone()), + } + } + + #[cfg(target_arch = "wasm32")] + pub fn new_from_ctx(_ctx: MmArc) -> BlockHeaderStorage { + BlockHeaderStorage { + inner: IndexedDBBlockHeadersStorage {}, + } + } +} + +#[async_trait] +impl BlockHeaderStorageOps for BlockHeaderStorage { + async fn init(&self, for_coin: &str) -> Result<(), MmError> { + self.inner.init(for_coin).await + } + + async fn is_initialized_for(&self, for_coin: &str) -> Result> { + self.inner.is_initialized_for(for_coin).await + } + + async fn add_electrum_block_headers_to_storage( + &self, + for_coin: &str, + headers: Vec, + ) -> Result<(), MmError> { + self.inner + .add_electrum_block_headers_to_storage(for_coin, headers) + .await + } + + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: HashMap, + ) -> Result<(), MmError> { + self.inner.add_block_headers_to_storage(for_coin, headers).await + } + + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError> { + self.inner.get_block_header(for_coin, height).await + } + + async fn get_block_header_raw( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError> { + self.inner.get_block_header_raw(for_coin, height).await + } +} From 7262e3f877df7e3725c8cfbacdb026623557fecd Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 24 Feb 2022 10:13:10 +0100 Subject: [PATCH 45/74] feat(storage): use the block header storage as optional and fix unit tests --- mm2src/coins/utxo.rs | 2 +- mm2src/coins/utxo/utxo_common.rs | 10 ++++++++-- mm2src/coins/utxo/utxo_tests.rs | 1 + .../utxo/utxo_wrapper_block_header_storage.rs | 14 +++++++------- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 32c0846499..7a3270d7af 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -502,7 +502,7 @@ pub struct UtxoCoinFields { pub history_sync_state: Mutex, /// Path to the TX cache directory pub tx_cache_directory: Option, - pub block_headers_storage: BlockHeaderStorage, + pub block_headers_storage: Option, /// The cache of recently send transactions used to track the spent UTXOs and replace them with new outputs /// The daemon needs some time to update the listunspent list for address which makes it return already spent UTXOs /// This cache helps to prevent UTXO reuse in such cases diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index df5ba1d077..08a6ce66dc 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3344,7 +3344,10 @@ pub async fn block_header_utxo_loop( None => return, }; let ticker = coin.ticker(); - let storage = &coin.as_ref().block_headers_storage; + let storage = match &coin.as_ref().block_headers_storage { + None => return, + Some(storage) => storage, + }; match storage.is_initialized_for(ticker).await { Ok(is_init) => { if !is_init { @@ -3373,7 +3376,10 @@ pub async fn block_header_utxo_loop( Ok((block_registry, block_headers)) => { match validate_headers(block_headers, difficulty_check, constant_difficulty) { Ok(_) => { - let storage = &coin.as_ref().block_headers_storage; + let storage = match &coin.as_ref().block_headers_storage { + None => break, + Some(storage) => storage, + }; let ticker = coin.as_ref().conf.ticker.as_str(); match storage.add_block_headers_to_storage(ticker, block_registry).await { Ok(_) => info!( diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 3f39488eae..ef0a318129 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -140,6 +140,7 @@ fn utxo_coin_fields_for_test( derivation_method, history_sync_state: Mutex::new(HistorySyncState::NotEnabled), tx_cache_directory: None, + block_headers_storage: None, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_hash_algo: TxHashAlgo::DSHA256, check_utxo_maturity: false, diff --git a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs index 616577a091..1c5edfe414 100644 --- a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs @@ -20,17 +20,17 @@ pub struct BlockHeaderStorage { impl BlockHeaderStorage { #[cfg(not(target_arch = "wasm32"))] - pub fn new_from_ctx(ctx: MmArc) -> BlockHeaderStorage { - BlockHeaderStorage { - inner: SqliteBlockHeadersStorage(ctx.sqlite_connection.as_option().unwrap().clone()), - } + pub fn new_from_ctx(ctx: MmArc) -> Option { + ctx.sqlite_connection.as_option().map(|connection| BlockHeaderStorage { + inner: SqliteBlockHeadersStorage(connection.clone()), + }) } #[cfg(target_arch = "wasm32")] - pub fn new_from_ctx(_ctx: MmArc) -> BlockHeaderStorage { - BlockHeaderStorage { + pub fn new_from_ctx(_ctx: MmArc) -> Option { + Some(BlockHeaderStorage { inner: IndexedDBBlockHeadersStorage {}, - } + }) } } From f0aeecbf87145c64544bab3acfc57814edadcf69 Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 24 Feb 2022 12:27:34 +0100 Subject: [PATCH 46/74] feat(storage): make the conf into coins settings --- mm2src/coins/utxo.rs | 7 +------ .../utxo/utxo_builder/utxo_arc_builder.rs | 20 +++++++++++-------- .../utxo/utxo_builder/utxo_conf_builder.rs | 2 ++ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 7a3270d7af..d2697d03fe 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -467,6 +467,7 @@ pub struct UtxoCoinConf { /// Block count for median time past calculation pub mtp_block_count: NonZeroU64, pub estimate_fee_mode: Option, + pub block_header_storage_params: Option, /// The minimum number of confirmations at which a transaction is considered mature pub mature_confirmations: u32, /// The number of blocks used for estimate_fee/estimate_smart_fee RPC calls @@ -1036,7 +1037,6 @@ pub struct UtxoBlockHeaderVerificationParams { pub struct UtxoActivationParams { pub mode: UtxoRpcMode, pub utxo_merge_params: Option, - pub utxo_block_header_verification_params: Option, #[serde(default)] pub tx_history: bool, pub required_confirmations: Option, @@ -1074,9 +1074,6 @@ impl UtxoActivationParams { let utxo_merge_params = json::from_value(req["utxo_merge_params"].clone()).map_to_mm(UtxoFromLegacyReqErr::InvalidMergeParams)?; - let utxo_block_header_verification_params = json::from_value(req["block_header_params"].clone()) - .map_to_mm(UtxoFromLegacyReqErr::InvalidBlockHeaderVerificationParams)?; - let tx_history = req["tx_history"].as_bool().unwrap_or_default(); let required_confirmations = json::from_value(req["required_confirmations"].clone()) .map_to_mm(UtxoFromLegacyReqErr::InvalidRequiredConfs)?; @@ -1089,7 +1086,6 @@ impl UtxoActivationParams { Ok(UtxoActivationParams { mode, - utxo_block_header_verification_params, utxo_merge_params, tx_history, required_confirmations, @@ -1454,7 +1450,6 @@ pub fn address_by_conf_and_pubkey_str( let params = UtxoActivationParams { mode: UtxoRpcMode::Native, utxo_merge_params: None, - utxo_block_header_verification_params: None, tx_history: false, required_confirmations: None, requires_notarization: None, diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 37e8d2ac4b..6c5fb1290d 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -4,7 +4,7 @@ use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBui UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::block_header_utxo_loop; use crate::utxo::utxo_common::merge_utxo_loop; -use crate::utxo::{UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoWeak}; +use crate::utxo::{UtxoArc, UtxoBlockHeaderVerificationParams, UtxoCoinFields, UtxoCommonOps, UtxoWeak}; use crate::{MarketCoinOps, PrivKeyBuildPolicy, UtxoActivationParams}; use async_trait::async_trait; use common::executor::spawn; @@ -99,12 +99,13 @@ where async fn build(self) -> MmResult { let utxo = self.build_utxo_fields().await?; + let utxo_block_header_params = utxo.conf.block_header_storage_params.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required(utxo_weak, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required(utxo_weak, utxo_block_header_params, self.constructor.clone()); Ok(result_coin) } } @@ -182,12 +183,13 @@ where async fn build(self) -> MmResult { let utxo = self.build_utxo_fields_with_iguana_priv_key(self.priv_key()).await?; + let utxo_block_header_params = utxo.conf.block_header_storage_params.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required(utxo_weak, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required(utxo_weak, utxo_block_header_params, self.constructor.clone()); Ok(result_coin) } } @@ -242,13 +244,15 @@ pub trait BlockHeaderUtxoArcOps: UtxoCoinBuilderCommonOps where T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, { - fn spawn_block_header_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) - where + fn spawn_block_header_utxo_loop_if_required( + &self, + weak: UtxoWeak, + utxo_header_params: Option, + constructor: F, + ) where F: Fn(UtxoArc) -> T + Send + Sync + 'static, { - if let Some(ref block_header_verification_params) = - self.activation_params().utxo_block_header_verification_params - { + if let Some(block_header_verification_params) = utxo_header_params { let fut = block_header_utxo_loop( weak, block_header_verification_params.check_every, diff --git a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs index baac34ea71..bbcd6dc7f7 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs @@ -93,6 +93,7 @@ impl<'a> UtxoConfBuilder<'a> { let is_pos = self.is_pos(); let segwit = self.segwit(); let force_min_relay_fee = self.conf["force_min_relay_fee"].as_bool().unwrap_or(false); + let block_header_storage_params = json::from_value(self.conf["block_header_params"].clone()).unwrap_or(None); let mtp_block_count = self.mtp_block_count(); let estimate_fee_mode = self.estimate_fee_mode(); let estimate_fee_blocks = self.estimate_fee_blocks(); @@ -126,6 +127,7 @@ impl<'a> UtxoConfBuilder<'a> { force_min_relay_fee, mtp_block_count, estimate_fee_mode, + block_header_storage_params, mature_confirmations, estimate_fee_blocks, lightning, From 7a4c689568123e4adee6487f11e22631829c572f Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 28 Feb 2022 09:39:18 +0100 Subject: [PATCH 47/74] feat(storage): implement header from storage or rpc + within validation --- mm2src/coins/utxo.rs | 27 ++++++ mm2src/coins/utxo/utxo_common.rs | 87 ++++++++++++++++--- mm2src/mm2_bitcoin/chain/src/block_header.rs | 2 + .../mm2_bitcoin/spv_validation/src/types.rs | 2 + 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index eeeac04ede..0eddac186a 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -75,6 +75,7 @@ use rpc::v1::types::{Bytes as BytesJson, Transaction as RpcTransaction, H256 as use script::{Builder, Script, SignatureVersion, TransactionInputSigner}; use serde_json::{self as json, Value as Json}; use serialization::{serialize, serialize_with_flags, SERIALIZE_TRANSACTION_WITNESS}; +use spv_validation::types::SPVError; use std::array::TryFromSliceError; use std::collections::{HashMap, HashSet}; use std::convert::TryInto; @@ -100,6 +101,7 @@ use super::{BalanceError, BalanceFut, BalanceResult, CoinsContext, DerivationMet Transaction, TransactionDetails, TransactionEnum, TransactionFut, WithdrawError, WithdrawRequest}; use crate::coin_balance::HDAddressBalanceChecker; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; #[cfg(test)] pub mod utxo_tests; #[cfg(target_arch = "wasm32")] pub mod utxo_wasm_tests; @@ -539,6 +541,31 @@ impl From for WithdrawError { fn from(e: UnsupportedAddr) -> Self { WithdrawError::InvalidAddress(e.to_string()) } } +#[derive(Debug)] +pub enum GetBlockHeaderError { + StorageError(BlockHeaderStorageError), + RpcError(JsonRpcError), + SerializationError(serialization::Error), + SPVError(SPVError), + NativeNotSupported(String), +} + +impl From for GetBlockHeaderError { + fn from(err: JsonRpcError) -> Self { GetBlockHeaderError::RpcError(err) } +} + +impl From for GetBlockHeaderError { + fn from(e: SPVError) -> Self { GetBlockHeaderError::SPVError(e) } +} + +impl From for GetBlockHeaderError { + fn from(err: serialization::Error) -> Self { GetBlockHeaderError::SerializationError(err) } +} + +impl From for GetBlockHeaderError { + fn from(err: BlockHeaderStorageError) -> Self { GetBlockHeaderError::StorageError(err) } +} + impl UtxoCoinFields { pub fn transaction_preimage(&self) -> TransactionInputSigner { let lock_time = if self.conf.ticker == "KMD" { diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index e0eef2e1f6..85cdbc0a84 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -1368,7 +1368,7 @@ pub fn validate_maker_payment( input: ValidatePaymentInput, ) -> Box + Send> where - T: AsRef + Clone + Send + Sync + 'static, + T: AsRef + Clone + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, { let my_public = try_fus!(Public::from_slice(&input.taker_pub)); let mut tx: UtxoTx = try_fus!(deserialize(input.payment_tx.as_slice()).map_err(|e| ERRL!("{:?}", e))); @@ -1391,7 +1391,7 @@ pub fn validate_taker_payment( input: ValidatePaymentInput, ) -> Box + Send> where - T: AsRef + Clone + Send + Sync + 'static, + T: AsRef + Clone + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, { let my_public = try_fus!(Public::from_slice(&input.maker_pub)); let mut tx: UtxoTx = try_fus!(deserialize(input.payment_tx.as_slice()).map_err(|e| ERRL!("{:?}", e))); @@ -2901,7 +2901,7 @@ pub fn address_from_pubkey( pub async fn validate_spv_proof(coin: T, tx: UtxoTx) -> Result<(), MmError> where - T: AsRef + Send + Sync + 'static, + T: AsRef + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, { let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => return Ok(()), @@ -2935,13 +2935,18 @@ where return MmError::err(SPVError::InvalidHeight); } - let block_header = client - .blockchain_block_header(height) - .compat() - .await - .map_to_mm(|_e| SPVError::UnableToGetHeader)?; - let raw_header = RawBlockHeader::new(block_header.0.clone())?; - let header: BlockHeader = deserialize(block_header.0.as_slice()).map_to_mm(|_e| SPVError::MalformattedHeader)?; + let (block_header, is_validated) = block_header_from_storage_or_rpc( + &coin, + height, + &coin.as_ref().block_headers_storage, + coin.as_ref().conf.block_header_storage_params.clone(), + ) + .await + .map_err(|_e| SPVError::UnableToGetHeader)?; + if !is_validated && coin.as_ref().conf.block_header_storage_params.is_some() { + return MmError::err(SPVError::BlockHeaderNotVerified); + } + let raw_header = RawBlockHeader::new(block_header.raw().take())?; let merkle_branch = client .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height) @@ -2958,7 +2963,7 @@ where vin: serialize_list(&tx.inputs).take(), vout: serialize_list(&tx.outputs).take(), index: merkle_branch.pos as u64, - confirming_header: header, + confirming_header: block_header, raw_header, intermediate_nodes, }; @@ -2980,7 +2985,7 @@ pub fn validate_payment( time_lock: u32, ) -> Box + Send> where - T: AsRef + Send + Sync + 'static, + T: AsRef + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, { let amount = try_fus!(sat_from_big_decimal(&amount, coin.as_ref().decimals)); @@ -3275,9 +3280,59 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } +pub async fn block_header_from_storage_or_rpc( + coin: &T, + height: u64, + storage: &Option, + header_params: Option, +) -> Result<(BlockHeader, bool), MmError> +where + T: AsRef + UtxoCommonOps + MarketCoinOps, +{ + let client = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => { + return MmError::err(GetBlockHeaderError::NativeNotSupported( + "Native client not supported".to_string(), + )) + }, + UtxoRpcClientEnum::Electrum(client) => client, + }; + match storage { + None => { + let bytes = client.blockchain_block_header(height).compat().await?; + let header: BlockHeader = deserialize(bytes.0.as_slice())?; + Ok((header, false)) + }, + Some(storage) => match storage.get_block_header(coin.ticker(), height).await? { + None => { + let bytes = client.blockchain_block_header(height).compat().await?; + let header: BlockHeader = deserialize(bytes.0.as_slice())?; + match header_params { + None => Ok((header, false)), + Some(params) => { + let (_, headers) = + utxo_common::retrieve_last_headers(coin, params.blocks_limit_to_check, Some(height)) + .await?; + match spv_validation::helpers_validation::validate_headers( + headers, + params.difficulty_check, + params.constant_difficulty, + ) { + Ok(_) => Ok((header, true)), + Err(_) => Ok((header, false)), + } + }, + } + }, + Some(header) => Ok((header, true)), + }, + } +} + pub async fn retrieve_last_headers( coin: &T, blocks_limit_to_check: u64, + current_block: Option, ) -> Result<(HashMap, Vec), MmError> where T: AsRef + UtxoCommonOps + MarketCoinOps, @@ -3287,7 +3342,11 @@ where UtxoRpcClientEnum::Electrum(electrum) => electrum, }; - let best_block = coin.current_block().compat().await; + let best_block = if let Some(block) = current_block { + Ok(block) + } else { + coin.current_block().compat().await + }; let (from, count) = match best_block { Ok(block) => { let block_height = block; @@ -3375,7 +3434,7 @@ pub async fn block_header_utxo_loop( } while let Some(arc) = weak.upgrade() { let coin = constructor(arc); - match retrieve_last_headers(&coin, blocks_limit_to_check).await { + match retrieve_last_headers(&coin, blocks_limit_to_check, None).await { Ok((block_registry, block_headers)) => { match validate_headers(block_headers, difficulty_check, constant_difficulty) { Ok(_) => { diff --git a/mm2src/mm2_bitcoin/chain/src/block_header.rs b/mm2src/mm2_bitcoin/chain/src/block_header.rs index 0e3b9ad446..962fca362f 100644 --- a/mm2src/mm2_bitcoin/chain/src/block_header.rs +++ b/mm2src/mm2_bitcoin/chain/src/block_header.rs @@ -2,6 +2,7 @@ use compact::Compact; use crypto::dhash256; use hash::H256; use hex::FromHex; +use primitives::bytes::Bytes; use primitives::U256; use ser::{deserialize, serialize, Deserializable, Reader, Serializable, Stream}; use std::io; @@ -265,6 +266,7 @@ impl Deserializable for BlockHeader { impl BlockHeader { pub fn hash(&self) -> H256 { dhash256(&serialize(self)) } + pub fn raw(&self) -> Bytes { serialize(self) } pub fn target(&self) -> Result { match self.bits { BlockHeaderBits::Compact(compact) => compact.to_u256(), diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index a7d2631355..cffb444c8a 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -55,6 +55,8 @@ pub enum SPVError { OutputLengthMismatch, /// Unable to retrieve block height / block height is zero. InvalidHeight, + /// Block Header Not Verified / Verification failed + BlockHeaderNotVerified, /// Any other error UnknownError, } From 43a311c93703076065f43ebd6df4d0fdc13a66c4 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 28 Feb 2022 10:10:00 +0100 Subject: [PATCH 48/74] feat(improvements): remove non-used function --- mm2src/coins/utxo/utxo_common.rs | 2 ++ mm2src/mm2_bitcoin/chain/src/raw_block.rs | 11 ----------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 85cdbc0a84..7e99d8e53a 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2943,6 +2943,8 @@ where ) .await .map_err(|_e| SPVError::UnableToGetHeader)?; + // Only specific chains will use the block header storage, for example QTUM will not + // Check block header verification only for chains that have storage or storage params. if !is_validated && coin.as_ref().conf.block_header_storage_params.is_some() { return MmError::err(SPVError::BlockHeaderNotVerified); } diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index f56ad735f4..5841af95f3 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -1,7 +1,6 @@ use crypto::dhash256; use primitives::bytes::Bytes; use primitives::hash::H256; -use primitives::U256; use ser::serialize; use BlockHeader; @@ -39,16 +38,6 @@ impl RawBlockHeader { root.as_mut().copy_from_slice(&self.0.as_ref()[4..36]); root } - - /// Extract the target from the header - pub fn target(&self) -> U256 { - let mantissa = U256::from_little_endian(&self.0.as_ref()[72..75]); - // We use saturating here to avoid panicking. - // This is safe because it saturates at `0`, which gives an unreachable target of `1` - let exponent = self.0.as_ref()[75].saturating_sub(3); - let offset = U256::from(256_u64).pow(exponent.into()); - mantissa * offset - } } impl From for RawBlockHeader { From 4e44bc34c259cc96fb18b5cf873f61849f78be24 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 28 Feb 2022 10:36:35 +0100 Subject: [PATCH 49/74] feat(storage): add to storage after validation in retrieve header from storage --- mm2src/coins/utxo/utxo_common.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 7e99d8e53a..a7ac9f4dab 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3312,7 +3312,7 @@ where match header_params { None => Ok((header, false)), Some(params) => { - let (_, headers) = + let (headers_registry, headers) = utxo_common::retrieve_last_headers(coin, params.blocks_limit_to_check, Some(height)) .await?; match spv_validation::helpers_validation::validate_headers( @@ -3320,7 +3320,12 @@ where params.difficulty_check, params.constant_difficulty, ) { - Ok(_) => Ok((header, true)), + Ok(_) => { + storage + .add_block_headers_to_storage(coin.ticker(), headers_registry) + .await?; + Ok((header, true)) + }, Err(_) => Ok((header, false)), } }, From 2ccd24271991d630af9de1daa472f12df573cafc Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 2 Mar 2022 09:27:23 +0100 Subject: [PATCH 50/74] feat(fix_review): first fixes batch --- mm2src/coins/utxo.rs | 2 +- .../coins/utxo/utxo_block_header_storage.rs | 16 +++ .../utxo/utxo_builder/utxo_coin_builder.rs | 8 +- mm2src/coins/utxo/utxo_common.rs | 9 +- .../utxo/utxo_sql_block_header_storage.rs | 131 ++++++++---------- .../utxo/utxo_wrapper_block_header_storage.rs | 21 +-- 6 files changed, 90 insertions(+), 97 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 7bd8194a5e..56236fd690 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -41,7 +41,7 @@ pub mod utxo_indexedb_block_header_storage; #[cfg(not(target_arch = "wasm32"))] pub mod utxo_sql_block_header_storage; pub mod utxo_wrapper_block_header_storage; -use utxo_wrapper_block_header_storage::BlockHeaderStorage; +use utxo_block_header_storage::BlockHeaderStorage; use async_trait::async_trait; use bigdecimal::BigDecimal; diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index f274912765..7777c3b829 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,9 +1,11 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; use async_trait::async_trait; use chain::BlockHeader; +use common::mm_ctx::MmArc; use common::mm_error::MmError; use derive_more::Display; use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; #[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] @@ -36,6 +38,20 @@ pub enum BlockHeaderStorageError { DecodeError { ticker: String, reason: String }, } +pub struct BlockHeaderStorage { + pub inner: Box, +} + +impl Debug for BlockHeaderStorage { + fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result { Ok(()) } +} + +pub trait InitBlockHeaderStorageOps: Send + Sync + 'static { + fn new_from_ctx(ctx: MmArc) -> Option + where + Self: Sized; +} + #[async_trait] pub trait BlockHeaderStorageOps: Send + Sync + 'static { /// Initializes collection/tables in storage for a specified coin diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 0577a47529..1c12f5c327 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -2,8 +2,8 @@ use crate::hd_pubkey::{HDExtractPubkeyError, HDXPubExtractor}; use crate::hd_wallet::{HDAccountsMap, HDAccountsMutex}; use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumRpcRequest, EstimateFeeMethod, UtxoRpcClientEnum}; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, InitBlockHeaderStorageOps}; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError, UtxoConfResult}; -use crate::utxo::utxo_wrapper_block_header_storage::BlockHeaderStorage; use crate::utxo::{output_script, utxo_common, ElectrumBuilderArgs, ElectrumProtoVerifier, RecentlySpentOutPoints, TxFee, UtxoCoinConf, UtxoCoinFields, UtxoHDAccount, UtxoHDWallet, UtxoRpcMode, DEFAULT_GAP_LIMIT, UTXO_DUST_AMOUNT}; @@ -182,6 +182,10 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { let tx_cache_directory = Some(self.ctx().dbdir().join("TX_CACHE")); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); + let block_headers_storage = match conf.block_header_storage_params { + None => None, + Some(_) => BlockHeaderStorage::new_from_ctx(self.ctx().clone()), + }; let coin = UtxoCoinFields { conf, @@ -192,7 +196,7 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { derivation_method, history_sync_state: Mutex::new(initial_history_state), tx_cache_directory, - block_headers_storage: BlockHeaderStorage::new_from_ctx(self.ctx().clone()), + block_headers_storage, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_fee, tx_hash_algo, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index a7ac9f4dab..a22518fdfd 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -16,7 +16,7 @@ pub use chain::Transaction as UtxoTx; use chain::{BlockHeader, OutPoint, RawBlockHeader, TransactionOutput}; use common::executor::Timer; use common::jsonrpc_client::JsonRpcErrorType; -use common::log::{error, info, warn}; +use common::log::{debug, error, info, warn}; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; use common::mm_metrics::MetricsArc; @@ -2969,10 +2969,7 @@ where raw_header, intermediate_nodes, }; - match proof.validate() { - Ok(_) => Ok(()), - Err(err) => MmError::err(err), - } + proof.validate().map_err(MmError::new) } #[allow(clippy::too_many_arguments)] @@ -3463,7 +3460,7 @@ pub async fn block_header_utxo_loop( }, Err(err) => error!("error: {:?}", err), } - info!("tick block_header_utxo_loop for {}", coin.as_ref().conf.ticker); + debug!("tick block_header_utxo_loop for {}", coin.as_ref().conf.ticker); Timer::sleep(check_every).await; } } diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index 19013d65d6..19d2f77949 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -7,7 +7,7 @@ use common::mm_error::MmError; use db_common::sqlite::rusqlite::Error as SqlError; use db_common::sqlite::rusqlite::{Connection, Row, ToSql, NO_PARAMS}; use db_common::sqlite::validate_table_name; -use serialization::{deserialize, serialize}; +use serialization::deserialize; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -84,6 +84,55 @@ where fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } +struct SqlBlockHeader(String, String); + +impl From for SqlBlockHeader { + fn from(header: ElectrumBlockHeader) -> Self { + match header { + ElectrumBlockHeader::V12(h) => { + let block_hex = h.as_hex(); + let block_height = h.block_height.to_string(); + SqlBlockHeader(block_height, block_hex) + }, + ElectrumBlockHeader::V14(h) => { + let block_hex = format!("{:02x}", h.hex); + let block_height = h.height.to_string(); + SqlBlockHeader(block_height, block_hex) + }, + } + } +} +async fn common_headers_insert( + for_coin: &str, + storage: SqliteBlockHeadersStorage, + headers: Vec, +) -> Result<(), MmError> { + let for_coin = for_coin.to_owned(); + let mut conn = storage.0.lock().unwrap(); + let sql_transaction = conn + .transaction() + .map_err(|e| BlockHeaderStorageError::TransactionError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; + for header in headers { + let block_cache_params = [&header.0, &header.1]; + sql_transaction + .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) + .map_err(|e| BlockHeaderStorageError::ExecutionError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; + } + sql_transaction + .commit() + .map_err(|e| BlockHeaderStorageError::CommitError { + ticker: for_coin.to_string(), + reason: e.to_string(), + })?; + Ok(()) +} + #[async_trait] impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { async fn init(&self, for_coin: &str) -> Result<(), MmError> { @@ -131,49 +180,8 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { for_coin: &str, headers: Vec, ) -> Result<(), MmError> { - let for_coin = for_coin.to_owned(); - let selfi = self.clone(); - async_blocking(move || { - let mut conn = selfi.0.lock().unwrap(); - let sql_transaction = conn - .transaction() - .map_err(|e| BlockHeaderStorageError::TransactionError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - for header in headers { - match header { - ElectrumBlockHeader::V12(h) => { - let block_hex = h.as_hex(); - let block_cache_params = [&h.block_height.to_string(), &block_hex]; - sql_transaction - .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) - .map_err(|e| BlockHeaderStorageError::ExecutionError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - }, - ElectrumBlockHeader::V14(h) => { - let block_hex = format!("{:02x}", h.hex); - let block_cache_params = [&h.height.to_string(), &block_hex]; - sql_transaction - .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) - .map_err(|e| BlockHeaderStorageError::ExecutionError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - }, - } - } - sql_transaction - .commit() - .map_err(|e| BlockHeaderStorageError::CommitError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - Ok(()) - }) - .await + let headers_for_sql = headers.into_iter().map(Into::into).collect(); + common_headers_insert(for_coin, self.clone(), headers_for_sql).await } async fn add_block_headers_to_storage( @@ -181,36 +189,11 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { for_coin: &str, headers: HashMap, ) -> Result<(), MmError> { - let for_coin = for_coin.to_owned(); - let selfi = self.clone(); - async_blocking(move || { - let mut conn = selfi.0.lock().unwrap(); - let sql_transaction = conn - .transaction() - .map_err(|e| BlockHeaderStorageError::TransactionError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - for (height, header) in headers { - let serialized = serialize(&header); - let block_hex = hex::encode(&serialized); - let block_cache_params = [&height.to_string(), &block_hex]; - sql_transaction - .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) - .map_err(|e| BlockHeaderStorageError::ExecutionError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - } - sql_transaction - .commit() - .map_err(|e| BlockHeaderStorageError::CommitError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - Ok(()) - }) - .await + let headers_for_sql = headers + .into_iter() + .map(|(height, header)| SqlBlockHeader(height.to_string(), hex::encode(header.raw()))) + .collect(); + common_headers_insert(for_coin, self.clone(), headers_for_sql).await } async fn get_block_header( diff --git a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs index 1c5edfe414..0e60b758cd 100644 --- a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs @@ -1,5 +1,6 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderStorageError, BlockHeaderStorageOps, + InitBlockHeaderStorageOps}; #[cfg(target_arch = "wasm32")] use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; #[cfg(not(target_arch = "wasm32"))] @@ -10,26 +11,18 @@ use common::mm_ctx::MmArc; use common::mm_error::MmError; use std::collections::HashMap; -#[derive(Debug)] -pub struct BlockHeaderStorage { +impl InitBlockHeaderStorageOps for BlockHeaderStorage { #[cfg(not(target_arch = "wasm32"))] - inner: SqliteBlockHeadersStorage, - #[cfg(target_arch = "wasm32")] - inner: IndexedDBBlockHeadersStorage, -} - -impl BlockHeaderStorage { - #[cfg(not(target_arch = "wasm32"))] - pub fn new_from_ctx(ctx: MmArc) -> Option { + fn new_from_ctx(ctx: MmArc) -> Option { ctx.sqlite_connection.as_option().map(|connection| BlockHeaderStorage { - inner: SqliteBlockHeadersStorage(connection.clone()), + inner: Box::new(SqliteBlockHeadersStorage(connection.clone())), }) } #[cfg(target_arch = "wasm32")] - pub fn new_from_ctx(_ctx: MmArc) -> Option { + fn new_from_ctx(_ctx: MmArc) -> Option { Some(BlockHeaderStorage { - inner: IndexedDBBlockHeadersStorage {}, + inner: Box::new(IndexedDBBlockHeadersStorage {}), }) } } From 33877d28463c4c7801683828e7a17e56af336324 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 2 Mar 2022 09:57:10 +0100 Subject: [PATCH 51/74] feat(fix_review): sql fixes + rename `from_address` to `sender_address` --- mm2src/coins/sql_tx_history_storage.rs | 14 ++-------- mm2src/coins/utxo/utxo_common.rs | 21 ++++++++------ .../utxo/utxo_sql_block_header_storage.rs | 28 ++++++++----------- mm2src/coins/utxo/utxo_withdraw.rs | 24 ++++++++-------- mm2src/db_common/src/sqlite.rs | 21 +++++++++++++- 5 files changed, 56 insertions(+), 52 deletions(-) diff --git a/mm2src/coins/sql_tx_history_storage.rs b/mm2src/coins/sql_tx_history_storage.rs index 71596b57a6..eef606f767 100644 --- a/mm2src/coins/sql_tx_history_storage.rs +++ b/mm2src/coins/sql_tx_history_storage.rs @@ -7,14 +7,12 @@ use common::{async_blocking, PagingOptionsEnum}; use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, ToSql, NO_PARAMS}; use db_common::sqlite::sql_builder::SqlBuilder; -use db_common::sqlite::{offset_by_id, validate_table_name}; +use db_common::sqlite::{offset_by_id, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use rpc::v1::types::Bytes as BytesJson; use serde_json::{self as json}; use std::convert::TryInto; use std::sync::{Arc, Mutex}; -const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; - fn tx_history_table(ticker: &str) -> String { ticker.to_owned() + "_tx_history" } fn tx_cache_table(ticker: &str) -> String { ticker.to_owned() + "_tx_cache" } @@ -194,17 +192,9 @@ where P::Item: ToSql, F: FnOnce(&Row<'_>) -> Result, { - let maybe_result = conn.query_row(query, params, map_fn); - if let Err(SqlError::QueryReturnedNoRows) = maybe_result { - return Ok(None); - } - - let result = maybe_result?; - Ok(Some(result)) + db_common::sqlite::query_single_row(conn, query, params, map_fn).map_err(MmError::new) } -fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } - fn tx_details_from_row(row: &Row<'_>) -> Result { let json_string: String = row.get(0)?; json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index a22518fdfd..d4ab72abf3 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -1368,7 +1368,7 @@ pub fn validate_maker_payment( input: ValidatePaymentInput, ) -> Box + Send> where - T: AsRef + Clone + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, + T: AsRef + Clone + Send + Sync + 'static, { let my_public = try_fus!(Public::from_slice(&input.taker_pub)); let mut tx: UtxoTx = try_fus!(deserialize(input.payment_tx.as_slice()).map_err(|e| ERRL!("{:?}", e))); @@ -1391,7 +1391,7 @@ pub fn validate_taker_payment( input: ValidatePaymentInput, ) -> Box + Send> where - T: AsRef + Clone + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, + T: AsRef + Clone + Send + Sync + 'static, { let my_public = try_fus!(Public::from_slice(&input.maker_pub)); let mut tx: UtxoTx = try_fus!(deserialize(input.payment_tx.as_slice()).map_err(|e| ERRL!("{:?}", e))); @@ -2901,7 +2901,7 @@ pub fn address_from_pubkey( pub async fn validate_spv_proof(coin: T, tx: UtxoTx) -> Result<(), MmError> where - T: AsRef + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, + T: AsRef + Send + Sync + 'static, { let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => return Ok(()), @@ -2984,7 +2984,7 @@ pub fn validate_payment( time_lock: u32, ) -> Box + Send> where - T: AsRef + Send + Sync + 'static + UtxoCommonOps + MarketCoinOps, + T: AsRef + Send + Sync + 'static, { let amount = try_fus!(sat_from_big_decimal(&amount, coin.as_ref().decimals)); @@ -3286,7 +3286,7 @@ pub async fn block_header_from_storage_or_rpc( header_params: Option, ) -> Result<(BlockHeader, bool), MmError> where - T: AsRef + UtxoCommonOps + MarketCoinOps, + T: AsRef, { let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => { @@ -3302,7 +3302,10 @@ where let header: BlockHeader = deserialize(bytes.0.as_slice())?; Ok((header, false)) }, - Some(storage) => match storage.get_block_header(coin.ticker(), height).await? { + Some(storage) => match storage + .get_block_header(coin.as_ref().conf.ticker.as_str(), height) + .await? + { None => { let bytes = client.blockchain_block_header(height).compat().await?; let header: BlockHeader = deserialize(bytes.0.as_slice())?; @@ -3319,7 +3322,7 @@ where ) { Ok(_) => { storage - .add_block_headers_to_storage(coin.ticker(), headers_registry) + .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), headers_registry) .await?; Ok((header, true)) }, @@ -3339,7 +3342,7 @@ pub async fn retrieve_last_headers( current_block: Option, ) -> Result<(HashMap, Vec), MmError> where - T: AsRef + UtxoCommonOps + MarketCoinOps, + T: AsRef, { let electrum_rpc_client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => return MmError::err(SPVError::UnknownError), @@ -3349,7 +3352,7 @@ where let best_block = if let Some(block) = current_block { Ok(block) } else { - coin.current_block().compat().await + coin.as_ref().rpc_client.get_block_count().compat().await }; let (from, count) = match best_block { Ok(block) => { diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index 19d2f77949..c133d1635d 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -4,15 +4,15 @@ use async_trait::async_trait; use chain::BlockHeader; use common::async_blocking; use common::mm_error::MmError; -use db_common::sqlite::rusqlite::Error as SqlError; -use db_common::sqlite::rusqlite::{Connection, Row, ToSql, NO_PARAMS}; -use db_common::sqlite::validate_table_name; +use db_common::{sqlite::rusqlite::Error as SqlError, + sqlite::rusqlite::{Connection, Row, ToSql, NO_PARAMS}, + sqlite::string_from_row, + sqlite::validate_table_name, + sqlite::CHECK_TABLE_EXISTS_SQL}; use serialization::deserialize; use std::collections::HashMap; use std::sync::{Arc, Mutex}; -const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; - fn block_headers_cache_table(ticker: &str) -> String { ticker.to_owned() + "_block_headers_cache" } fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { @@ -70,20 +70,14 @@ where P::Item: ToSql, F: FnOnce(&Row<'_>) -> Result, { - let maybe_result = conn.query_row(query, params, map_fn); - if let Err(SqlError::QueryReturnedNoRows) = maybe_result { - return Ok(None); - } - - let result = maybe_result.map_err(|e| BlockHeaderStorageError::QueryError { - query: query.to_string(), - reason: e.to_string(), - })?; - Ok(Some(result)) + db_common::sqlite::query_single_row(conn, query, params, map_fn).map_err(|e| { + MmError::new(BlockHeaderStorageError::QueryError { + query: query.to_string(), + reason: e.to_string(), + }) + }) } -fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } - struct SqlBlockHeader(String, String); impl From for SqlBlockHeader { diff --git a/mm2src/coins/utxo/utxo_withdraw.rs b/mm2src/coins/utxo/utxo_withdraw.rs index 3b5486e698..a3fd8c6f20 100644 --- a/mm2src/coins/utxo/utxo_withdraw.rs +++ b/mm2src/coins/utxo/utxo_withdraw.rs @@ -104,22 +104,20 @@ where { fn coin(&self) -> &Coin; - #[allow(clippy::wrong_self_convention)] - fn from_address(&self) -> Address; + fn sender_address(&self) -> Address; - #[allow(clippy::wrong_self_convention)] - fn from_address_string(&self) -> String; + fn sender_address_string(&self) -> String; fn request(&self) -> &WithdrawRequest; fn signature_version(&self) -> SignatureVersion { - match self.from_address().addr_format { + match self.sender_address().addr_format { UtxoAddressFormat::Segwit => SignatureVersion::WitnessV0, _ => self.coin().as_ref().conf.signature_version, } } - fn prev_script(&self) -> Script { Builder::build_p2pkh(&self.from_address().hash) } + fn prev_script(&self) -> Script { Builder::build_p2pkh(&self.sender_address().hash) } fn on_generating_transaction(&self) -> Result<(), MmError>; @@ -154,7 +152,7 @@ where let script_pubkey = output_script(&to, script_type).to_bytes(); let _utxo_lock = UTXO_LOCK.lock().await; - let (unspents, _) = coin.list_unspent_ordered(&self.from_address()).await?; + let (unspents, _) = coin.list_unspent_ordered(&self.sender_address()).await?; let (value, fee_policy) = if req.max { ( unspents.iter().fold(0, |sum, unspent| sum + unspent.value), @@ -167,7 +165,7 @@ where let outputs = vec![TransactionOutput { value, script_pubkey }]; let mut tx_builder = UtxoTxBuilder::new(coin) - .with_from_address(self.from_address()) + .with_from_address(self.sender_address()) .add_available_inputs(unspents) .add_outputs(outputs) .with_fee_policy(fee_policy); @@ -210,7 +208,7 @@ where _ => serialize(&signed).into(), }; Ok(TransactionDetails { - from: vec![self.from_address_string()], + from: vec![self.sender_address_string()], to: vec![req.to.clone()], total_amount: big_decimal_from_sat(data.spent_by_me as i64, decimals), spent_by_me: big_decimal_from_sat(data.spent_by_me as i64, decimals), @@ -250,9 +248,9 @@ where { fn coin(&self) -> &Coin { &self.coin } - fn from_address(&self) -> Address { self.from_address.clone() } + fn sender_address(&self) -> Address { self.from_address.clone() } - fn from_address_string(&self) -> String { self.from_address_string.clone() } + fn sender_address_string(&self) -> String { self.from_address_string.clone() } fn request(&self) -> &WithdrawRequest { &self.req } @@ -438,9 +436,9 @@ where { fn coin(&self) -> &Coin { &self.coin } - fn from_address(&self) -> Address { self.my_address.clone() } + fn sender_address(&self) -> Address { self.my_address.clone() } - fn from_address_string(&self) -> String { self.my_address_string.clone() } + fn sender_address_string(&self) -> String { self.my_address_string.clone() } fn request(&self) -> &WithdrawRequest { &self.req } diff --git a/mm2src/db_common/src/sqlite.rs b/mm2src/db_common/src/sqlite.rs index e1803a841d..54957fdfdf 100644 --- a/mm2src/db_common/src/sqlite.rs +++ b/mm2src/db_common/src/sqlite.rs @@ -2,10 +2,29 @@ pub use rusqlite; pub use sql_builder; use log::debug; -use rusqlite::{Connection, Error as SqlError, Result as SqlResult, ToSql}; +use rusqlite::{Connection, Error as SqlError, Result as SqlResult, Row, ToSql}; use sql_builder::SqlBuilder; use uuid::Uuid; +pub const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;"; + +pub fn string_from_row(row: &Row<'_>) -> Result { row.get(0) } + +pub fn query_single_row(conn: &Connection, query: &str, params: P, map_fn: F) -> Result, SqlError> +where + P: IntoIterator, + P::Item: ToSql, + F: FnOnce(&Row<'_>) -> Result, +{ + let maybe_result = conn.query_row(query, params, map_fn); + if let Err(SqlError::QueryReturnedNoRows) = maybe_result { + return Ok(None); + } + + let result = maybe_result?; + Ok(Some(result)) +} + pub fn validate_table_name(table_name: &str) -> SqlResult<()> { // As per https://stackoverflow.com/a/3247553, tables can't be the target of parameter substitution. // So we have to use a plain concatenation disallowing any characters in the table name that may lead to SQL injection. From b63dfd6b9ef0be02c454cc19fb97af14add607d0 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 2 Mar 2022 10:44:54 +0100 Subject: [PATCH 52/74] feat(fix_review): simplify the way to get height, use into_iter + find --- mm2src/coins/utxo/utxo_common.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index d4ab72abf3..c04f54e105 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2921,14 +2921,15 @@ where if history.is_empty() { continue; } - for item in history { - if item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0 { + match history + .into_iter() + .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) + { + None => {}, + Some(item) => { height = item.height as u64; break; - } - } - if height != 0 { - break; + }, } } if height == 0 { From 3fc93ecf735d7ea4dde7bb84e9bda4aea0145c47 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 2 Mar 2022 11:30:37 +0100 Subject: [PATCH 53/74] feat(fix_review): add a get_tx_height function + remove more marketcoinops in the arc builder --- .../utxo/utxo_builder/utxo_arc_builder.rs | 12 +-- mm2src/coins/utxo/utxo_common.rs | 93 +++++++++---------- 2 files changed, 51 insertions(+), 54 deletions(-) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 6c5fb1290d..f8010a8915 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -5,7 +5,7 @@ use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBui use crate::utxo::utxo_common::block_header_utxo_loop; use crate::utxo::utxo_common::merge_utxo_loop; use crate::utxo::{UtxoArc, UtxoBlockHeaderVerificationParams, UtxoCoinFields, UtxoCommonOps, UtxoWeak}; -use crate::{MarketCoinOps, PrivKeyBuildPolicy, UtxoActivationParams}; +use crate::{PrivKeyBuildPolicy, UtxoActivationParams}; use async_trait::async_trait; use common::executor::spawn; use common::log::info; @@ -87,7 +87,7 @@ where impl<'a, F, T, XPubExtractor> UtxoCoinBuilder for UtxoArcBuilder<'a, F, T, XPubExtractor> where F: Fn(UtxoArc) -> T + Clone + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + Send + Sync + 'static, XPubExtractor: HDXPubExtractor + Send + Sync, { type ResultCoin = T; @@ -121,7 +121,7 @@ where impl<'a, F, T, XPubExtractor> BlockHeaderUtxoArcOps for UtxoArcBuilder<'a, F, T, XPubExtractor> where F: Fn(UtxoArc) -> T + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + Send + Sync + 'static, XPubExtractor: HDXPubExtractor + Send + Sync, { } @@ -166,7 +166,7 @@ where impl<'a, F, T> BlockHeaderUtxoArcOps for UtxoArcWithIguanaPrivKeyBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + Send + Sync + 'static, { } @@ -174,7 +174,7 @@ where impl<'a, F, T> UtxoCoinWithIguanaPrivKeyBuilder for UtxoArcWithIguanaPrivKeyBuilder<'a, F, T> where F: Fn(UtxoArc) -> T + Clone + Send + Sync + 'static, - T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + Send + Sync + 'static, { type ResultCoin = T; type Error = UtxoCoinBuildError; @@ -242,7 +242,7 @@ where pub trait BlockHeaderUtxoArcOps: UtxoCoinBuilderCommonOps where - T: AsRef + UtxoCommonOps + MarketCoinOps + Send + Sync + 'static, + T: AsRef + UtxoCommonOps + Send + Sync + 'static, { fn spawn_block_header_utxo_loop_if_required( &self, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index c04f54e105..5acd0c37e1 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2910,29 +2910,8 @@ where if tx.outputs.is_empty() { return MmError::err(SPVError::InvalidVout); } - let mut height: u64 = 0; - for output in tx.outputs.clone() { - let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); - let history = client - .scripthash_get_history(script_pubkey_str.as_str()) - .compat() - .await - .unwrap_or_default(); - if history.is_empty() { - continue; - } - match history - .into_iter() - .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) - { - None => {}, - Some(item) => { - height = item.height as u64; - break; - }, - } - } - if height == 0 { + let height = get_tx_height(&tx, client).await; + if height.is_zero() { return MmError::err(SPVError::InvalidHeight); } @@ -2973,6 +2952,29 @@ where proof.validate().map_err(MmError::new) } +pub async fn get_tx_height(tx: &UtxoTx, client: &ElectrumClient) -> u64 { + let mut height: u64 = 0; + for output in tx.outputs.clone() { + let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); + let history = client + .scripthash_get_history(script_pubkey_str.as_str()) + .compat() + .await + .unwrap_or_default(); + if history.is_empty() { + continue; + } + if let Some(item) = history + .into_iter() + .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) + { + height = item.height as u64; + break; + } + } + height +} + #[allow(clippy::too_many_arguments)] pub fn validate_payment( coin: T, @@ -3314,8 +3316,7 @@ where None => Ok((header, false)), Some(params) => { let (headers_registry, headers) = - utxo_common::retrieve_last_headers(coin, params.blocks_limit_to_check, Some(height)) - .await?; + utxo_common::retrieve_last_headers(coin, params.blocks_limit_to_check, height).await?; match spv_validation::helpers_validation::validate_headers( headers, params.difficulty_check, @@ -3340,7 +3341,7 @@ where pub async fn retrieve_last_headers( coin: &T, blocks_limit_to_check: u64, - current_block: Option, + block_height: u64, ) -> Result<(HashMap, Vec), MmError> where T: AsRef, @@ -3350,25 +3351,13 @@ where UtxoRpcClientEnum::Electrum(electrum) => electrum, }; - let best_block = if let Some(block) = current_block { - Ok(block) - } else { - coin.as_ref().rpc_client.get_block_count().compat().await - }; - let (from, count) = match best_block { - Ok(block) => { - let block_height = block; - let from = if block_height < blocks_limit_to_check { - 0 - } else { - block_height - blocks_limit_to_check - }; - (from, NonZeroU64::new(blocks_limit_to_check).unwrap()) - }, - Err(_) => { - error!("Unable to retrieve last block height - skipping"); - return MmError::err(SPVError::UnableToGetHeader); - }, + let (from, count) = { + let from = if block_height < blocks_limit_to_check { + 0 + } else { + block_height - blocks_limit_to_check + }; + (from, NonZeroU64::new(blocks_limit_to_check).unwrap()) }; let headers_resp = electrum_rpc_client.blockchain_block_headers(from, count).compat().await; let (block_registry, block_headers) = match headers_resp { @@ -3406,14 +3395,14 @@ pub async fn block_header_utxo_loop( blocks_limit_to_check: u64, constructor: impl Fn(UtxoArc) -> T, ) where - T: AsRef + UtxoCommonOps + MarketCoinOps, + T: AsRef + UtxoCommonOps, { { let coin = match weak.upgrade() { Some(arc) => constructor(arc), None => return, }; - let ticker = coin.ticker(); + let ticker = coin.as_ref().conf.ticker.as_str(); let storage = match &coin.as_ref().block_headers_storage { None => return, Some(storage) => storage, @@ -3442,7 +3431,15 @@ pub async fn block_header_utxo_loop( } while let Some(arc) = weak.upgrade() { let coin = constructor(arc); - match retrieve_last_headers(&coin, blocks_limit_to_check, None).await { + let height = match coin.as_ref().rpc_client.get_block_count().compat().await { + Ok(height) => height, + Err(err) => { + error!("error: {:?}", err); + Timer::sleep(check_every).await; + continue; + }, + }; + match retrieve_last_headers(&coin, blocks_limit_to_check, height).await { Ok((block_registry, block_headers)) => { match validate_headers(block_headers, difficulty_check, constant_difficulty) { Ok(_) => { From 676d30c40ad8075962e9ccf07ba551c75904ce6a Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 2 Mar 2022 12:13:20 +0100 Subject: [PATCH 54/74] feat(fix_review): simplify download_loop with a `try_loop` macro --- mm2src/coins/utxo/utxo_common.rs | 59 ++++++++++++++++---------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 5acd0c37e1..fc8cd35615 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3387,6 +3387,19 @@ where Ok((block_registry, block_headers)) } +macro_rules! try_loop { + ($e:expr, $delay: ident) => { + match $e { + Ok(res) => res, + Err(e) => { + error!("error {:?}", e); + Timer::sleep($delay).await; + continue; + }, + } + }; +} + pub async fn block_header_utxo_loop( weak: UtxoWeak, check_every: f64, @@ -3431,36 +3444,24 @@ pub async fn block_header_utxo_loop( } while let Some(arc) = weak.upgrade() { let coin = constructor(arc); - let height = match coin.as_ref().rpc_client.get_block_count().compat().await { - Ok(height) => height, - Err(err) => { - error!("error: {:?}", err); - Timer::sleep(check_every).await; - continue; - }, + let height = try_loop!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); + let (block_registry, block_headers) = try_loop!( + retrieve_last_headers(&coin, blocks_limit_to_check, height).await, + check_every + ); + try_loop!( + validate_headers(block_headers, difficulty_check, constant_difficulty), + check_every + ); + let storage = match &coin.as_ref().block_headers_storage { + None => break, + Some(storage) => storage, }; - match retrieve_last_headers(&coin, blocks_limit_to_check, height).await { - Ok((block_registry, block_headers)) => { - match validate_headers(block_headers, difficulty_check, constant_difficulty) { - Ok(_) => { - let storage = match &coin.as_ref().block_headers_storage { - None => break, - Some(storage) => storage, - }; - let ticker = coin.as_ref().conf.ticker.as_str(); - match storage.add_block_headers_to_storage(ticker, block_registry).await { - Ok(_) => info!( - "Successfully add block header to storage after validation for {}", - ticker - ), - Err(err) => error!("error: {:?}", err), - } - }, - Err(err) => error!("error: {:?}", err), - } - }, - Err(err) => error!("error: {:?}", err), - } + let ticker = coin.as_ref().conf.ticker.as_str(); + try_loop!( + storage.add_block_headers_to_storage(ticker, block_registry).await, + check_every + ); debug!("tick block_header_utxo_loop for {}", coin.as_ref().conf.ticker); Timer::sleep(check_every).await; } From e4504b405bca02655ae101602ee0de1042f97791 Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 3 Mar 2022 09:22:09 +0100 Subject: [PATCH 55/74] feat(fix_review): improve get_tx_height --- mm2src/coins/utxo.rs | 13 +++++++++++++ mm2src/coins/utxo/utxo_common.rs | 14 ++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 56236fd690..a6f66a1ff1 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -547,6 +547,19 @@ impl From for WithdrawError { fn from(e: UnsupportedAddr) -> Self { WithdrawError::InvalidAddress(e.to_string()) } } +#[derive(Debug)] +pub enum GetTxHeightError { + HeightNotFound, +} + +impl From for SPVError { + fn from(e: GetTxHeightError) -> Self { + match e { + GetTxHeightError::HeightNotFound => SPVError::InvalidHeight, + } + } +} + #[derive(Debug)] pub enum GetBlockHeaderError { StorageError(BlockHeaderStorageError), diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index fc8cd35615..5e36d718e6 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2910,11 +2910,7 @@ where if tx.outputs.is_empty() { return MmError::err(SPVError::InvalidVout); } - let height = get_tx_height(&tx, client).await; - if height.is_zero() { - return MmError::err(SPVError::InvalidHeight); - } - + let height = get_tx_height(&tx, client).await?; let (block_header, is_validated) = block_header_from_storage_or_rpc( &coin, height, @@ -2952,8 +2948,7 @@ where proof.validate().map_err(MmError::new) } -pub async fn get_tx_height(tx: &UtxoTx, client: &ElectrumClient) -> u64 { - let mut height: u64 = 0; +pub async fn get_tx_height(tx: &UtxoTx, client: &ElectrumClient) -> Result> { for output in tx.outputs.clone() { let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); let history = client @@ -2968,11 +2963,10 @@ pub async fn get_tx_height(tx: &UtxoTx, client: &ElectrumClient) -> u64 { .into_iter() .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) { - height = item.height as u64; - break; + return Ok(item.height as u64); } } - height + MmError::err(GetTxHeightError::HeightNotFound) } #[allow(clippy::too_many_arguments)] From 900e62a3b7afb0aa46fb674f40315d22136bfeaf Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 3 Mar 2022 10:00:29 +0100 Subject: [PATCH 56/74] feat(fix_review): continue review improvements, move retrieve_last_headers to electrum --- mm2src/coins/utxo.rs | 12 +++++++++ mm2src/coins/utxo/rpc_clients.rs | 43 ++++++++++++++++++++++++++++++++ mm2src/coins/utxo/utxo_common.rs | 41 ++++++++++++++++-------------- 3 files changed, 77 insertions(+), 19 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index a6f66a1ff1..8fa02e94c8 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -565,14 +565,26 @@ pub enum GetBlockHeaderError { StorageError(BlockHeaderStorageError), RpcError(JsonRpcError), SerializationError(serialization::Error), + InvalidResponse(String), SPVError(SPVError), NativeNotSupported(String), + Internal(String), } impl From for GetBlockHeaderError { fn from(err: JsonRpcError) -> Self { GetBlockHeaderError::RpcError(err) } } +impl From for GetBlockHeaderError { + fn from(e: UtxoRpcError) -> Self { + match e { + UtxoRpcError::Transport(e) | UtxoRpcError::ResponseParseError(e) => GetBlockHeaderError::RpcError(e), + UtxoRpcError::InvalidResponse(e) => GetBlockHeaderError::InvalidResponse(e), + UtxoRpcError::Internal(e) => GetBlockHeaderError::Internal(e), + } + } +} + impl From for GetBlockHeaderError { fn from(e: SPVError) -> Self { GetBlockHeaderError::SPVError(e) } } diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 79856e59dd..220d67f92c 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1668,6 +1668,49 @@ impl ElectrumClient { rpc_func!(self, "blockchain.block.headers", start_height, count) } + pub fn retrieve_last_headers( + &self, + blocks_limit_to_check: u64, + block_height: u64, + ) -> UtxoRpcFut<(HashMap, Vec)> { + let (from, count) = { + let from = if block_height < blocks_limit_to_check { + 0 + } else { + block_height - blocks_limit_to_check + }; + (from, NonZeroU64::new(blocks_limit_to_check).unwrap()) + }; + Box::new( + self.blockchain_block_headers(from, count) + .map_to_mm_fut(UtxoRpcError::from) + .and_then(move |headers| { + let (block_registry, block_headers) = { + if headers.count == 0 { + return MmError::err(UtxoRpcError::Internal("No headers available".to_string())); + } + let len = CompactInteger::from(headers.count); + let mut serialized = serialize(&len).take(); + serialized.extend(headers.hex.0.into_iter()); + let mut reader = Reader::new_with_coin_variant(serialized.as_slice(), CoinVariant::Standard); + let maybe_block_headers = reader.read_list::(); + let block_headers = match maybe_block_headers { + Ok(headers) => headers, + Err(e) => return MmError::err(UtxoRpcError::InvalidResponse(format!("{:?}", e))), + }; + let mut block_registry: HashMap = HashMap::new(); + let mut starting_height = from; + for block_header in &block_headers { + block_registry.insert(starting_height, block_header.clone()); + starting_height += 1; + } + (block_registry, block_headers) + }; + Ok((block_registry, block_headers)) + }), + ) + } + /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-transaction-get-merkle pub fn blockchain_transaction_get_merkle(&self, txid: H256Json, height: u64) -> RpcRes { rpc_func!(self, "blockchain.transaction.get_merkle", txid, height) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 5e36d718e6..479756646e 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -34,7 +34,7 @@ use rpc::v1::types::{Bytes as BytesJson, TransactionInputEnum, H256 as H256Json} use script::{Builder, Opcode, Script, ScriptAddress, TransactionInputSigner, UnsignedTransactionInput}; use secp256k1::{PublicKey, Signature}; use serde_json::{self as json}; -use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, CompactInteger, Reader, +use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, SERIALIZE_TRANSACTION_WITNESS}; use spv_validation::helpers_validation::validate_headers; use spv_validation::spv_proof::SPVProof; @@ -2951,19 +2951,13 @@ where pub async fn get_tx_height(tx: &UtxoTx, client: &ElectrumClient) -> Result> { for output in tx.outputs.clone() { let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); - let history = client - .scripthash_get_history(script_pubkey_str.as_str()) - .compat() - .await - .unwrap_or_default(); - if history.is_empty() { - continue; - } - if let Some(item) = history - .into_iter() - .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) - { - return Ok(item.height as u64); + if let Ok(history) = client.scripthash_get_history(script_pubkey_str.as_str()).compat().await { + if let Some(item) = history + .into_iter() + .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) + { + return Ok(item.height as u64); + } } } MmError::err(GetTxHeightError::HeightNotFound) @@ -3309,8 +3303,10 @@ where match header_params { None => Ok((header, false)), Some(params) => { - let (headers_registry, headers) = - utxo_common::retrieve_last_headers(coin, params.blocks_limit_to_check, height).await?; + let (headers_registry, headers) = client + .retrieve_last_headers(params.blocks_limit_to_check, height) + .compat() + .await?; match spv_validation::helpers_validation::validate_headers( headers, params.difficulty_check, @@ -3332,7 +3328,7 @@ where } } -pub async fn retrieve_last_headers( +/*pub async fn retrieve_last_headers( coin: &T, blocks_limit_to_check: u64, block_height: u64, @@ -3379,7 +3375,7 @@ where Err(_) => return MmError::err(SPVError::UnableToGetHeader), }; Ok((block_registry, block_headers)) -} +}*/ macro_rules! try_loop { ($e:expr, $delay: ident) => { @@ -3439,8 +3435,15 @@ pub async fn block_header_utxo_loop( while let Some(arc) = weak.upgrade() { let coin = constructor(arc); let height = try_loop!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); + let client = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => break, + UtxoRpcClientEnum::Electrum(client) => client, + }; let (block_registry, block_headers) = try_loop!( - retrieve_last_headers(&coin, blocks_limit_to_check, height).await, + client + .retrieve_last_headers(blocks_limit_to_check, height) + .compat() + .await, check_every ); try_loop!( From 1bba0dfe06058ffc11729f02bd7a3673f7e73aa7 Mon Sep 17 00:00:00 2001 From: milerius Date: Thu, 3 Mar 2022 10:14:07 +0100 Subject: [PATCH 57/74] feat(fix_review): remove dead code comment --- mm2src/coins/utxo/utxo_common.rs | 49 -------------------------------- 1 file changed, 49 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 479756646e..f647331477 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3328,55 +3328,6 @@ where } } -/*pub async fn retrieve_last_headers( - coin: &T, - blocks_limit_to_check: u64, - block_height: u64, -) -> Result<(HashMap, Vec), MmError> -where - T: AsRef, -{ - let electrum_rpc_client = match &coin.as_ref().rpc_client { - UtxoRpcClientEnum::Native(_) => return MmError::err(SPVError::UnknownError), - UtxoRpcClientEnum::Electrum(electrum) => electrum, - }; - - let (from, count) = { - let from = if block_height < blocks_limit_to_check { - 0 - } else { - block_height - blocks_limit_to_check - }; - (from, NonZeroU64::new(blocks_limit_to_check).unwrap()) - }; - let headers_resp = electrum_rpc_client.blockchain_block_headers(from, count).compat().await; - let (block_registry, block_headers) = match headers_resp { - Ok(headers) => { - if headers.count == 0 { - return MmError::err(SPVError::UnableToGetHeader); - } - let len = CompactInteger::from(headers.count); - let mut serialized = serialize(&len).take(); - serialized.extend(headers.hex.0.into_iter()); - let mut reader = Reader::new_with_coin_variant(serialized.as_slice(), CoinVariant::Standard); - let maybe_block_headers = reader.read_list::(); - let block_headers = match maybe_block_headers { - Ok(headers) => headers, - Err(_) => return MmError::err(SPVError::MalformattedHeader), - }; - let mut block_registry: HashMap = HashMap::new(); - let mut starting_height = from; - for block_header in &block_headers { - block_registry.insert(starting_height, block_header.clone()); - starting_height += 1; - } - (block_registry, block_headers) - }, - Err(_) => return MmError::err(SPVError::UnableToGetHeader), - }; - Ok((block_registry, block_headers)) -}*/ - macro_rules! try_loop { ($e:expr, $delay: ident) => { match $e { From a341fed2efc6ebac0dc0b57ac364f3440b3ba94e Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 4 Mar 2022 09:58:05 +0100 Subject: [PATCH 58/74] feat(spv): next batch of fixes --- mm2src/coins/utxo.rs | 15 +++++----- .../coins/utxo/utxo_block_header_storage.rs | 4 ++- .../utxo/utxo_builder/utxo_arc_builder.rs | 30 +++++++++---------- .../utxo/utxo_builder/utxo_coin_builder.rs | 13 ++++---- mm2src/coins/utxo/utxo_common.rs | 26 ++++++++-------- .../utxo/utxo_wrapper_block_header_storage.rs | 7 +++-- 6 files changed, 52 insertions(+), 43 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 8fa02e94c8..da2329e69d 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -35,14 +35,6 @@ pub mod utxo_common; pub mod utxo_standard; pub mod utxo_withdraw; -#[cfg(not(target_arch = "wasm32"))] pub mod tx_cache; -#[cfg(target_arch = "wasm32")] -pub mod utxo_indexedb_block_header_storage; -#[cfg(not(target_arch = "wasm32"))] -pub mod utxo_sql_block_header_storage; -pub mod utxo_wrapper_block_header_storage; -use utxo_block_header_storage::BlockHeaderStorage; - use async_trait::async_trait; use bigdecimal::BigDecimal; use bitcoin::network::constants::Network as BitcoinNetwork; @@ -103,6 +95,13 @@ use super::{BalanceError, BalanceFut, BalanceResult, CoinsContext, DerivationMet use crate::coin_balance::HDAddressBalanceChecker; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; +use utxo_block_header_storage::BlockHeaderStorage; +#[cfg(not(target_arch = "wasm32"))] pub mod tx_cache; +#[cfg(target_arch = "wasm32")] +pub mod utxo_indexedb_block_header_storage; +#[cfg(not(target_arch = "wasm32"))] +pub mod utxo_sql_block_header_storage; +pub mod utxo_wrapper_block_header_storage; #[cfg(test)] pub mod utxo_tests; #[cfg(target_arch = "wasm32")] pub mod utxo_wasm_tests; diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 7777c3b829..6bbfa09283 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,4 +1,5 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; +use crate::utxo::UtxoBlockHeaderVerificationParams; use async_trait::async_trait; use chain::BlockHeader; use common::mm_ctx::MmArc; @@ -40,6 +41,7 @@ pub enum BlockHeaderStorageError { pub struct BlockHeaderStorage { pub inner: Box, + pub params: UtxoBlockHeaderVerificationParams, } impl Debug for BlockHeaderStorage { @@ -47,7 +49,7 @@ impl Debug for BlockHeaderStorage { } pub trait InitBlockHeaderStorageOps: Send + Sync + 'static { - fn new_from_ctx(ctx: MmArc) -> Option + fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option where Self: Sized; } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index f8010a8915..4c13fd3800 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -1,10 +1,11 @@ use crate::hd_pubkey::HDXPubExtractor; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::block_header_utxo_loop; use crate::utxo::utxo_common::merge_utxo_loop; -use crate::utxo::{UtxoArc, UtxoBlockHeaderVerificationParams, UtxoCoinFields, UtxoCommonOps, UtxoWeak}; +use crate::utxo::{UtxoArc, UtxoCoinFields, UtxoCommonOps, UtxoWeak}; use crate::{PrivKeyBuildPolicy, UtxoActivationParams}; use async_trait::async_trait; use common::executor::spawn; @@ -99,13 +100,16 @@ where async fn build(self) -> MmResult { let utxo = self.build_utxo_fields().await?; - let utxo_block_header_params = utxo.conf.block_header_storage_params.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required(utxo_weak, utxo_block_header_params, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required( + utxo_weak, + &result_coin.as_ref().block_headers_storage, + self.constructor.clone(), + ); Ok(result_coin) } } @@ -183,13 +187,16 @@ where async fn build(self) -> MmResult { let utxo = self.build_utxo_fields_with_iguana_priv_key(self.priv_key()).await?; - let utxo_block_header_params = utxo.conf.block_header_storage_params.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required(utxo_weak, utxo_block_header_params, self.constructor.clone()); + self.spawn_block_header_utxo_loop_if_required( + utxo_weak, + &result_coin.as_ref().block_headers_storage, + self.constructor.clone(), + ); Ok(result_coin) } } @@ -247,20 +254,13 @@ where fn spawn_block_header_utxo_loop_if_required( &self, weak: UtxoWeak, - utxo_header_params: Option, + maybe_storage: &Option, constructor: F, ) where F: Fn(UtxoArc) -> T + Send + Sync + 'static, { - if let Some(block_header_verification_params) = utxo_header_params { - let fut = block_header_utxo_loop( - weak, - block_header_verification_params.check_every, - block_header_verification_params.difficulty_check, - block_header_verification_params.constant_difficulty, - block_header_verification_params.blocks_limit_to_check, - constructor, - ); + if maybe_storage.is_some() { + let fut = block_header_utxo_loop(weak, constructor); info!("Starting UTXO block header loop for coin {}", self.ticker()); spawn(fut); } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 1c12f5c327..cf507789d9 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -182,10 +182,7 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { let tx_cache_directory = Some(self.ctx().dbdir().join("TX_CACHE")); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); - let block_headers_storage = match conf.block_header_storage_params { - None => None, - Some(_) => BlockHeaderStorage::new_from_ctx(self.ctx().clone()), - }; + let block_headers_storage = self.block_headers_storage(); let coin = UtxoCoinFields { conf, @@ -241,6 +238,7 @@ where let tx_cache_directory = Some(self.ctx().dbdir().join("TX_CACHE")); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); + let block_headers_storage = self.block_headers_storage(); let coin = UtxoCoinFields { conf, @@ -250,7 +248,7 @@ where priv_key_policy: PrivKeyPolicy::HardwareWallet, derivation_method: DerivationMethod::HDWallet(hd_wallet), history_sync_state: Mutex::new(initial_history_state), - block_headers_storage: BlockHeaderStorage::new_from_ctx(self.ctx().clone()), + block_headers_storage, tx_cache_directory, recently_spent_outpoints, tx_fee, @@ -322,6 +320,11 @@ pub trait UtxoCoinBuilderCommonOps { fn ticker(&self) -> &str; + fn block_headers_storage(&self) -> Option { + let params = json::from_value(self.conf()["block_header_params"].clone()).ok()?; + BlockHeaderStorage::new_from_ctx(self.ctx().clone(), params) + } + fn address_format(&self) -> UtxoCoinBuildResult { let format_from_req = self.activation_params().address_format.clone(); let format_from_conf = json::from_value::>(self.conf()["address_format"].clone()) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index f647331477..ca5229dcf8 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3341,14 +3341,8 @@ macro_rules! try_loop { }; } -pub async fn block_header_utxo_loop( - weak: UtxoWeak, - check_every: f64, - difficulty_check: bool, - constant_difficulty: bool, - blocks_limit_to_check: u64, - constructor: impl Fn(UtxoArc) -> T, -) where +pub async fn block_header_utxo_loop(weak: UtxoWeak, constructor: impl Fn(UtxoArc) -> T) +where T: AsRef + UtxoCommonOps, { { @@ -3385,6 +3379,17 @@ pub async fn block_header_utxo_loop( } while let Some(arc) = weak.upgrade() { let coin = constructor(arc); + let storage = match &coin.as_ref().block_headers_storage { + None => break, + Some(storage) => storage, + }; + let params = storage.params.clone(); + let (check_every, blocks_limit_to_check, difficulty_check, constant_difficulty) = ( + params.check_every, + params.blocks_limit_to_check, + params.difficulty_check, + params.constant_difficulty, + ); let height = try_loop!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => break, @@ -3401,10 +3406,7 @@ pub async fn block_header_utxo_loop( validate_headers(block_headers, difficulty_check, constant_difficulty), check_every ); - let storage = match &coin.as_ref().block_headers_storage { - None => break, - Some(storage) => storage, - }; + let ticker = coin.as_ref().conf.ticker.as_str(); try_loop!( storage.add_block_headers_to_storage(ticker, block_registry).await, diff --git a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs index 0e60b758cd..1445addb8a 100644 --- a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs @@ -5,6 +5,7 @@ use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderStor use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; #[cfg(not(target_arch = "wasm32"))] use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; +use crate::utxo::UtxoBlockHeaderVerificationParams; use async_trait::async_trait; use chain::BlockHeader; use common::mm_ctx::MmArc; @@ -13,16 +14,18 @@ use std::collections::HashMap; impl InitBlockHeaderStorageOps for BlockHeaderStorage { #[cfg(not(target_arch = "wasm32"))] - fn new_from_ctx(ctx: MmArc) -> Option { + fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { ctx.sqlite_connection.as_option().map(|connection| BlockHeaderStorage { inner: Box::new(SqliteBlockHeadersStorage(connection.clone())), + params, }) } #[cfg(target_arch = "wasm32")] - fn new_from_ctx(_ctx: MmArc) -> Option { + fn new_from_ctx(_ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { Some(BlockHeaderStorage { inner: Box::new(IndexedDBBlockHeadersStorage {}), + params, }) } } From b99859188b236ea177dd1efd85f22e29c167cfb4 Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 4 Mar 2022 10:16:25 +0100 Subject: [PATCH 59/74] feat(review): remove utxo_wrapper_block_header_storage.rs --- mm2src/coins/utxo.rs | 1 - mm2src/coins/utxo/rpc_clients.rs | 8 +- .../coins/utxo/utxo_block_header_storage.rs | 67 ++++++++++++++++ mm2src/coins/utxo/utxo_common.rs | 16 ++-- .../utxo/utxo_wrapper_block_header_storage.rs | 76 ------------------- 5 files changed, 81 insertions(+), 87 deletions(-) delete mode 100644 mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index da2329e69d..23f89f1ddc 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -101,7 +101,6 @@ use utxo_block_header_storage::BlockHeaderStorage; pub mod utxo_indexedb_block_header_storage; #[cfg(not(target_arch = "wasm32"))] pub mod utxo_sql_block_header_storage; -pub mod utxo_wrapper_block_header_storage; #[cfg(test)] pub mod utxo_tests; #[cfg(target_arch = "wasm32")] pub mod utxo_wasm_tests; diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 220d67f92c..6b1f837b94 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1670,16 +1670,16 @@ impl ElectrumClient { pub fn retrieve_last_headers( &self, - blocks_limit_to_check: u64, + blocks_limit_to_check: NonZeroU64, block_height: u64, ) -> UtxoRpcFut<(HashMap, Vec)> { let (from, count) = { - let from = if block_height < blocks_limit_to_check { + let from = if block_height < blocks_limit_to_check.get() { 0 } else { - block_height - blocks_limit_to_check + block_height - blocks_limit_to_check.get() }; - (from, NonZeroU64::new(blocks_limit_to_check).unwrap()) + (from, blocks_limit_to_check) }; Box::new( self.blockchain_block_headers(from, count) diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 6bbfa09283..3cb802a499 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,4 +1,8 @@ use crate::utxo::rpc_clients::ElectrumBlockHeader; +#[cfg(target_arch = "wasm32")] +use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; +#[cfg(not(target_arch = "wasm32"))] +use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; use crate::utxo::UtxoBlockHeaderVerificationParams; use async_trait::async_trait; use chain::BlockHeader; @@ -93,3 +97,66 @@ pub trait BlockHeaderStorageOps: Send + Sync + 'static { height: u64, ) -> Result, MmError>; } + +impl InitBlockHeaderStorageOps for BlockHeaderStorage { + #[cfg(not(target_arch = "wasm32"))] + fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { + ctx.sqlite_connection.as_option().map(|connection| BlockHeaderStorage { + inner: Box::new(SqliteBlockHeadersStorage(connection.clone())), + params, + }) + } + + #[cfg(target_arch = "wasm32")] + fn new_from_ctx(_ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { + Some(BlockHeaderStorage { + inner: Box::new(IndexedDBBlockHeadersStorage {}), + params, + }) + } +} + +#[async_trait] +impl BlockHeaderStorageOps for BlockHeaderStorage { + async fn init(&self, for_coin: &str) -> Result<(), MmError> { + self.inner.init(for_coin).await + } + + async fn is_initialized_for(&self, for_coin: &str) -> Result> { + self.inner.is_initialized_for(for_coin).await + } + + async fn add_electrum_block_headers_to_storage( + &self, + for_coin: &str, + headers: Vec, + ) -> Result<(), MmError> { + self.inner + .add_electrum_block_headers_to_storage(for_coin, headers) + .await + } + + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: HashMap, + ) -> Result<(), MmError> { + self.inner.add_block_headers_to_storage(for_coin, headers).await + } + + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError> { + self.inner.get_block_header(for_coin, height).await + } + + async fn get_block_header_raw( + &self, + for_coin: &str, + height: u64, + ) -> Result, MmError> { + self.inner.get_block_header_raw(for_coin, height).await + } +} diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index ca5229dcf8..6517e9a99d 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3303,10 +3303,10 @@ where match header_params { None => Ok((header, false)), Some(params) => { - let (headers_registry, headers) = client - .retrieve_last_headers(params.blocks_limit_to_check, height) - .compat() - .await?; + let blocks_limit = NonZeroU64::new(params.blocks_limit_to_check) + .ok_or_else(|| GetBlockHeaderError::Internal("invalid block limit to check".to_string()))?; + let (headers_registry, headers) = + client.retrieve_last_headers(blocks_limit, height).compat().await?; match spv_validation::helpers_validation::validate_headers( headers, params.difficulty_check, @@ -3384,12 +3384,16 @@ where Some(storage) => storage, }; let params = storage.params.clone(); - let (check_every, blocks_limit_to_check, difficulty_check, constant_difficulty) = ( + let (check_every, maybe_blocks_limit_to_check, difficulty_check, constant_difficulty) = ( params.check_every, - params.blocks_limit_to_check, + NonZeroU64::new(params.blocks_limit_to_check), params.difficulty_check, params.constant_difficulty, ); + let blocks_limit_to_check = match maybe_blocks_limit_to_check { + None => break, + Some(blocks_limit_to_check) => blocks_limit_to_check, + }; let height = try_loop!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => break, diff --git a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs b/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs deleted file mode 100644 index 1445addb8a..0000000000 --- a/mm2src/coins/utxo/utxo_wrapper_block_header_storage.rs +++ /dev/null @@ -1,76 +0,0 @@ -use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderStorageError, BlockHeaderStorageOps, - InitBlockHeaderStorageOps}; -#[cfg(target_arch = "wasm32")] -use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; -#[cfg(not(target_arch = "wasm32"))] -use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; -use crate::utxo::UtxoBlockHeaderVerificationParams; -use async_trait::async_trait; -use chain::BlockHeader; -use common::mm_ctx::MmArc; -use common::mm_error::MmError; -use std::collections::HashMap; - -impl InitBlockHeaderStorageOps for BlockHeaderStorage { - #[cfg(not(target_arch = "wasm32"))] - fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { - ctx.sqlite_connection.as_option().map(|connection| BlockHeaderStorage { - inner: Box::new(SqliteBlockHeadersStorage(connection.clone())), - params, - }) - } - - #[cfg(target_arch = "wasm32")] - fn new_from_ctx(_ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { - Some(BlockHeaderStorage { - inner: Box::new(IndexedDBBlockHeadersStorage {}), - params, - }) - } -} - -#[async_trait] -impl BlockHeaderStorageOps for BlockHeaderStorage { - async fn init(&self, for_coin: &str) -> Result<(), MmError> { - self.inner.init(for_coin).await - } - - async fn is_initialized_for(&self, for_coin: &str) -> Result> { - self.inner.is_initialized_for(for_coin).await - } - - async fn add_electrum_block_headers_to_storage( - &self, - for_coin: &str, - headers: Vec, - ) -> Result<(), MmError> { - self.inner - .add_electrum_block_headers_to_storage(for_coin, headers) - .await - } - - async fn add_block_headers_to_storage( - &self, - for_coin: &str, - headers: HashMap, - ) -> Result<(), MmError> { - self.inner.add_block_headers_to_storage(for_coin, headers).await - } - - async fn get_block_header( - &self, - for_coin: &str, - height: u64, - ) -> Result, MmError> { - self.inner.get_block_header(for_coin, height).await - } - - async fn get_block_header_raw( - &self, - for_coin: &str, - height: u64, - ) -> Result, MmError> { - self.inner.get_block_header_raw(for_coin, height).await - } -} From 36999e56fc2837b641175b38e5a5579fea0a8a39 Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 4 Mar 2022 10:54:00 +0100 Subject: [PATCH 60/74] feat(review): next batche of fixes --- mm2src/coins/utxo/utxo_common.rs | 89 ++++++++----------- .../utxo/utxo_sql_block_header_storage.rs | 51 ++++++----- mm2src/mm2_bitcoin/chain/src/raw_block.rs | 10 ++- .../mm2_bitcoin/spv_validation/src/types.rs | 2 +- 4 files changed, 69 insertions(+), 83 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 6517e9a99d..b81989a8a7 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2911,17 +2911,13 @@ where return MmError::err(SPVError::InvalidVout); } let height = get_tx_height(&tx, client).await?; - let (block_header, is_validated) = block_header_from_storage_or_rpc( - &coin, - height, - &coin.as_ref().block_headers_storage, - coin.as_ref().conf.block_header_storage_params.clone(), - ) - .await - .map_err(|_e| SPVError::UnableToGetHeader)?; + let (block_header, is_validated) = + block_header_from_storage_or_rpc(&coin, height, &coin.as_ref().block_headers_storage) + .await + .map_err(|_e| SPVError::UnableToGetHeader)?; // Only specific chains will use the block header storage, for example QTUM will not - // Check block header verification only for chains that have storage or storage params. - if !is_validated && coin.as_ref().conf.block_header_storage_params.is_some() { + // Check block header verification only for chains that have storage. + if !is_validated && coin.as_ref().block_headers_storage.is_some() { return MmError::err(SPVError::BlockHeaderNotVerified); } let raw_header = RawBlockHeader::new(block_header.raw().take())?; @@ -3274,7 +3270,6 @@ pub async fn block_header_from_storage_or_rpc( coin: &T, height: u64, storage: &Option, - header_params: Option, ) -> Result<(BlockHeader, bool), MmError> where T: AsRef, @@ -3300,27 +3295,22 @@ where None => { let bytes = client.blockchain_block_header(height).compat().await?; let header: BlockHeader = deserialize(bytes.0.as_slice())?; - match header_params { - None => Ok((header, false)), - Some(params) => { - let blocks_limit = NonZeroU64::new(params.blocks_limit_to_check) - .ok_or_else(|| GetBlockHeaderError::Internal("invalid block limit to check".to_string()))?; - let (headers_registry, headers) = - client.retrieve_last_headers(blocks_limit, height).compat().await?; - match spv_validation::helpers_validation::validate_headers( - headers, - params.difficulty_check, - params.constant_difficulty, - ) { - Ok(_) => { - storage - .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), headers_registry) - .await?; - Ok((header, true)) - }, - Err(_) => Ok((header, false)), - } + let params = storage.params.clone(); + let blocks_limit = NonZeroU64::new(params.blocks_limit_to_check) + .ok_or_else(|| GetBlockHeaderError::Internal("invalid block limit to check".to_string()))?; + let (headers_registry, headers) = client.retrieve_last_headers(blocks_limit, height).compat().await?; + match spv_validation::helpers_validation::validate_headers( + headers, + params.difficulty_check, + params.constant_difficulty, + ) { + Ok(_) => { + storage + .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), headers_registry) + .await?; + Ok((header, true)) }, + Err(_) => Ok((header, false)), } }, Some(header) => Ok((header, true)), @@ -3328,7 +3318,7 @@ where } } -macro_rules! try_loop { +macro_rules! try_loop_with_sleep { ($e:expr, $delay: ident) => { match $e { Ok(res) => res, @@ -3356,26 +3346,19 @@ where Some(storage) => storage, }; match storage.is_initialized_for(ticker).await { - Ok(is_init) => { - if !is_init { - match storage.init(ticker).await { - Ok(()) => { - info!("Block Header Storage successfully initialized for {}", ticker); - }, - Err(e) => { - error!( - "Couldn't initiate storage - aborting the block_header_utxo_loop: {:?}", - e - ); - return; - }, - } - } else { - info!("Block Header Storage already initialized for {}", ticker); + Ok(true) => info!("Block Header Storage already initialized for {}", ticker), + Ok(false) => { + if let Err(e) = storage.init(ticker).await { + error!( + "Couldn't initiate storage - aborting the block_header_utxo_loop: {:?}", + e + ); + return; } + info!("Block Header Storage successfully initialized for {}", ticker); }, Err(_e) => return, - } + }; } while let Some(arc) = weak.upgrade() { let coin = constructor(arc); @@ -3394,25 +3377,25 @@ where None => break, Some(blocks_limit_to_check) => blocks_limit_to_check, }; - let height = try_loop!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); + let height = try_loop_with_sleep!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => break, UtxoRpcClientEnum::Electrum(client) => client, }; - let (block_registry, block_headers) = try_loop!( + let (block_registry, block_headers) = try_loop_with_sleep!( client .retrieve_last_headers(blocks_limit_to_check, height) .compat() .await, check_every ); - try_loop!( + try_loop_with_sleep!( validate_headers(block_headers, difficulty_check, constant_difficulty), check_every ); let ticker = coin.as_ref().conf.ticker.as_str(); - try_loop!( + try_loop_with_sleep!( storage.add_block_headers_to_storage(ticker, block_registry).await, check_every ); diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index c133d1635d..d94e7ea3e2 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -15,13 +15,17 @@ use std::sync::{Arc, Mutex}; fn block_headers_cache_table(ticker: &str) -> String { ticker.to_owned() + "_block_headers_cache" } -fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { +fn get_table_name_and_validate(for_coin: &str) -> Result> { let table_name = block_headers_cache_table(for_coin); validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { ticker: for_coin.to_string(), reason: e.to_string(), })?; + Ok(table_name) +} +fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { + let table_name = get_table_name_and_validate(for_coin)?; let sql = "CREATE TABLE IF NOT EXISTS ".to_owned() + &table_name + " ( @@ -33,24 +37,14 @@ fn create_block_header_cache_table_sql(for_coin: &str) -> Result Result> { - let table_name = block_headers_cache_table(for_coin); - validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - + let table_name = get_table_name_and_validate(for_coin)?; // We can simply ignore the repetitive attempt to insert the same block_height let sql = "INSERT OR IGNORE INTO ".to_owned() + &table_name + " (block_height, hex) VALUES (?1, ?2);"; Ok(sql) } fn get_block_header_by_height(for_coin: &str) -> Result> { - let table_name = block_headers_cache_table(for_coin); - validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - + let table_name = get_table_name_and_validate(for_coin)?; let sql = "SELECT hex FROM ".to_owned() + &table_name + " WHERE block_height=?1;"; Ok(sql) @@ -78,7 +72,10 @@ where }) } -struct SqlBlockHeader(String, String); +struct SqlBlockHeader { + block_height: String, + block_hex: String, +} impl From for SqlBlockHeader { fn from(header: ElectrumBlockHeader) -> Self { @@ -86,12 +83,18 @@ impl From for SqlBlockHeader { ElectrumBlockHeader::V12(h) => { let block_hex = h.as_hex(); let block_height = h.block_height.to_string(); - SqlBlockHeader(block_height, block_hex) + SqlBlockHeader { + block_height, + block_hex, + } }, ElectrumBlockHeader::V14(h) => { let block_hex = format!("{:02x}", h.hex); let block_height = h.height.to_string(); - SqlBlockHeader(block_height, block_hex) + SqlBlockHeader { + block_height, + block_hex, + } }, } } @@ -110,7 +113,7 @@ async fn common_headers_insert( reason: e.to_string(), })?; for header in headers { - let block_cache_params = [&header.0, &header.1]; + let block_cache_params = [&header.block_height, &header.block_hex]; sql_transaction .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) .map_err(|e| BlockHeaderStorageError::ExecutionError { @@ -147,14 +150,7 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { } async fn is_initialized_for(&self, for_coin: &str) -> Result> { - let block_headers_cache_table = block_headers_cache_table(for_coin); - validate_table_name(&block_headers_cache_table).map_err(|e| { - BlockHeaderStorageError::CantRetrieveTableError { - ticker: for_coin.to_string(), - reason: e.to_string(), - } - })?; - + let block_headers_cache_table = get_table_name_and_validate(for_coin)?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -185,7 +181,10 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { ) -> Result<(), MmError> { let headers_for_sql = headers .into_iter() - .map(|(height, header)| SqlBlockHeader(height.to_string(), hex::encode(header.raw()))) + .map(|(height, header)| SqlBlockHeader { + block_height: height.to_string(), + block_hex: hex::encode(header.raw()), + }) .collect(); common_headers_insert(for_coin, self.clone(), headers_for_sql).await } diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index 5841af95f3..075d15ad6a 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -4,19 +4,23 @@ use primitives::hash::H256; use ser::serialize; use BlockHeader; +pub const MIN_RAW_HEADER_SIZE: usize = 80_usize; + /// Hex-encoded block #[derive(Default, PartialEq, Clone, Eq, Hash)] pub struct RawBlockHeader(Bytes); #[derive(Debug, PartialEq, Eq, Clone)] pub enum RawHeaderError { - WrongLengthHeader, + WrongLengthHeader { min_length: usize }, } impl RawBlockHeader { pub fn new(buf: Vec) -> Result { - if buf.len() < 80 { - return Err(RawHeaderError::WrongLengthHeader); + if buf.len() < MIN_RAW_HEADER_SIZE { + return Err(RawHeaderError::WrongLengthHeader { + min_length: MIN_RAW_HEADER_SIZE, + }); } let header = RawBlockHeader(buf.into()); Ok(header) diff --git a/mm2src/mm2_bitcoin/spv_validation/src/types.rs b/mm2src/mm2_bitcoin/spv_validation/src/types.rs index cffb444c8a..4840f7556b 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/types.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/types.rs @@ -91,7 +91,7 @@ impl From for SPVError { impl From for SPVError { fn from(e: RawHeaderError) -> Self { match e { - RawHeaderError::WrongLengthHeader => SPVError::WrongLengthHeader, + RawHeaderError::WrongLengthHeader { .. } => SPVError::WrongLengthHeader, } } } From dfd4bd4d2bb6f11d052ea14e3724a875cd87f64c Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 4 Mar 2022 12:49:38 +0100 Subject: [PATCH 61/74] feat(review): refactor block_header_from_storage_or_rpc --- mm2src/coins/utxo/utxo_common.rs | 89 +++++++++++++++++--------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index b81989a8a7..f4f9faaef5 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -2911,15 +2911,9 @@ where return MmError::err(SPVError::InvalidVout); } let height = get_tx_height(&tx, client).await?; - let (block_header, is_validated) = - block_header_from_storage_or_rpc(&coin, height, &coin.as_ref().block_headers_storage) - .await - .map_err(|_e| SPVError::UnableToGetHeader)?; - // Only specific chains will use the block header storage, for example QTUM will not - // Check block header verification only for chains that have storage. - if !is_validated && coin.as_ref().block_headers_storage.is_some() { - return MmError::err(SPVError::BlockHeaderNotVerified); - } + let block_header = block_header_from_storage_or_rpc(&coin, height, &coin.as_ref().block_headers_storage) + .await + .map_err(|_e| SPVError::UnableToGetHeader)?; let raw_header = RawBlockHeader::new(block_header.raw().take())?; let merkle_branch = client @@ -3266,11 +3260,49 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } +pub async fn valid_block_header_from_storage( + coin: &T, + height: u64, + storage: &BlockHeaderStorage, + client: &ElectrumClient, +) -> Result> +where + T: AsRef, +{ + match storage + .get_block_header(coin.as_ref().conf.ticker.as_str(), height) + .await? + { + None => { + let bytes = client.blockchain_block_header(height).compat().await?; + let header: BlockHeader = deserialize(bytes.0.as_slice())?; + let params = storage.params.clone(); + let blocks_limit = NonZeroU64::new(params.blocks_limit_to_check) + .ok_or_else(|| GetBlockHeaderError::Internal("invalid block limit to check".to_string()))?; + let (headers_registry, headers) = client.retrieve_last_headers(blocks_limit, height).compat().await?; + match spv_validation::helpers_validation::validate_headers( + headers, + params.difficulty_check, + params.constant_difficulty, + ) { + Ok(_) => { + storage + .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), headers_registry) + .await?; + Ok(header) + }, + Err(err) => MmError::err(GetBlockHeaderError::SPVError(err)), + } + }, + Some(header) => Ok(header), + } +} + pub async fn block_header_from_storage_or_rpc( coin: &T, height: u64, storage: &Option, -) -> Result<(BlockHeader, bool), MmError> +) -> Result> where T: AsRef, { @@ -3282,39 +3314,12 @@ where }, UtxoRpcClientEnum::Electrum(client) => client, }; + match storage { - None => { - let bytes = client.blockchain_block_header(height).compat().await?; - let header: BlockHeader = deserialize(bytes.0.as_slice())?; - Ok((header, false)) - }, - Some(storage) => match storage - .get_block_header(coin.as_ref().conf.ticker.as_str(), height) - .await? - { - None => { - let bytes = client.blockchain_block_header(height).compat().await?; - let header: BlockHeader = deserialize(bytes.0.as_slice())?; - let params = storage.params.clone(); - let blocks_limit = NonZeroU64::new(params.blocks_limit_to_check) - .ok_or_else(|| GetBlockHeaderError::Internal("invalid block limit to check".to_string()))?; - let (headers_registry, headers) = client.retrieve_last_headers(blocks_limit, height).compat().await?; - match spv_validation::helpers_validation::validate_headers( - headers, - params.difficulty_check, - params.constant_difficulty, - ) { - Ok(_) => { - storage - .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), headers_registry) - .await?; - Ok((header, true)) - }, - Err(_) => Ok((header, false)), - } - }, - Some(header) => Ok((header, true)), - }, + Some(ref storage) => valid_block_header_from_storage(&coin, height, storage, client).await, + None => Ok(deserialize( + client.blockchain_block_header(height).compat().await?.as_slice(), + )?), } } From dfbfceb4b43b67993020992b319c3a5a40cdc410 Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 9 Mar 2022 10:01:54 +0100 Subject: [PATCH 62/74] feat(wasm): use explicitly instant wasm bindgen --- Cargo.lock | 4 ++++ Cargo.toml | 1 + 2 files changed, 5 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index a95d8e2339..7b459d154a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2480,6 +2480,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -3396,6 +3399,7 @@ dependencies = [ "http 0.2.1", "hw_common", "hyper", + "instant", "itertools 0.9.0", "js-sys", "keys", diff --git a/Cargo.toml b/Cargo.toml index 5b5a3da891..3ab42fb392 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,6 +125,7 @@ wasm-bindgen = { version = "0.2.50", features = ["serde-serialize", "nightly"] } wasm-bindgen-futures = { version = "0.4.1" } wasm-bindgen-test = { version = "0.3.1" } web-sys = { version = "0.3.55", features = ["console"] } +instant = {version = "0.1.12", features = ["wasm-bindgen"]} [target.'cfg(not(target_arch = "wasm32"))'.dependencies] dirs = { version = "1" } From 291b76e400565049d0b1ddf51969f10198d3cfef Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 11 Mar 2022 08:18:20 +0100 Subject: [PATCH 63/74] feat(spv): fix wasm tests --- mm2src/coins/lp_coins.rs | 1 + mm2src/coins/qrc20/qrc20_tests.rs | 2 ++ mm2src/coins/utxo/slp.rs | 21 ++++++++++++++++++--- mm2src/coins/utxo/utxo_common.rs | 8 +++++++- mm2src/docker_tests/qrc20_tests.rs | 2 ++ mm2src/lp_swap/maker_swap.rs | 4 +++- mm2src/lp_swap/taker_swap.rs | 4 +++- 7 files changed, 36 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index d124278286..670be59bcd 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -244,6 +244,7 @@ pub struct ValidatePaymentInput { pub secret_hash: Vec, pub amount: BigDecimal, pub swap_contract_address: Option, + pub confirmations: u64, } /// Swap operations (mostly based on the Hash/Time locked transactions implemented by coin wallets). diff --git a/mm2src/coins/qrc20/qrc20_tests.rs b/mm2src/coins/qrc20/qrc20_tests.rs index c34e7e018b..41790cb4ab 100644 --- a/mm2src/coins/qrc20/qrc20_tests.rs +++ b/mm2src/coins/qrc20/qrc20_tests.rs @@ -131,6 +131,7 @@ fn test_validate_maker_payment() { secret_hash: vec![1; 20], amount: correct_amount.clone(), swap_contract_address: coin.swap_contract_address(), + confirmations: 1, }; coin.validate_maker_payment(input.clone()).wait().unwrap(); @@ -851,6 +852,7 @@ fn test_validate_maker_payment_malicious() { secret_hash, amount, swap_contract_address: coin.swap_contract_address(), + confirmations: 1, }; let error = coin .validate_maker_payment(input) diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index 3e4465ddf9..f1a0a45f0f 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -433,6 +433,7 @@ impl SlpToken { .await } + #[allow(clippy::too_many_arguments)] async fn validate_htlc( &self, tx: &[u8], @@ -441,6 +442,7 @@ impl SlpToken { time_lock: u32, secret_hash: &[u8], amount: BigDecimal, + confirmations: u64, ) -> Result<(), MmError> { let mut tx: UtxoTx = deserialize(tx).map_to_mm(ValidateHtlcError::TxParseError)?; tx.tx_hash_algo = self.platform_coin.as_ref().tx_hash_algo; @@ -491,6 +493,7 @@ impl SlpToken { secret_hash, self.platform_dust_dec(), time_lock, + confirmations, ); validate_fut @@ -1342,12 +1345,21 @@ impl SwapOps for SlpToken { let secret_hash = input.secret_hash.to_owned(); let time_lock = input.time_lock; let amount = input.amount; + let confirmations = input.confirmations; let coin = self.clone(); let fut = async move { try_s!( - coin.validate_htlc(&tx, &maker_pub, &taker_pub, time_lock, &secret_hash, amount) - .await + coin.validate_htlc( + &tx, + &maker_pub, + &taker_pub, + time_lock, + &secret_hash, + amount, + confirmations + ) + .await ); Ok(()) }; @@ -1357,6 +1369,7 @@ impl SwapOps for SlpToken { fn validate_taker_payment(&self, input: ValidatePaymentInput) -> Box + Send> { let taker_pub = try_fus!(Public::from_slice(&input.taker_pub)); let maker_pub = try_fus!(Public::from_slice(&input.maker_pub)); + let confirmations = input.confirmations; let coin = self.clone(); let fut = async move { @@ -1367,7 +1380,8 @@ impl SwapOps for SlpToken { &maker_pub, input.time_lock, &input.secret_hash, - input.amount + input.amount, + confirmations ) .await ); @@ -2040,6 +2054,7 @@ mod slp_tests { &secret_hash, fusd.platform_dust_dec(), lock_time, + 1, ) .wait() .unwrap(); diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index f4f9faaef5..0ed6bdf777 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -1383,6 +1383,7 @@ where &input.secret_hash, input.amount, input.time_lock, + input.confirmations, ) } @@ -1406,6 +1407,7 @@ where &input.secret_hash, input.amount, input.time_lock, + input.confirmations, ) } @@ -2963,6 +2965,7 @@ pub fn validate_payment( priv_bn_hash: &[u8], amount: BigDecimal, time_lock: u32, + confirmations: u64, ) -> Box + Send> where T: AsRef + Send + Sync + 'static, @@ -3018,7 +3021,10 @@ where expected_output ); } - return validate_spv_proof(coin, tx).await.map_err(|e| format!("{:?}", e)); + return match confirmations { + 0 => Ok(()), + _ => validate_spv_proof(coin, tx).await.map_err(|e| format!("{:?}", e)), + }; } }; Box::new(fut.boxed().compat()) diff --git a/mm2src/docker_tests/qrc20_tests.rs b/mm2src/docker_tests/qrc20_tests.rs index 85e5f10177..d45fa66de9 100644 --- a/mm2src/docker_tests/qrc20_tests.rs +++ b/mm2src/docker_tests/qrc20_tests.rs @@ -207,6 +207,7 @@ fn test_taker_spends_maker_payment() { secret_hash, amount: amount.clone(), swap_contract_address: taker_coin.swap_contract_address(), + confirmations, }; taker_coin.validate_maker_payment(input).wait().unwrap(); @@ -297,6 +298,7 @@ fn test_maker_spends_taker_payment() { secret_hash: secret_hash.clone(), amount: amount.clone(), swap_contract_address: maker_coin.swap_contract_address(), + confirmations, }; maker_coin.validate_taker_payment(input).wait().unwrap(); diff --git a/mm2src/lp_swap/maker_swap.rs b/mm2src/lp_swap/maker_swap.rs index ca84c482e5..72b9f8769e 100644 --- a/mm2src/lp_swap/maker_swap.rs +++ b/mm2src/lp_swap/maker_swap.rs @@ -767,12 +767,13 @@ impl MakerSwap { async fn validate_taker_payment(&self) -> Result<(Option, Vec), String> { let wait_duration = (self.r().data.lock_duration * 4) / 5; let wait_taker_payment = self.r().data.started_at + wait_duration; + let confirmations = self.r().data.taker_payment_confirmations; let wait_f = self .taker_coin .wait_for_confirmations( &self.r().taker_payment.clone().unwrap().tx_hex, - self.r().data.taker_payment_confirmations, + confirmations, self.r().data.taker_payment_requires_nota.unwrap_or(false), wait_taker_payment, WAIT_CONFIRM_INTERVAL, @@ -797,6 +798,7 @@ impl MakerSwap { secret_hash: dhash160(&self.r().data.secret.0).to_vec(), amount: self.taker_amount.clone(), swap_contract_address: self.r().data.taker_coin_swap_contract_address.clone(), + confirmations, }; let validated_f = self.taker_coin.validate_taker_payment(validate_input).compat(); diff --git a/mm2src/lp_swap/taker_swap.rs b/mm2src/lp_swap/taker_swap.rs index 3e32e0a08c..57aa0e5b38 100644 --- a/mm2src/lp_swap/taker_swap.rs +++ b/mm2src/lp_swap/taker_swap.rs @@ -1097,9 +1097,10 @@ impl TakerSwap { async fn validate_maker_payment(&self) -> Result<(Option, Vec), String> { log!({ "Before wait confirm" }); + let confirmations = self.r().data.maker_payment_confirmations; let f = self.maker_coin.wait_for_confirmations( &self.r().maker_payment.clone().unwrap().tx_hex, - self.r().data.maker_payment_confirmations, + confirmations, self.r().data.maker_payment_requires_nota.unwrap_or(false), self.r().data.maker_payment_wait, WAIT_CONFIRM_INTERVAL, @@ -1121,6 +1122,7 @@ impl TakerSwap { secret_hash: self.r().secret_hash.0.to_vec(), amount: self.maker_amount.to_decimal(), swap_contract_address: self.r().data.maker_coin_swap_contract_address.clone(), + confirmations, }; let validated = self.maker_coin.validate_maker_payment(validate_input).compat().await; From 0e38525c7d1692f5852cbd3cf5ecd04cf12ef34a Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 11 Mar 2022 09:11:06 +0100 Subject: [PATCH 64/74] feat(spv): fix unit test compilation --- mm2src/coins/utxo/slp.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index f1a0a45f0f..3f828b5075 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -1959,7 +1959,7 @@ mod slp_tests { let secret_hash = hex::decode("5d9e149ad9ccb20e9f931a69b605df2ffde60242").unwrap(); let amount = "0.1".parse().unwrap(); - block_on(fusd.validate_htlc(&tx, &other_pub, &my_pub, lock_time, &secret_hash, amount)).unwrap(); + block_on(fusd.validate_htlc(&tx, &other_pub, &my_pub, lock_time, &secret_hash, amount, 1)).unwrap(); } #[test] @@ -2060,7 +2060,7 @@ mod slp_tests { .unwrap(); let validity_err = - block_on(fusd.validate_htlc(&tx, &other_pub, my_pub, lock_time, &secret_hash, amount)).unwrap_err(); + block_on(fusd.validate_htlc(&tx, &other_pub, my_pub, lock_time, &secret_hash, amount, 1)).unwrap_err(); match validity_err.into_inner() { ValidateHtlcError::InvalidSlpUtxo(e) => println!("{:?}", e), err @ _ => panic!("Unexpected err {:?}", err), From ed97f351c5a0d14beab8d120647164bd7ca2571e Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 11 Mar 2022 17:09:57 +0100 Subject: [PATCH 65/74] feat(eth): ignore polygon unit test (unstable) --- mm2src/coins/eth/eth_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/coins/eth/eth_tests.rs b/mm2src/coins/eth/eth_tests.rs index 8d0f9fd6cd..120cdb93c3 100644 --- a/mm2src/coins/eth/eth_tests.rs +++ b/mm2src/coins/eth/eth_tests.rs @@ -1190,6 +1190,7 @@ fn test_negotiate_swap_contract_addr_has_fallback() { } #[test] +#[ignore] fn polygon_check_if_my_payment_sent() { let ctx = MmCtxBuilder::new().into_mm_arc(); let conf = json!({ From 6621d5b8b15bc0a0d7de08e8af2196949b8ea9e8 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 14 Mar 2022 10:57:54 +0100 Subject: [PATCH 66/74] feat(unit tests): increase to 6 seconds for the docker unit test --- mm2src/docker_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/docker_tests.rs b/mm2src/docker_tests.rs index 7984e3c0d3..098310089e 100644 --- a/mm2src/docker_tests.rs +++ b/mm2src/docker_tests.rs @@ -1529,7 +1529,7 @@ mod docker_tests { .unwrap(); // TODO when sell call is made immediately swap might be not put into swap ctx yet so locked // amount returns 0 - thread::sleep(Duration::from_secs(3)); + thread::sleep(Duration::from_secs(6)); let rc = block_on(mm_alice.rpc(json! ({ "userpass": mm_alice.userpass, From 27a4e8b94d7398a3c62d21a7c9fa6991c28c0105 Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 15 Mar 2022 11:26:38 +0100 Subject: [PATCH 67/74] feat(utxo): simplify block_headers_storage, add error --- .../coins/utxo/utxo_builder/utxo_coin_builder.rs | 14 +++++++++----- .../coins/utxo/utxo_builder/utxo_conf_builder.rs | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index cf507789d9..36f3775e3d 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -182,7 +182,7 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { let tx_cache_directory = Some(self.ctx().dbdir().join("TX_CACHE")); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); - let block_headers_storage = self.block_headers_storage(); + let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { conf, @@ -238,7 +238,7 @@ where let tx_cache_directory = Some(self.ctx().dbdir().join("TX_CACHE")); let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); - let block_headers_storage = self.block_headers_storage(); + let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { conf, @@ -320,9 +320,13 @@ pub trait UtxoCoinBuilderCommonOps { fn ticker(&self) -> &str; - fn block_headers_storage(&self) -> Option { - let params = json::from_value(self.conf()["block_header_params"].clone()).ok()?; - BlockHeaderStorage::new_from_ctx(self.ctx().clone(), params) + fn block_headers_storage(&self) -> UtxoCoinBuildResult> { + let params: Option<_> = json::from_value(self.conf()["block_header_params"].clone()) + .map_to_mm(|e| UtxoConfError::InvalidBlockHeaderParams(e.to_string()))?; + match params { + None => Ok(None), + Some(params) => Ok(BlockHeaderStorage::new_from_ctx(self.ctx().clone(), params)), + } } fn address_format(&self) -> UtxoCoinBuildResult { diff --git a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs index 769f0795ec..4f96e8d488 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs @@ -40,6 +40,7 @@ pub enum UtxoConfError { InvalidConsensusBranchId(String), InvalidVersionGroupId(String), InvalidAddressFormat(String), + InvalidBlockHeaderParams(String), InvalidDecimals(String), } From 863fcc2a207809218af3193a2222a31592e4b07b Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 18 Mar 2022 08:25:47 +0100 Subject: [PATCH 68/74] feat(review): batch of review fixes --- mm2src/coins/utxo.rs | 1 - mm2src/coins/utxo/slp.rs | 107 ++++++++++-------- .../coins/utxo/utxo_block_header_storage.rs | 3 +- .../utxo/utxo_builder/utxo_arc_builder.rs | 30 +++-- .../utxo/utxo_builder/utxo_conf_builder.rs | 2 - mm2src/coins/utxo/utxo_common.rs | 2 +- mm2src/coins/utxo/utxo_tests.rs | 1 - 7 files changed, 85 insertions(+), 61 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 23f89f1ddc..faaf4c02b7 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -478,7 +478,6 @@ pub struct UtxoCoinConf { /// Block count for median time past calculation pub mtp_block_count: NonZeroU64, pub estimate_fee_mode: Option, - pub block_header_storage_params: Option, /// The minimum number of confirmations at which a transaction is considered mature pub mature_confirmations: u32, /// The number of blocks used for estimate_fee/estimate_smart_fee RPC calls diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index 3f828b5075..2daefa556f 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -293,6 +293,17 @@ pub struct SlpProtocolConf { pub required_confirmations: Option, } +#[derive(Debug)] +pub struct ValidateHtlcInput { + tx: Vec, + other_pub: Public, + my_pub: Public, + time_lock: u32, + secret_hash: Vec, + amount: BigDecimal, + confirmations: u64, +} + impl SlpToken { pub fn new( decimals: u8, @@ -433,24 +444,14 @@ impl SlpToken { .await } - #[allow(clippy::too_many_arguments)] - async fn validate_htlc( - &self, - tx: &[u8], - other_pub: &Public, - my_pub: &Public, - time_lock: u32, - secret_hash: &[u8], - amount: BigDecimal, - confirmations: u64, - ) -> Result<(), MmError> { - let mut tx: UtxoTx = deserialize(tx).map_to_mm(ValidateHtlcError::TxParseError)?; + async fn validate_htlc(&self, input: ValidateHtlcInput) -> Result<(), MmError> { + let mut tx: UtxoTx = deserialize(input.tx.as_slice()).map_to_mm(ValidateHtlcError::TxParseError)?; tx.tx_hash_algo = self.platform_coin.as_ref().tx_hash_algo; if tx.outputs.len() < 2 { return MmError::err(ValidateHtlcError::TxLackOfOutputs); } - let slp_satoshis = sat_from_big_decimal(&amount, self.decimals())?; + let slp_satoshis = sat_from_big_decimal(&input.amount, self.decimals())?; let slp_unspent = SlpUnspent { bch_unspent: UnspentInfo { @@ -488,12 +489,12 @@ impl SlpToken { self.platform_coin.clone(), tx, SLP_SWAP_VOUT, - other_pub, - my_pub, - secret_hash, + &input.other_pub, + &input.my_pub, + &input.secret_hash, self.platform_dust_dec(), - time_lock, - confirmations, + input.time_lock, + input.confirmations, ); validate_fut @@ -1348,19 +1349,17 @@ impl SwapOps for SlpToken { let confirmations = input.confirmations; let coin = self.clone(); + let input = ValidateHtlcInput { + tx, + other_pub: maker_pub, + my_pub: taker_pub, + time_lock, + secret_hash, + amount, + confirmations, + }; let fut = async move { - try_s!( - coin.validate_htlc( - &tx, - &maker_pub, - &taker_pub, - time_lock, - &secret_hash, - amount, - confirmations - ) - .await - ); + try_s!(coin.validate_htlc(input).await); Ok(()) }; Box::new(fut.boxed().compat()) @@ -1372,19 +1371,17 @@ impl SwapOps for SlpToken { let confirmations = input.confirmations; let coin = self.clone(); + let input = ValidateHtlcInput { + tx: input.payment_tx, + other_pub: taker_pub, + my_pub: maker_pub, + time_lock: input.time_lock, + secret_hash: input.secret_hash, + amount: input.amount, + confirmations, + }; let fut = async move { - try_s!( - coin.validate_htlc( - &input.payment_tx, - &taker_pub, - &maker_pub, - input.time_lock, - &input.secret_hash, - input.amount, - confirmations - ) - .await - ); + try_s!(coin.validate_htlc(input).await); Ok(()) }; Box::new(fut.boxed().compat()) @@ -1957,9 +1954,17 @@ mod slp_tests { let lock_time = 1624547837; let secret_hash = hex::decode("5d9e149ad9ccb20e9f931a69b605df2ffde60242").unwrap(); - let amount = "0.1".parse().unwrap(); - - block_on(fusd.validate_htlc(&tx, &other_pub, &my_pub, lock_time, &secret_hash, amount, 1)).unwrap(); + let amount: BigDecimal = "0.1".parse().unwrap(); + let input = ValidateHtlcInput { + tx, + other_pub, + my_pub, + time_lock: lock_time, + secret_hash, + amount, + confirmations: 1, + }; + block_on(fusd.validate_htlc(input)).unwrap(); } #[test] @@ -2059,8 +2064,16 @@ mod slp_tests { .wait() .unwrap(); - let validity_err = - block_on(fusd.validate_htlc(&tx, &other_pub, my_pub, lock_time, &secret_hash, amount, 1)).unwrap_err(); + let input = ValidateHtlcInput { + tx, + other_pub, + my_pub: my_pub.clone(), + time_lock: lock_time, + secret_hash, + amount, + confirmations: 1, + }; + let validity_err = block_on(fusd.validate_htlc(input)).unwrap_err(); match validity_err.into_inner() { ValidateHtlcError::InvalidSlpUtxo(e) => println!("{:?}", e), err @ _ => panic!("Unexpected err {:?}", err), diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 3cb802a499..5d376566c2 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -12,8 +12,7 @@ use derive_more::Display; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; -#[derive(Debug, Deserialize, Display, Serialize, SerializeErrorType)] -#[serde(tag = "error_type", content = "error_data")] +#[derive(Debug, Display)] pub enum BlockHeaderStorageError { #[display(fmt = "The storage cannot be initialized for {}", _0)] InitializationError(String), diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 4c13fd3800..253625f235 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -12,6 +12,7 @@ use common::executor::spawn; use common::log::info; use common::mm_ctx::MmArc; use common::mm_error::prelude::*; +use futures::future::{abortable, AbortHandle}; use serde_json::Value as Json; pub struct UtxoArcBuilder<'a, F, T, XPubExtractor> @@ -105,11 +106,13 @@ where let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required( + if let Some(abort_handler) = self.spawn_block_header_utxo_loop_if_required( utxo_weak, &result_coin.as_ref().block_headers_storage, self.constructor.clone(), - ); + ) { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } Ok(result_coin) } } @@ -192,11 +195,13 @@ where let result_coin = (self.constructor)(utxo_arc); self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - self.spawn_block_header_utxo_loop_if_required( + if let Some(abort_handler) = self.spawn_block_header_utxo_loop_if_required( utxo_weak, &result_coin.as_ref().block_headers_storage, self.constructor.clone(), - ); + ) { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } Ok(result_coin) } } @@ -256,13 +261,24 @@ where weak: UtxoWeak, maybe_storage: &Option, constructor: F, - ) where + ) -> Option + where F: Fn(UtxoArc) -> T + Send + Sync + 'static, { if maybe_storage.is_some() { - let fut = block_header_utxo_loop(weak, constructor); + let ticker = self.ticker().to_owned(); + let (fut, abort_handle) = abortable(block_header_utxo_loop(weak, constructor)); info!("Starting UTXO block header loop for coin {}", self.ticker()); - spawn(fut); + spawn(async move { + if let Err(e) = fut.await { + info!( + "spawn_block_header_utxo_loop_if_required stopped for {}, reason {}", + ticker, e + ); + } + }); + return Some(abort_handle); } + None } } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs index 4f96e8d488..451ccc19e9 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs @@ -93,7 +93,6 @@ impl<'a> UtxoConfBuilder<'a> { let is_pos = self.is_pos(); let segwit = self.segwit(); let force_min_relay_fee = self.conf["force_min_relay_fee"].as_bool().unwrap_or(false); - let block_header_storage_params = json::from_value(self.conf["block_header_params"].clone()).unwrap_or(None); let mtp_block_count = self.mtp_block_count(); let estimate_fee_mode = self.estimate_fee_mode(); let estimate_fee_blocks = self.estimate_fee_blocks(); @@ -125,7 +124,6 @@ impl<'a> UtxoConfBuilder<'a> { force_min_relay_fee, mtp_block_count, estimate_fee_mode, - block_header_storage_params, mature_confirmations, estimate_fee_blocks, trezor_coin, diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 0ed6bdf777..59226594cf 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3282,7 +3282,7 @@ where None => { let bytes = client.blockchain_block_header(height).compat().await?; let header: BlockHeader = deserialize(bytes.0.as_slice())?; - let params = storage.params.clone(); + let params = &storage.params; let blocks_limit = NonZeroU64::new(params.blocks_limit_to_check) .ok_or_else(|| GetBlockHeaderError::Internal("invalid block limit to check".to_string()))?; let (headers_registry, headers) = client.retrieve_last_headers(blocks_limit, height).compat().await?; diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 2f7ee109d5..de2227d8e3 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -126,7 +126,6 @@ fn utxo_coin_fields_for_test( force_min_relay_fee: false, mtp_block_count: NonZeroU64::new(11).unwrap(), estimate_fee_mode: None, - block_header_storage_params: None, mature_confirmations: MATURE_CONFIRMATIONS_DEFAULT, estimate_fee_blocks: 1, trezor_coin: None, From 0d090943f259429d7a14680f36ac004ceddbe9d7 Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 18 Mar 2022 09:16:57 +0100 Subject: [PATCH 69/74] feat(review): another batch of review fixes --- mm2src/coins/eth/eth_tests.rs | 39 +++++++++++-------- mm2src/coins/utxo.rs | 2 +- .../coins/utxo/utxo_block_header_storage.rs | 10 +---- mm2src/coins/utxo/utxo_common.rs | 11 ++---- .../utxo/utxo_sql_block_header_storage.rs | 8 ++-- 5 files changed, 32 insertions(+), 38 deletions(-) diff --git a/mm2src/coins/eth/eth_tests.rs b/mm2src/coins/eth/eth_tests.rs index 120cdb93c3..428f9e42cb 100644 --- a/mm2src/coins/eth/eth_tests.rs +++ b/mm2src/coins/eth/eth_tests.rs @@ -1190,7 +1190,6 @@ fn test_negotiate_swap_contract_addr_has_fallback() { } #[test] -#[ignore] fn polygon_check_if_my_payment_sent() { let ctx = MmCtxBuilder::new().into_mm_arc(); let conf = json!({ @@ -1210,7 +1209,7 @@ fn polygon_check_if_my_payment_sent() { let request = json!({ "method": "enable", "coin": "MATIC", - "urls": ["https://polygon-rpc.com"], + "urls": ["https://polygon-mainnet.g.alchemy.com/v2/9YYl6iMLmXXLoflMPHnMTC4Dcm2L2tFH"], "swap_contract_address": "0x9130b257d37a52e52f21054c4da3450c72f595ce", }); @@ -1228,19 +1227,25 @@ fn polygon_check_if_my_payment_sent() { println!("{:02x}", coin.my_address); let secret_hash = hex::decode("fc33114b389f0ee1212abf2867e99e89126f4860").unwrap(); - let swap_contract_address = "9130b257d37a52e52f21054c4da3450c72f595ce".into(); - let my_payment = coin - .check_if_my_payment_sent( - 1638764369, - &[], - &[], - &secret_hash, - 22185109, - &Some(swap_contract_address), - ) - .wait() - .unwrap() - .unwrap(); - let expected_hash = BytesJson::from("69a20008cea0c15ee483b5bbdff942752634aa072dfd2ff715fe87eec302de11"); - assert_eq!(expected_hash, my_payment.tx_hash()); + + let mut i = 0; + while i < 10 { + let swap_contract_address = "9130b257d37a52e52f21054c4da3450c72f595ce".into(); + let my_payment = coin + .check_if_my_payment_sent( + 1638764369, + &[], + &[], + &secret_hash, + 22185109, + &Some(swap_contract_address), + ) + .wait() + .unwrap() + .unwrap(); + let expected_hash = BytesJson::from("69a20008cea0c15ee483b5bbdff942752634aa072dfd2ff715fe87eec302de11"); + assert_eq!(expected_hash, my_payment.tx_hash()); + Timer::sleep(5.0); + i = i + 1; + } } diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index faaf4c02b7..6984b7abea 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -1086,7 +1086,7 @@ pub struct UtxoMergeParams { pub struct UtxoBlockHeaderVerificationParams { pub difficulty_check: bool, pub constant_difficulty: bool, - pub blocks_limit_to_check: u64, + pub blocks_limit_to_check: NonZeroU64, pub check_every: f64, } diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 5d376566c2..3cb7ea0aef 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -30,14 +30,8 @@ pub enum BlockHeaderStorageError { CantRetrieveTableError { ticker: String, reason: String }, #[display(fmt = "Can't query from the storage - query: {} - reason: {}", query, reason)] QueryError { query: String, reason: String }, - #[display(fmt = "Can't retrieve string from row - reason: {}", reason)] - StringRowError { reason: String }, - #[display(fmt = "Can't execute query from storage for {} - reason: {}", ticker, reason)] - ExecutionError { ticker: String, reason: String }, - #[display(fmt = "Can't start a transaction from storage for {} - reason: {}", ticker, reason)] - TransactionError { ticker: String, reason: String }, - #[display(fmt = "Can't commit a transaction from storage for {} - reason: {}", ticker, reason)] - CommitError { ticker: String, reason: String }, + #[display(fmt = "Can't init from the storage - ticker: {} - reason: {}", ticker, reason)] + InitError { ticker: String, reason: String }, #[display(fmt = "Can't decode/deserialize from storage for {} - reason: {}", ticker, reason)] DecodeError { ticker: String, reason: String }, } diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index 59226594cf..c891048a3c 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -3283,8 +3283,7 @@ where let bytes = client.blockchain_block_header(height).compat().await?; let header: BlockHeader = deserialize(bytes.0.as_slice())?; let params = &storage.params; - let blocks_limit = NonZeroU64::new(params.blocks_limit_to_check) - .ok_or_else(|| GetBlockHeaderError::Internal("invalid block limit to check".to_string()))?; + let blocks_limit = params.blocks_limit_to_check; let (headers_registry, headers) = client.retrieve_last_headers(blocks_limit, height).compat().await?; match spv_validation::helpers_validation::validate_headers( headers, @@ -3378,16 +3377,12 @@ where Some(storage) => storage, }; let params = storage.params.clone(); - let (check_every, maybe_blocks_limit_to_check, difficulty_check, constant_difficulty) = ( + let (check_every, blocks_limit_to_check, difficulty_check, constant_difficulty) = ( params.check_every, - NonZeroU64::new(params.blocks_limit_to_check), + params.blocks_limit_to_check, params.difficulty_check, params.constant_difficulty, ); - let blocks_limit_to_check = match maybe_blocks_limit_to_check { - None => break, - Some(blocks_limit_to_check) => blocks_limit_to_check, - }; let height = try_loop_with_sleep!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); let client = match &coin.as_ref().rpc_client { UtxoRpcClientEnum::Native(_) => break, diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index d94e7ea3e2..6eeee14128 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -108,7 +108,7 @@ async fn common_headers_insert( let mut conn = storage.0.lock().unwrap(); let sql_transaction = conn .transaction() - .map_err(|e| BlockHeaderStorageError::TransactionError { + .map_err(|e| BlockHeaderStorageError::AddToStorageError { ticker: for_coin.to_string(), reason: e.to_string(), })?; @@ -116,14 +116,14 @@ async fn common_headers_insert( let block_cache_params = [&header.block_height, &header.block_hex]; sql_transaction .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) - .map_err(|e| BlockHeaderStorageError::ExecutionError { + .map_err(|e| BlockHeaderStorageError::AddToStorageError { ticker: for_coin.to_string(), reason: e.to_string(), })?; } sql_transaction .commit() - .map_err(|e| BlockHeaderStorageError::CommitError { + .map_err(|e| BlockHeaderStorageError::AddToStorageError { ticker: for_coin.to_string(), reason: e.to_string(), })?; @@ -140,7 +140,7 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { let conn = selfi.0.lock().unwrap(); conn.execute(&sql_cache, NO_PARAMS) .map(|_| ()) - .map_err(|e| BlockHeaderStorageError::ExecutionError { + .map_err(|e| BlockHeaderStorageError::InitError { ticker, reason: e.to_string(), })?; From 5d197241ff2bdfd5deab4bd1462d77631d2425ab Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 18 Mar 2022 09:18:36 +0100 Subject: [PATCH 70/74] feat(eth_test): remove the loop --- mm2src/coins/eth/eth_tests.rs | 36 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/mm2src/coins/eth/eth_tests.rs b/mm2src/coins/eth/eth_tests.rs index 428f9e42cb..d6d8d306c8 100644 --- a/mm2src/coins/eth/eth_tests.rs +++ b/mm2src/coins/eth/eth_tests.rs @@ -1227,25 +1227,19 @@ fn polygon_check_if_my_payment_sent() { println!("{:02x}", coin.my_address); let secret_hash = hex::decode("fc33114b389f0ee1212abf2867e99e89126f4860").unwrap(); - - let mut i = 0; - while i < 10 { - let swap_contract_address = "9130b257d37a52e52f21054c4da3450c72f595ce".into(); - let my_payment = coin - .check_if_my_payment_sent( - 1638764369, - &[], - &[], - &secret_hash, - 22185109, - &Some(swap_contract_address), - ) - .wait() - .unwrap() - .unwrap(); - let expected_hash = BytesJson::from("69a20008cea0c15ee483b5bbdff942752634aa072dfd2ff715fe87eec302de11"); - assert_eq!(expected_hash, my_payment.tx_hash()); - Timer::sleep(5.0); - i = i + 1; - } + let swap_contract_address = "9130b257d37a52e52f21054c4da3450c72f595ce".into(); + let my_payment = coin + .check_if_my_payment_sent( + 1638764369, + &[], + &[], + &secret_hash, + 22185109, + &Some(swap_contract_address), + ) + .wait() + .unwrap() + .unwrap(); + let expected_hash = BytesJson::from("69a20008cea0c15ee483b5bbdff942752634aa072dfd2ff715fe87eec302de11"); + assert_eq!(expected_hash, my_payment.tx_hash()); } From b071fcdddebb4a4ca9329740f12228fa78c7abc3 Mon Sep 17 00:00:00 2001 From: milerius Date: Fri, 18 Mar 2022 09:19:34 +0100 Subject: [PATCH 71/74] feat(eth_test): remove the loop --- mm2src/docker_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/docker_tests.rs b/mm2src/docker_tests.rs index 098310089e..968cb6b0e1 100644 --- a/mm2src/docker_tests.rs +++ b/mm2src/docker_tests.rs @@ -1432,7 +1432,7 @@ mod docker_tests { .unwrap(); // TODO when buy call is made immediately swap might be not put into swap ctx yet so locked // amount returns 0 - thread::sleep(Duration::from_secs(3)); + thread::sleep(Duration::from_secs(6)); let rc = block_on(mm_alice.rpc(json! ({ "userpass": mm_alice.userpass, From 3beaf24167da509b57a3b554adfb505961a11c88 Mon Sep 17 00:00:00 2001 From: milerius Date: Mon, 21 Mar 2022 06:46:00 +0100 Subject: [PATCH 72/74] feat(spv_proof): review fixes --- mm2src/coins/utxo/utxo_block_header_storage.rs | 6 +----- mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs | 2 +- mm2src/coins/utxo/utxo_sql_block_header_storage.rs | 8 ++++---- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 3cb7ea0aef..7cf332954d 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -14,10 +14,6 @@ use std::fmt::{Debug, Formatter}; #[derive(Debug, Display)] pub enum BlockHeaderStorageError { - #[display(fmt = "The storage cannot be initialized for {}", _0)] - InitializationError(String), - #[display(fmt = "The storage is not initialized for {} - reason: {}", ticker, reason)] - NotInitializedError { ticker: String, reason: String }, #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] AddToStorageError { ticker: String, reason: String }, #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] @@ -31,7 +27,7 @@ pub enum BlockHeaderStorageError { #[display(fmt = "Can't query from the storage - query: {} - reason: {}", query, reason)] QueryError { query: String, reason: String }, #[display(fmt = "Can't init from the storage - ticker: {} - reason: {}", ticker, reason)] - InitError { ticker: String, reason: String }, + InitializationError { ticker: String, reason: String }, #[display(fmt = "Can't decode/deserialize from storage for {} - reason: {}", ticker, reason)] DecodeError { ticker: String, reason: String }, } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 253625f235..7a0b83db64 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -268,7 +268,7 @@ where if maybe_storage.is_some() { let ticker = self.ticker().to_owned(); let (fut, abort_handle) = abortable(block_header_utxo_loop(weak, constructor)); - info!("Starting UTXO block header loop for coin {}", self.ticker()); + info!("Starting UTXO block header loop for coin {}", ticker); spawn(async move { if let Err(e) = fut.await { info!( diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index 6eeee14128..75dfe4f4eb 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -138,12 +138,12 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { let ticker = for_coin.to_owned(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - conn.execute(&sql_cache, NO_PARAMS) - .map(|_| ()) - .map_err(|e| BlockHeaderStorageError::InitError { + conn.execute(&sql_cache, NO_PARAMS).map(|_| ()).map_err(|e| { + BlockHeaderStorageError::InitializationError { ticker, reason: e.to_string(), - })?; + } + })?; Ok(()) }) .await From b5cadc52374ef8de10ecf24374e471fce6e0adaa Mon Sep 17 00:00:00 2001 From: milerius Date: Tue, 22 Mar 2022 09:00:13 +0100 Subject: [PATCH 73/74] feat(spv_proof): review fixes --- mm2src/coins/utxo/utxo_tests.rs | 2 +- .../spv_validation/src/helpers_validation.rs | 15 ++++++--------- .../mm2_bitcoin/spv_validation/src/spv_proof.rs | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index e18f0df35f..15e89dfc02 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -934,7 +934,7 @@ fn test_spv_proof() { let tx_str = "0400008085202f8902bf17bf7d1daace52e08f732a6b8771743ca4b1cb765a187e72fd091a0aabfd52000000006a47304402203eaaa3c4da101240f80f9c5e9de716a22b1ec6d66080de6a0cca32011cd77223022040d9082b6242d6acf9a1a8e658779e1c655d708379862f235e8ba7b8ca4e69c6012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffffff023ca13c0e9e085dd13f481f193e8a3e8fd609020936e98b5587342d994f4d020000006b483045022100c0ba56adb8de923975052312467347d83238bd8d480ce66e8b709a7997373994022048507bcac921fdb2302fa5224ce86e41b7efc1a2e20ae63aa738dfa99b7be826012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9141ee6d4c38a3c078eab87ad1a5e4b00f21259b10d870000000000000000166a1400000000000000000000000000000000000000001b94d736000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac2d08e35e000000000000000000000000000000"; let tx: UtxoTx = tx_str.into(); let res = block_on(utxo_common::validate_spv_proof(coin.clone(), tx)); - assert_eq!(res.is_err(), false); + res.unwrap() } #[test] diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index ff2323b445..5665af9bcc 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -20,10 +20,7 @@ pub fn merkle_prove(txid: H256, merkle_root: H256, intermediate_nodes: Vec if txid == merkle_root && index == 0 && intermediate_nodes.is_empty() { return Ok(()); } - let mut vec: Vec = vec![]; - for merkle_node in intermediate_nodes { - vec.append(&mut merkle_node.as_slice().to_vec()); - } + let vec: Vec = intermediate_nodes.into_iter().flat_map(|node| node.take()).collect(); let nodes = bitcoin_spv::types::MerkleArray::new(vec.as_slice())?; if !verify_hash256_merkle(txid.take().into(), merkle_root.take().into(), &nodes, index) { return Err(SPVError::BadMerkleProof); @@ -110,7 +107,7 @@ mod tests { H256::from_reversed_str("4274d707b2308d39a04f2940024d382fa80d994152a50d4258f5a7feead2a563"), ]; let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); - assert_eq!(result.is_err(), false); + result.unwrap() } #[test] @@ -125,7 +122,7 @@ mod tests { "5a4ebf66822b0b2d56bd9dc64ece0bc38ee7844a23ff1d7320a88c5fdb2ad3e2", )]; let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); - assert_eq!(result.is_err(), false); + result.unwrap() } #[test] @@ -151,7 +148,7 @@ mod tests { H256::from_reversed_str("3104295d99163e16405b80321238a97d02e2448bb634017e2e027281cc4af9e8"), ]; let result = merkle_prove(tx_id, merkle_root, merkle_nodes, merkle_pos); - assert_eq!(result.is_err(), false); + result.unwrap() } #[test] @@ -162,7 +159,7 @@ mod tests { "04000000001f22e1bc88c53b1554f8fdcf261fdb09f4cae6ef5e5032b788515f4a60d30d67d1b35fda68abc05f5af39e5ade224a5312b8dcd1f3629a7ff33355bb7ca93e32d719d14c15e565c05e84ead95a2f101a1b658ee2f36eb7ca65206e27cfca478be6146220bb071f49000b055b22a7a4bbafd6b52efb90f963d5f80126c27e437005fb47720e0000fd4005004d9875d71c540f558813142e263f597243bdd8d8105ff3d1ffd62ae51ccf22729debe510f97ab0631701dbd34b73e570597dc8825be6bd669e693037fb701040c273b44745f4e850c2d8aeca7ccab6ef7f462206a16d75358f2e8fddf9d0dbc6333ff55b1813a37f0ba240bd2d897fbd6cfdb1989ac8f3ec93b15ae4360edf84088ac9a4ea7d3d71290532bb51675e7310be1210aa33c184d693f6f7c15c5be1e89356ae3d663d0c548fceac0974fe4cb6c6559f50643280df9508460fd04f9cde55521b4c6d61c644c6c7b7473f9e39b412e3776f5e47b6c466aaf1dc76ff2114e716eb6b9614d0c93cdc229ec13b07057a7f7446c1aac51ef0950d4361fa2d20f22f29ff490bf6d6a2a267c45d88d3152d9f5291695f2f4fba65ca9763cb4176506c73b8162611b6004af7ec8d1ea55a225cca2576e4ac84ac333b663693a2f19f7786340ad9d2212d576a0b4e7700bd7d60de88940dce1f01481f9c41350eefd7b496218bcf70c4c8922dfd18d666d37d10cb0f14dd38e1225ec179dcab5501a4434674d6f9ff9f23c4df5f445cc2accf43189fc99ac56693df373a4207b0dc991009fae4796fd7e49cea4dd139ee72264dfd47f4e1ad2420d635c7a1f37950d022ffdcccc7651b645db0ba0ce94c18dcc902279b4601806beefe05016f1f85411e6562b584da0854db2e36f602d8c4974d385aee4a01d1132082c8cd7c71443162f7d7487c73d8a46f830f72a0d352d957bef5afc33c4447ef33b2491e28000d1f4687e95ffc2b9532d28ae4c48f8551bf527dbe18c672204495f2bd546566fd5770189e28c2de0974130a492ccd8737a8c6e971d02a23c4f9f27410348d1f666f93385bdc81bad8e9a9d1dbffdfa2609ebae52740b457ecd67a3bf0db02a14f5bdf3e25b35b2d3d303094e46e0e3daef559d9f0e074e512bcaf9fcc9d035083eec16806af8a93d27b4ad46754a425b6a02b1ac22f682e48f214d66b379d7042aa39f2c5f3448d05ca4b6360e162f31f197225f4ad579d69207c666711fb3f6ca814efcf430899360cced1168cd69ec0e809a89cf2cf2015f9f895a3dadd4ced6d94793e98201b1da6a0a5d90be5d06925e3ad60b9227f84b9c3060a6db6e7857d8731f975d4a993abf10d84590da02b114625109d864de070813179b651d528f66036c30a0700ee84fc5e59757a509745b64e76fa3396f3c8b01a7724cd434e6d774dad36be8a73ad29f6859352aa15236e7825947396cb98e26b912b19ddc127590e59200c4334d1d96d7585a0e349b920f2e4e59cdedac911214c42c0894f72c8a7423d7aef3ea5ef9a5b650821f46537c65509ad8dcf6558c16c04f9877c737ff81875d9fbe01d23d37e937444cf257b0b57bc1c2a774f2e2bf5f3b0881be0e2282ba97ef6aad797f8fdb4053da4e478575805c7a93076c09847544a8e89f1cb3838df7870bcf61deb2144c6f6349c966b67545703058f9227965b97835b049538fb428431a8461586b022368626d20e9b6bfdd7232a5cc6a0aa214319cb440c45443a2446d1e17713c0e1049f0fd759d1dbff493302140376cfb153330ed455a043189260cb7d2d90333a37d3584f2d907d0a73dccee299ad14141d60d1409cda688464a13b5dab37476641741717d599a60c0ac84d85869ed449f83933ad30e2591157fd1f07b73ecf26f34e91bc00f1ca86ae34ca8231b372cdc2ed18d463ac42f92859d6f0e2c483dbb23d785f1233db2033458af9d7c1e7029ac5cc33ca7d25b2b49fd71b1ae5f5ce969b6e77333bf5fbb5e6645dd0a4d0c6e82eb534ac264ddbe28513e4b82b3578c1a6cbfaa2522aa50985fe2cce43cf3363eaacca0e09c721fd603d43c3a4fdf8dde0c9ff2c054910b16aeef7c4d86b31".into(), "04000000fcead9a1b425124f11aa97e0614120ce87bdddcad655672916f9c4564dc057002bd3df07a4602620282b276359529114ba89b59b16bec235d584c3cf5cc6b2d132d719d14c15e565c05e84ead95a2f101a1b658ee2f36eb7ca65206e27cfca47bfe61462d5b9071f1a001daf299c51afbd74fd75a98ba49a6e40ae8ad92b3afdc1cf215fd6190000fd40050044b5e035b02d138a9704f9513c0865f2733b7c09294ee504c155c283f4895559b6ac39828eac98ad393a642330589e8849040f55ce44f8f2197529d0b0ed57ccdda41f1971e153ec28ac5b4eba968741db374104d65ee234580a83bea1c0cdb67b8bc207057486eb1d90e21ba0cd4f5e9fd834821fafc1517c5d1fceb50ba6f6b102a9b4edac46f2359aec795a4e2458f51114a41289634b3b1cf250e3e38f3689f951278dfa7202a7dfe311cc098fd4a8d02c8f8a74e4a5010b18ee2e60578d5e9f1c094433a73f26e6546e20a574fc261baaa79e9910ab86ed607786a1cc88e7de51ff928d434e26eaef1437f7068c743f26d7c0eea6791e869b101fee8ab41b50af6174c5e6b731a1719f31ee3e6529efef49f31665baedc9382e9665278a84467d479f139fc7a8ef66fef9bd2fd17f7779ee315d458f691a290fa7c2179de8bb91a78458c5290d4aa45b163254006800ba2fce7479511f744fd7de96495c39be93413d8b0b187fe092537e1a7646a66a125b33333f6ecd10085e23ad168b24ee7be69d01ea021a39401e4bd41d818499e7174dd9b85542076c78cb89eeec1c190301b4709dbc963d47926e31bb0235ba6a7029d49458150f6491ac9c973b8a2c893258f907baf4bcb7c39f12b900ba2b2382cd5dd84314ee504ade835ad9a1cb13a7f5928a483ebc9415429810fd99893f2f8f83970b8b47143d617e6f9853e4d86ff378be664218f1c32531143e209f171590dd48216fec879a6b9cbf04432bf4f1a3734b69b6a9f1a358a259a0f9082cfb6c1f3d9d2d9e4522ad651ccce565f06b30c1c0b27252270c2f6608cf4f3288a7e7d4b174e646de05341f7db62b00b5ccb295f058d34b87201148828e9b3f7e08f60e100f810be27eb7f4c471cda7621106fe78bc69ec2bd27acabd55dc094b8626913b7d24d9b60939754700f32574a733a195f8b0220d56f6797de0bcd7b80d561896b816586593409f76e85a7a1035f821dee32a02fdbc26bc4cca375bed418b9d678ac589249a1a5a5b24447ee9b42e33f817066caf3d4e17d0347f6acf0cbf426d4df49413b3d12350edec2681ab9cfecd0825ccfb2649a57391d3f153050dfb4350d60e5e464229ddd6e49ece95557b8ef48c18cbffbe9fc8d7700f611a4b33a2a254afcec638c485e36daf0364da7d4302e488db7b6c41297571048cfea5452e324abb9f9e1043e625fd0853b7e03063d1c3a43aa1ee62d45d890b5e4d10640e775cff6852b6d1acd4a503b3ece3b319cbcf33ff9fdf17b8f852d748db1e05af80507f5d0e1bc44444b155d7da20f7f0b4d6d83368c3bb9e1321b39472a8677ea1d3aca43b453d35edca37b7536d19c26b764958b3c7c30f3211d7b7bb7f6a6d7fd7bf2dda6e7d7b1e533556863549bbe1394a3828596f25029b7e30495e1235f084e5edd133bc29fce4f1e5e514eb1d1cb19fd8dfbb0d130fbec4e288f23dae86311ffd6f4afbaacc2ffe1cc8811a455ba6f5659f82515b56c6ac84277bff5bef98fefc74e002e4a11866a417a429541f8a62df4108e4730d3045f92984bcf1ab2f7d03f8bb1767e91791530cd8eec412919e1f2e341e66a1588a8f485f7aa005787af946b9cb10f6685420b7e1663f66374fddc5e70720507ee2134f3b02df042fcf6db4a5bdd74cc5010793634816fe447cc68e076b225cc1ca872929ef246ce356dc8d8964ff6d7119d071eccb6dc37f75b932c44cdc30723b8357a2761c6de6ab2713e6f6a782538cb731b07950d3f459760a00cc0af406d6848014746b02653636f479d952b46fdeff976e1d159ba46ae7363d5b0042d3905a0bda12aaa6eaae1a5a0d55d4c1930aa1c004cd610866853a247239366aa20f8968ea9ca3d5d6d7321a5d0f2c".into() ]; - assert_eq!(true, validate_headers(headers, false, false).is_ok()); + validate_headers(headers, false, false).unwrap() } #[test] @@ -171,6 +168,6 @@ mod tests { let headers: Vec = vec!["00200020eab6fa183da8f9e4c761b31a67a76fa6a7658eb84c760200000000000000000063cd9585d434ec0db25894ec4b1f03735f10e31709c4395ea67c50c8378f134b972f166278100a17bfd87203".into(), "0000402045c698413fbe8b5bf10635658d2a1cec72062798e51200000000000000000000869617420a4c95b1d3d6d012419d2b6c199cff9b68dd9a790892a4da8466fb056033166278100a1743ac4d5b".into(), "0400e02019d733c1fd76a1fa5950de7bee9d80f107276b93a67204000000000000000000a0d1dee718f5f732c041800e9aa2c25e92be3f6de28278545388db8a6ae27df64c37166278100a170a970c19".into()]; - assert_eq!(true, validate_headers(headers, true, true).is_ok()); + validate_headers(headers, true, true).unwrap() } } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 0aa378388d..38e8461c88 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -88,6 +88,6 @@ mod spv_proof_tests { raw_header: RawBlockHeader::new(header_bytes).unwrap(), intermediate_nodes: vec![], }; - assert!(spv_proof.validate_block_header().is_ok()) + spv_proof.validate_block_header().unwrap() } } From 4ff3f4f2f4647f6c062317acefe75ceb18e0b91e Mon Sep 17 00:00:00 2001 From: milerius Date: Wed, 23 Mar 2022 11:13:15 +0100 Subject: [PATCH 74/74] feat(spv): fix wrong copy paste description --- mm2src/coins/utxo/utxo_block_header_storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 7cf332954d..ada1453824 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -16,7 +16,7 @@ use std::fmt::{Debug, Formatter}; pub enum BlockHeaderStorageError { #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] AddToStorageError { ticker: String, reason: String }, - #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] + #[display(fmt = "Can't get from the storage for {} - reason: {}", ticker, reason)] GetFromStorageError { ticker: String, reason: String }, #[display( fmt = "Can't retrieve the table from the storage for {} - reason: {}",