From 30323f1150d0b05858ad57c7b219c47635fe6783 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:45:06 +0000 Subject: [PATCH 01/18] feat: scaffolding for generic balance fetcher and cosmos impl --- rust/Cargo.lock | 2 + rust/agents/relayer/src/relayer.rs | 13 +- .../hyperlane-cosmos/src/aggregation_ism.rs | 13 +- .../hyperlane-cosmos/src/interchain_gas.rs | 11 +- .../src/interchain_security_module.rs | 15 +- rust/chains/hyperlane-cosmos/src/lib.rs | 6 +- .../hyperlane-cosmos/src/libs/address.rs | 2 +- rust/chains/hyperlane-cosmos/src/mailbox.rs | 26 ++- .../hyperlane-cosmos/src/merkle_tree_hook.rs | 22 ++- .../hyperlane-cosmos/src/metrics_fetcher.rs | 52 ++++++ .../hyperlane-cosmos/src/multisig_ism.rs | 18 +- .../hyperlane-cosmos/src/providers/grpc.rs | 55 +++++- .../hyperlane-cosmos/src/providers/mod.rs | 53 +++++- .../hyperlane-cosmos/src/routing_ism.rs | 14 +- .../src/validator_announce.rs | 18 +- .../hyperlane-ethereum/src/agent_metrics.rs | 11 ++ rust/chains/hyperlane-ethereum/src/lib.rs | 1 + .../chains/hyperlane-ethereum/src/provider.rs | 6 +- rust/chains/hyperlane-fuel/src/provider.rs | 6 +- .../chains/hyperlane-sealevel/src/provider.rs | 6 +- rust/ethers-prometheus/src/middleware/mod.rs | 167 +++--------------- rust/hyperlane-base/Cargo.toml | 2 + rust/hyperlane-base/src/metrics/agent.rs | 112 ++++++++++++ rust/hyperlane-base/src/metrics/mod.rs | 1 + rust/hyperlane-base/src/metrics/provider.rs | 5 - rust/hyperlane-base/src/settings/chains.rs | 66 +++++-- rust/hyperlane-core/src/chain.rs | 3 +- rust/hyperlane-core/src/lib.rs | 1 + rust/hyperlane-core/src/metrics/agent.rs | 17 ++ rust/hyperlane-core/src/metrics/mod.rs | 1 + rust/hyperlane-core/src/traits/provider.rs | 5 +- rust/utils/run-locally/src/cosmos/mod.rs | 2 +- 32 files changed, 496 insertions(+), 236 deletions(-) create mode 100644 rust/chains/hyperlane-cosmos/src/metrics_fetcher.rs create mode 100644 rust/chains/hyperlane-ethereum/src/agent_metrics.rs create mode 100644 rust/hyperlane-base/src/metrics/agent.rs create mode 100644 rust/hyperlane-core/src/metrics/agent.rs create mode 100644 rust/hyperlane-core/src/metrics/mod.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8d1b909db0..61fd90ded1 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4035,6 +4035,7 @@ dependencies = [ "config", "convert_case 0.6.0", "derive-new", + "derive_builder", "ed25519-dalek", "ethers", "ethers-prometheus", @@ -4048,6 +4049,7 @@ dependencies = [ "hyperlane-sealevel", "hyperlane-test", "itertools 0.11.0", + "maplit", "paste", "prometheus", "rocksdb", diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index 5ff91466a2..7badb1174a 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -188,9 +188,20 @@ impl BaseAgent for Relayer { .collect(); let mut msg_ctxs = HashMap::new(); + // let mut custom_metrics = HashMap::new(); for destination in &settings.destination_chains { let destination_chain_setup = core.settings.chain_setup(destination).unwrap().clone(); - + let agent_metrics_conf = destination_chain_setup + .agent_metrics_conf("relayer".to_owned()) + .await?; + println!("~~~ agent metrics: {:?}", agent_metrics_conf); + println!("~~~ agent signer: {:?}", destination_chain_setup.signer); + // custom_metrics.insert( + // destination.id(), + // destination_chain_setup + // .metrics(destination) + // .expect("Missing metrics config"), + // ); let transaction_gas_limit: Option = if skip_transaction_gas_limit_for.contains(&destination.id()) { None diff --git a/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs b/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs index c9d7200117..a2439f92a3 100644 --- a/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs @@ -18,7 +18,7 @@ use tracing::instrument; pub struct CosmosAggregationIsm { domain: HyperlaneDomain, address: H256, - provider: Box, + provider: Box, } impl CosmosAggregationIsm { @@ -28,7 +28,12 @@ impl CosmosAggregationIsm { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider = WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { domain: locator.domain.clone(), @@ -50,7 +55,7 @@ impl HyperlaneChain for CosmosAggregationIsm { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -63,7 +68,7 @@ impl AggregationIsm for CosmosAggregationIsm { ) -> ChainResult<(Vec, u8)> { let payload = ModulesAndThresholdRequest::new(message); - let data = self.provider.wasm_query(payload, None).await?; + let data = self.provider.grpc().wasm_query(payload, None).await?; let response: ModulesAndThresholdResponse = serde_json::from_slice(&data)?; let modules: ChainResult> = response diff --git a/rust/chains/hyperlane-cosmos/src/interchain_gas.rs b/rust/chains/hyperlane-cosmos/src/interchain_gas.rs index d96bfb0bab..83044913b4 100644 --- a/rust/chains/hyperlane-cosmos/src/interchain_gas.rs +++ b/rust/chains/hyperlane-cosmos/src/interchain_gas.rs @@ -22,6 +22,7 @@ use crate::{ pub struct CosmosInterchainGasPaymaster { domain: HyperlaneDomain, address: H256, + provider: CosmosProvider, } impl HyperlaneContract for CosmosInterchainGasPaymaster { @@ -36,7 +37,7 @@ impl HyperlaneChain for CosmosInterchainGasPaymaster { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -49,11 +50,17 @@ impl CosmosInterchainGasPaymaster { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider = WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { domain: locator.domain.clone(), address: locator.address, + provider, }) } } diff --git a/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs b/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs index 72a0ac984d..f4c6932cf5 100644 --- a/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs +++ b/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs @@ -22,7 +22,7 @@ pub struct CosmosInterchainSecurityModule { /// The address of the ISM contract. address: H256, /// The provider for the ISM contract. - provider: Box, + provider: CosmosProvider, } /// The Cosmos Interchain Security Module Implementation. @@ -33,13 +33,17 @@ impl CosmosInterchainSecurityModule { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider: WasmGrpcProvider = - WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { domain: locator.domain.clone(), address: locator.address, - provider: Box::new(provider), + provider, }) } } @@ -56,7 +60,7 @@ impl HyperlaneChain for CosmosInterchainSecurityModule { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -71,6 +75,7 @@ impl InterchainSecurityModule for CosmosInterchainSecurityModule { let data = self .provider + .grpc() .wasm_query(QueryIsmGeneralRequest { ism: query }, None) .await?; diff --git a/rust/chains/hyperlane-cosmos/src/lib.rs b/rust/chains/hyperlane-cosmos/src/lib.rs index 82a4a0ece1..92b3a56730 100644 --- a/rust/chains/hyperlane-cosmos/src/lib.rs +++ b/rust/chains/hyperlane-cosmos/src/lib.rs @@ -12,6 +12,7 @@ mod interchain_security_module; mod libs; mod mailbox; mod merkle_tree_hook; +mod metrics_fetcher; mod multisig_ism; mod payloads; mod providers; @@ -24,6 +25,7 @@ mod validator_announce; pub use self::{ aggregation_ism::*, error::*, interchain_gas::*, interchain_security_module::*, libs::*, - mailbox::*, merkle_tree_hook::*, multisig_ism::*, providers::*, routing_ism::*, signers::*, - trait_builder::*, trait_builder::*, validator_announce::*, validator_announce::*, + mailbox::*, merkle_tree_hook::*, metrics_fetcher::*, multisig_ism::*, providers::*, + routing_ism::*, signers::*, trait_builder::*, trait_builder::*, validator_announce::*, + validator_announce::*, }; diff --git a/rust/chains/hyperlane-cosmos/src/libs/address.rs b/rust/chains/hyperlane-cosmos/src/libs/address.rs index d5970b9b82..7cfdb83caf 100644 --- a/rust/chains/hyperlane-cosmos/src/libs/address.rs +++ b/rust/chains/hyperlane-cosmos/src/libs/address.rs @@ -12,7 +12,7 @@ use tendermint::public_key::PublicKey as TendermintPublicKey; use crate::HyperlaneCosmosError; /// Wrapper around the cosmrs AccountId type that abstracts bech32 encoding -#[derive(new, Debug)] +#[derive(new, Debug, Clone)] pub struct CosmosAddress { /// Bech32 encoded cosmos account account_id: AccountId, diff --git a/rust/chains/hyperlane-cosmos/src/mailbox.rs b/rust/chains/hyperlane-cosmos/src/mailbox.rs index 4df968edc9..51e8529bc3 100644 --- a/rust/chains/hyperlane-cosmos/src/mailbox.rs +++ b/rust/chains/hyperlane-cosmos/src/mailbox.rs @@ -40,7 +40,7 @@ pub struct CosmosMailbox { config: ConnectionConf, domain: HyperlaneDomain, address: H256, - provider: Box, + provider: CosmosProvider, } impl CosmosMailbox { @@ -51,13 +51,18 @@ impl CosmosMailbox { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider = WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { config: conf, domain: locator.domain.clone(), address: locator.address, - provider: Box::new(provider), + provider, }) } @@ -79,7 +84,7 @@ impl HyperlaneChain for CosmosMailbox { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -94,7 +99,7 @@ impl Debug for CosmosMailbox { impl Mailbox for CosmosMailbox { #[instrument(level = "debug", err, ret, skip(self))] async fn count(&self, lag: Option) -> ChainResult { - let block_height = get_block_height_for_lag(&self.provider, lag).await?; + let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; self.nonce_at_block(block_height).await } @@ -107,6 +112,7 @@ impl Mailbox for CosmosMailbox { let delivered = match self .provider + .grpc() .wasm_query(GeneralMailboxQuery { mailbox: payload }, None) .await { @@ -136,6 +142,7 @@ impl Mailbox for CosmosMailbox { let data = self .provider + .grpc() .wasm_query(GeneralMailboxQuery { mailbox: payload }, None) .await?; let response: mailbox::DefaultIsmResponse = serde_json::from_slice(&data)?; @@ -157,6 +164,7 @@ impl Mailbox for CosmosMailbox { let data = self .provider + .grpc() .wasm_query(GeneralMailboxQuery { mailbox: payload }, None) .await?; let response: mailbox::RecipientIsmResponse = serde_json::from_slice(&data)?; @@ -182,6 +190,7 @@ impl Mailbox for CosmosMailbox { let response: TxResponse = self .provider + .grpc() .wasm_send(process_message, tx_gas_limit) .await?; @@ -201,7 +210,11 @@ impl Mailbox for CosmosMailbox { }, }; - let gas_limit = self.provider.wasm_estimate_gas(process_message).await?; + let gas_limit = self + .provider + .grpc() + .wasm_estimate_gas(process_message) + .await?; let result = TxCostEstimate { gas_limit: gas_limit.into(), @@ -226,6 +239,7 @@ impl CosmosMailbox { let data = self .provider + .grpc() .wasm_query(GeneralMailboxQuery { mailbox: payload }, block_height) .await?; diff --git a/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs b/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs index 15db14ff7a..f18f5a2a83 100644 --- a/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs +++ b/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs @@ -33,7 +33,7 @@ pub struct CosmosMerkleTreeHook { /// Contract address address: H256, /// Provider - provider: Box, + provider: CosmosProvider, } impl CosmosMerkleTreeHook { @@ -43,12 +43,17 @@ impl CosmosMerkleTreeHook { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider = WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { domain: locator.domain.clone(), address: locator.address, - provider: Box::new(provider), + provider, }) } } @@ -65,7 +70,7 @@ impl HyperlaneChain for CosmosMerkleTreeHook { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -78,10 +83,11 @@ impl MerkleTreeHook for CosmosMerkleTreeHook { tree: general::EmptyStruct {}, }; - let block_height = get_block_height_for_lag(&self.provider, lag).await?; + let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; let data = self .provider + .grpc() .wasm_query( merkle_tree_hook::MerkleTreeGenericRequest { merkle_hook: payload, @@ -111,7 +117,7 @@ impl MerkleTreeHook for CosmosMerkleTreeHook { count: general::EmptyStruct {}, }; - let block_height = get_block_height_for_lag(&self.provider, lag).await?; + let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; self.count_at_block(block_height).await } @@ -122,10 +128,11 @@ impl MerkleTreeHook for CosmosMerkleTreeHook { check_point: general::EmptyStruct {}, }; - let block_height = get_block_height_for_lag(&self.provider, lag).await?; + let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; let data = self .provider + .grpc() .wasm_query( merkle_tree_hook::MerkleTreeGenericRequest { merkle_hook: payload, @@ -153,6 +160,7 @@ impl CosmosMerkleTreeHook { let data = self .provider + .grpc() .wasm_query( merkle_tree_hook::MerkleTreeGenericRequest { merkle_hook: payload, diff --git a/rust/chains/hyperlane-cosmos/src/metrics_fetcher.rs b/rust/chains/hyperlane-cosmos/src/metrics_fetcher.rs new file mode 100644 index 0000000000..3cb20419c0 --- /dev/null +++ b/rust/chains/hyperlane-cosmos/src/metrics_fetcher.rs @@ -0,0 +1,52 @@ +use async_trait::async_trait; +use hyperlane_core::{ + metrics::agent::AgenMetricsFetcher, ChainResult, ContractLocator, HyperlaneChain, + HyperlaneDomain, HyperlaneProvider, U256, +}; + +use crate::{address::CosmosAddress, ConnectionConf, CosmosProvider}; + +#[derive(Debug)] +pub struct CosmosMetricsFetcher { + address: CosmosAddress, + provider: CosmosProvider, + domain: HyperlaneDomain, +} + +impl CosmosMetricsFetcher { + pub fn new( + conf: ConnectionConf, + locator: ContractLocator, + address: CosmosAddress, + ) -> ChainResult { + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + None, + )?; + + Ok(Self { + address, + provider, + domain: locator.domain.clone(), + }) + } +} + +impl HyperlaneChain for CosmosMetricsFetcher { + fn domain(&self) -> &HyperlaneDomain { + &self.domain + } + + fn provider(&self) -> Box { + Box::new(self.provider.clone()) + } +} + +#[async_trait] +impl AgenMetricsFetcher for CosmosMetricsFetcher { + async fn get_balance(&self) -> ChainResult { + self.provider.get_balance(self.address.address()).await + } +} diff --git a/rust/chains/hyperlane-cosmos/src/multisig_ism.rs b/rust/chains/hyperlane-cosmos/src/multisig_ism.rs index a9d84dec7f..d558acfa37 100644 --- a/rust/chains/hyperlane-cosmos/src/multisig_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/multisig_ism.rs @@ -1,9 +1,7 @@ use std::str::FromStr; use crate::{ - grpc::{WasmGrpcProvider, WasmProvider}, - payloads::ism_routes::QueryIsmGeneralRequest, - signers::Signer, + grpc::WasmProvider, payloads::ism_routes::QueryIsmGeneralRequest, signers::Signer, ConnectionConf, CosmosProvider, }; use async_trait::async_trait; @@ -19,7 +17,7 @@ use crate::payloads::multisig_ism::{self, VerifyInfoRequest, VerifyInfoRequestIn pub struct CosmosMultisigIsm { domain: HyperlaneDomain, address: H256, - provider: Box, + provider: CosmosProvider, } impl CosmosMultisigIsm { @@ -29,12 +27,17 @@ impl CosmosMultisigIsm { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider = WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { domain: locator.domain.clone(), address: locator.address, - provider: Box::new(provider), + provider, }) } } @@ -51,7 +54,7 @@ impl HyperlaneChain for CosmosMultisigIsm { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -70,6 +73,7 @@ impl MultisigIsm for CosmosMultisigIsm { let data = self .provider + .grpc() .wasm_query(QueryIsmGeneralRequest { ism: payload }, None) .await?; let response: multisig_ism::VerifyInfoResponse = serde_json::from_slice(&data)?; diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index a47d660ded..3a23564fc3 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -5,6 +5,7 @@ use cosmrs::{ auth::v1beta1::{ query_client::QueryClient as QueryAccountClient, BaseAccount, QueryAccountRequest, }, + bank::v1beta1::{query_client::QueryClient as QueryBalanceClient, QueryBalanceRequest}, base::{ abci::v1beta1::TxResponse, tendermint::v1beta1::{service_client::ServiceClient, GetLatestBlockRequest}, @@ -77,14 +78,14 @@ pub trait WasmProvider: Send + Sync { async fn wasm_estimate_gas(&self, payload: T) -> ChainResult; } -#[derive(Debug)] +#[derive(Debug, Clone)] /// CosmWasm GRPC provider. pub struct WasmGrpcProvider { /// Connection configuration. conf: ConnectionConf, /// A contract address that can be used as the default /// for queries / sends / estimates. - contract_address: CosmosAddress, + contract_address: Option, /// Signer for transactions. signer: Option, /// GRPC Channel that can be cheaply cloned. @@ -96,13 +97,15 @@ impl WasmGrpcProvider { /// Create new CosmWasm GRPC Provider. pub fn new( conf: ConnectionConf, - locator: ContractLocator, + locator: Option, signer: Option, ) -> ChainResult { let endpoint = Endpoint::new(conf.get_grpc_url()).map_err(Into::::into)?; let channel = endpoint.connect_lazy(); - let contract_address = CosmosAddress::from_h256(locator.address, &conf.get_prefix())?; + let contract_address = locator + .map(|l| CosmosAddress::from_h256(l.address, &conf.get_prefix())) + .transpose()?; Ok(Self { conf, @@ -220,6 +223,35 @@ impl WasmGrpcProvider { Ok(gas_estimate) } + /// Estimates gas for a transaction containing `msgs`. + pub async fn get_balance(&self, address: String, denom: String) -> ChainResult { + let mut client = QueryBalanceClient::new(self.channel.clone()); + + let balance_request = tonic::Request::new(QueryBalanceRequest { + address, + /// denom is the coin denom to query balances for. + denom, + }); + let response = client + .balance(balance_request) + .await + .map_err(ChainCommunicationError::from_other)? + .into_inner(); + // let coin = u128::decode( + // response + // .balance + // .ok_or_else(|| ChainCommunicationError::from_other_str("account not present"))? + // .amount + // ); + + let balance = response + .balance + .ok_or_else(|| ChainCommunicationError::from_other_str("account not present"))?; + println!("~~~ relayer balance: {:?}", balance); + + Ok(balance.amount.parse()?) + } + /// Queries an account. async fn account_query(&self, account: String) -> ChainResult { let mut client = QueryAccountClient::new(self.channel.clone()); @@ -268,7 +300,10 @@ impl WasmProvider for WasmGrpcProvider { where T: Serialize + Send + Sync, { - self.wasm_query_to(self.contract_address.address(), payload, block_height) + let contract_address = self.contract_address.as_ref().ok_or_else(|| { + ChainCommunicationError::from_other_str("No contract address available") + })?; + self.wasm_query_to(contract_address.address(), payload, block_height) .await } @@ -308,10 +343,13 @@ impl WasmProvider for WasmGrpcProvider { { let signer = self.get_signer()?; let mut client = TxServiceClient::new(self.channel.clone()); + let contract_address = self.contract_address.as_ref().ok_or_else(|| { + ChainCommunicationError::from_other_str("No contract address available") + })?; let msgs = vec![MsgExecuteContract { sender: signer.address.clone(), - contract: self.contract_address.address(), + contract: contract_address.address(), msg: serde_json::to_string(&payload)?.as_bytes().to_vec(), funds: vec![], } @@ -354,9 +392,12 @@ impl WasmProvider for WasmGrpcProvider { // Estimating gas requires a signer, which we can reasonably expect to have // since we need one to send a tx with the estimated gas anyways. let signer = self.get_signer()?; + let contract_address = self.contract_address.as_ref().ok_or_else(|| { + ChainCommunicationError::from_other_str("No contract address available") + })?; let msg = MsgExecuteContract { sender: signer.address.clone(), - contract: self.contract_address.address(), + contract: contract_address.address(), msg: serde_json::to_string(&payload)?.as_bytes().to_vec(), funds: vec![], }; diff --git a/rust/chains/hyperlane-cosmos/src/providers/mod.rs b/rust/chains/hyperlane-cosmos/src/providers/mod.rs index cf9422b2f8..77296ca756 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/mod.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/mod.rs @@ -1,7 +1,13 @@ use async_trait::async_trait; use hyperlane_core::{ - BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, + BlockInfo, ChainResult, ContractLocator, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, + TxnInfo, H256, U256, }; +use tendermint_rpc::{client::CompatMode, HttpClient}; + +use crate::{ConnectionConf, HyperlaneCosmosError, Signer}; + +use self::grpc::WasmGrpcProvider; /// cosmos grpc provider pub mod grpc; @@ -9,15 +15,41 @@ pub mod grpc; pub mod rpc; /// A reference to a Cosmos chain -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct CosmosProvider { domain: HyperlaneDomain, + grpc_client: WasmGrpcProvider, + _rpc_client: HttpClient, } impl CosmosProvider { /// Create a reference to a Cosmos chain - pub fn new(domain: HyperlaneDomain) -> Self { - Self { domain } + pub fn new( + domain: HyperlaneDomain, + conf: ConnectionConf, + locator: Option, + signer: Option, + ) -> ChainResult { + let grpc_client = WasmGrpcProvider::new(conf.clone(), locator, signer)?; + let _rpc_client = HttpClient::builder( + conf.get_rpc_url() + .parse() + .map_err(Into::::into)?, + ) + // Consider supporting different compatibility modes. + .compat_mode(CompatMode::latest()) + .build() + .map_err(Into::::into)?; + + Ok(Self { + domain, + _rpc_client, + grpc_client, + }) + } + + pub fn grpc(&self) -> WasmGrpcProvider { + self.grpc_client.clone() } } @@ -27,9 +59,7 @@ impl HyperlaneChain for CosmosProvider { } fn provider(&self) -> Box { - Box::new(CosmosProvider { - domain: self.domain.clone(), - }) + Box::new(self.clone()) } } @@ -47,4 +77,13 @@ impl HyperlaneProvider for CosmosProvider { // FIXME Ok(true) } + + async fn get_balance(&self, address: String) -> ChainResult { + // denom is of the form "untrn" + Ok(self + .grpc_client + .get_balance(address, "".to_string()) + .await? + .into()) + } } diff --git a/rust/chains/hyperlane-cosmos/src/routing_ism.rs b/rust/chains/hyperlane-cosmos/src/routing_ism.rs index 0a646c005b..97360546f4 100644 --- a/rust/chains/hyperlane-cosmos/src/routing_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/routing_ism.rs @@ -22,7 +22,7 @@ use crate::{ pub struct CosmosRoutingIsm { domain: HyperlaneDomain, address: H256, - provider: Box, + provider: CosmosProvider, } impl CosmosRoutingIsm { @@ -32,12 +32,17 @@ impl CosmosRoutingIsm { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider = WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { domain: locator.domain.clone(), address: locator.address, - provider: Box::new(provider), + provider, }) } } @@ -54,7 +59,7 @@ impl HyperlaneChain for CosmosRoutingIsm { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -69,6 +74,7 @@ impl RoutingIsm for CosmosRoutingIsm { let data = self .provider + .grpc() .wasm_query( QueryRoutingIsmGeneralRequest { routing_ism: payload, diff --git a/rust/chains/hyperlane-cosmos/src/validator_announce.rs b/rust/chains/hyperlane-cosmos/src/validator_announce.rs index 69f7121b88..6b0ee04930 100644 --- a/rust/chains/hyperlane-cosmos/src/validator_announce.rs +++ b/rust/chains/hyperlane-cosmos/src/validator_announce.rs @@ -7,7 +7,7 @@ use hyperlane_core::{ }; use crate::{ - grpc::{WasmGrpcProvider, WasmProvider}, + grpc::WasmProvider, payloads::validator_announce::{ self, AnnouncementRequest, AnnouncementRequestInner, GetAnnounceStorageLocationsRequest, GetAnnounceStorageLocationsRequestInner, @@ -22,7 +22,7 @@ use crate::{ pub struct CosmosValidatorAnnounce { domain: HyperlaneDomain, address: H256, - provider: Box, + provider: CosmosProvider, } impl CosmosValidatorAnnounce { @@ -32,12 +32,17 @@ impl CosmosValidatorAnnounce { locator: ContractLocator, signer: Option, ) -> ChainResult { - let provider = WasmGrpcProvider::new(conf.clone(), locator.clone(), signer)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + signer, + )?; Ok(Self { domain: locator.domain.clone(), address: locator.address, - provider: Box::new(provider), + provider, }) } } @@ -54,7 +59,7 @@ impl HyperlaneChain for CosmosValidatorAnnounce { } fn provider(&self) -> Box { - Box::new(CosmosProvider::new(self.domain.clone())) + Box::new(self.provider.clone()) } } @@ -76,7 +81,7 @@ impl ValidatorAnnounce for CosmosValidatorAnnounce { }, }; - let data: Vec = self.provider.wasm_query(payload, None).await?; + let data: Vec = self.provider.grpc().wasm_query(payload, None).await?; let response: validator_announce::GetAnnounceStorageLocationsResponse = serde_json::from_slice(&data)?; @@ -102,6 +107,7 @@ impl ValidatorAnnounce for CosmosValidatorAnnounce { let response: TxResponse = self .provider + .grpc() .wasm_send(announce_request, tx_gas_limit) .await?; diff --git a/rust/chains/hyperlane-ethereum/src/agent_metrics.rs b/rust/chains/hyperlane-ethereum/src/agent_metrics.rs new file mode 100644 index 0000000000..a78528ef05 --- /dev/null +++ b/rust/chains/hyperlane-ethereum/src/agent_metrics.rs @@ -0,0 +1,11 @@ +use async_trait::async_trait; +use hyperlane_core::{metrics::agent::AgenMetricsFetcher, ChainResult, U256}; + +pub struct EthereumMetricsFetcher {} + +#[async_trait] +impl AgenMetricsFetcher for EthereumMetricsFetcher { + async fn get_balance(&self) -> ChainResult { + Ok(0.into()) + } +} diff --git a/rust/chains/hyperlane-ethereum/src/lib.rs b/rust/chains/hyperlane-ethereum/src/lib.rs index 2d42850bc4..27d75b3d57 100644 --- a/rust/chains/hyperlane-ethereum/src/lib.rs +++ b/rust/chains/hyperlane-ethereum/src/lib.rs @@ -74,6 +74,7 @@ mod signers; #[cfg(not(doctest))] mod singleton_signer; +pub mod agent_metrics; mod config; fn extract_fn_map(abi: &'static Lazy) -> HashMap, &'static str> { diff --git a/rust/chains/hyperlane-ethereum/src/provider.rs b/rust/chains/hyperlane-ethereum/src/provider.rs index 6ea06433d6..871fc2a464 100644 --- a/rust/chains/hyperlane-ethereum/src/provider.rs +++ b/rust/chains/hyperlane-ethereum/src/provider.rs @@ -6,7 +6,7 @@ use std::time::Duration; use async_trait::async_trait; use derive_new::new; use ethers::prelude::Middleware; -use hyperlane_core::ethers_core_types; +use hyperlane_core::{ethers_core_types, U256}; use tokio::time::sleep; use tracing::instrument; @@ -105,6 +105,10 @@ where .map_err(ChainCommunicationError::from_other)?; Ok(!code.is_empty()) } + + async fn get_balance(&self, address: String) -> ChainResult { + todo!() + } } impl EthereumProvider diff --git a/rust/chains/hyperlane-fuel/src/provider.rs b/rust/chains/hyperlane-fuel/src/provider.rs index 92303f5795..8048076e04 100644 --- a/rust/chains/hyperlane-fuel/src/provider.rs +++ b/rust/chains/hyperlane-fuel/src/provider.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use hyperlane_core::{ - BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, + BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, }; /// A wrapper around a fuel provider to get generic blockchain information. @@ -31,4 +31,8 @@ impl HyperlaneProvider for FuelProvider { async fn is_contract(&self, address: &H256) -> ChainResult { todo!() } + + async fn get_balance(&self, address: String) -> ChainResult { + todo!() + } } diff --git a/rust/chains/hyperlane-sealevel/src/provider.rs b/rust/chains/hyperlane-sealevel/src/provider.rs index b853e30e4b..3f7449aef2 100644 --- a/rust/chains/hyperlane-sealevel/src/provider.rs +++ b/rust/chains/hyperlane-sealevel/src/provider.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use hyperlane_core::{ - BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, + BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, }; /// A wrapper around a Sealevel provider to get generic blockchain information. @@ -43,4 +43,8 @@ impl HyperlaneProvider for SealevelProvider { // FIXME Ok(true) } + + async fn get_balance(&self, _address: String) -> ChainResult { + todo!() + } } diff --git a/rust/ethers-prometheus/src/middleware/mod.rs b/rust/ethers-prometheus/src/middleware/mod.rs index 18db31d764..19129af612 100644 --- a/rust/ethers-prometheus/src/middleware/mod.rs +++ b/rust/ethers-prometheus/src/middleware/mod.rs @@ -14,7 +14,7 @@ use ethers::abi::AbiEncode; use ethers::prelude::*; use ethers::types::transaction::eip2718::TypedTransaction; use ethers::utils::hex::ToHex; -use log::{debug, trace, warn}; +use log::{debug, trace}; use maplit::hashmap; use prometheus::{CounterVec, GaugeVec, IntCounterVec, IntGaugeVec}; use static_assertions::assert_impl_all; @@ -23,35 +23,11 @@ use tokio::time::MissedTickBehavior; pub use error::PrometheusMiddlewareError; -use crate::contracts::erc_20::Erc20; use crate::u256_as_scaled_f64; pub use crate::ChainInfo; mod error; -/// Some basic information about a token. -#[derive(Clone, Debug)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize))] -#[cfg_attr(feature = "serde", serde(tag = "type", rename_all = "camelCase"))] -pub struct TokenInfo { - /// Full name of the token. E.g. Ether. - pub name: String, - /// Token symbol. E.g. ETH. - pub symbol: String, - /// Number of - pub decimals: u8, -} - -impl Default for TokenInfo { - fn default() -> Self { - Self { - name: "Unknown".into(), - symbol: "".into(), - decimals: 18, - } - } -} - /// Some basic information about a wallet. #[derive(Clone, Debug)] #[cfg_attr(feature = "serde", derive(serde::Deserialize))] @@ -148,18 +124,6 @@ pub const TRANSACTION_SEND_TOTAL_LABELS: &[&str] = /// Help string for the metric. pub const TRANSACTION_SEND_TOTAL_HELP: &str = "Number of transactions sent"; -/// Expected label names for the `wallet_balance` metric. -pub const WALLET_BALANCE_LABELS: &[&str] = &[ - "chain", - "wallet_address", - "wallet_name", - "token_address", - "token_symbol", - "token_name", -]; -/// Help string for the metric. -pub const WALLET_BALANCE_HELP: &str = "Current balance of eth and other tokens in the `tokens` map for the wallet addresses in the `wallets` set"; - /// Container for all the relevant middleware metrics. #[derive(Clone, Builder)] pub struct MiddlewareMetrics { @@ -238,24 +202,23 @@ pub struct MiddlewareMetrics { /// - `txn_status`: `dispatched`, `completed`, or `failed` #[builder(setter(into, strip_option), default)] transaction_send_total: Option, - // /// Gas spent on completed transactions. // /// - `chain`: the chain name (or ID if the name is unknown) of the chain the tx occurred // on. /// - `address_from`: source address of the transaction. // /// - `address_to`: destination address of the transaction. // #[builder(setter(into, strip_option), default)] // transaction_send_gas_eth_total: Option, - /// Current balance of eth and other tokens in the `tokens` map for the - /// wallet addresses in the `wallets` set. - /// - `chain`: the chain name (or chain ID if the name is unknown) of the - /// chain the tx occurred on. - /// - `wallet_address`: Address of the wallet holding the funds. - /// - `wallet_name`: Name of the address holding the funds. - /// - `token_address`: Address of the token. - /// - `token_symbol`: Symbol of the token. - /// - `token_name`: Full name of the token. - #[builder(setter(into, strip_option), default)] - wallet_balance: Option, + // /// Current balance of eth and other tokens in the `tokens` map for the + // /// wallet addresses in the `wallets` set. + // /// - `chain`: the chain name (or chain ID if the name is unknown) of the + // /// chain the tx occurred on. + // /// - `wallet_address`: Address of the wallet holding the funds. + // /// - `wallet_name`: Name of the address holding the funds. + // /// - `token_address`: Address of the token. + // /// - `token_symbol`: Symbol of the token. + // /// - `token_name`: Full name of the token. + // #[builder(setter(into, strip_option), default)] + // wallet_balance: Option, } /// An ethers-rs middleware that instruments calls with prometheus metrics. To @@ -273,14 +236,9 @@ pub struct PrometheusMiddleware { #[cfg_attr(feature = "serde", derive(serde::Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type", rename_all = "camelCase"))] pub struct PrometheusMiddlewareConf { - /// The tokens to track and identifying info - #[cfg_attr(feature = "serde", serde(default))] - pub tokens: HashMap, - - /// The wallets to track and identifying info - #[cfg_attr(feature = "serde", serde(default))] - pub wallets: HashMap, - + // /// The wallets to track and identifying info + // #[cfg_attr(feature = "serde", serde(default))] + // pub wallets: HashMap, /// Contract info for more useful metrics #[cfg_attr(feature = "serde", serde(default))] pub contracts: HashMap, @@ -521,32 +479,6 @@ impl PrometheusMiddleware { conf: Arc::new(RwLock::new(conf)), } } - - /// Start tracking metrics for a new token. - pub async fn track_new_token(&self, addr: Address, info: TokenInfo) { - self.track_new_tokens([(addr, info)]).await; - } - - /// Start tacking metrics for new tokens. - pub async fn track_new_tokens(&self, iter: impl IntoIterator) { - let mut data = self.conf.write().await; - for (addr, info) in iter { - data.tokens.insert(addr, info); - } - } - - /// Start tracking metrics for a new wallet. - pub async fn track_new_wallet(&self, addr: Address, info: WalletInfo) { - self.track_new_wallets([(addr, info)]).await; - } - - /// Start tracking metrics for new wallets. - pub async fn track_new_wallets(&self, iter: impl IntoIterator) { - let mut data = self.conf.write().await; - for (addr, info) in iter { - data.wallets.insert(addr, info); - } - } } impl PrometheusMiddleware { @@ -580,7 +512,7 @@ impl PrometheusMiddleware { /// prometheus scrape interval. pub fn update(&self) -> impl Future { // all metrics are Arcs internally so just clone the ones we want to report for. - let wallet_balance = self.metrics.wallet_balance.clone(); + // let wallet_balance = self.metrics.wallet_balance.clone(); let block_height = self.metrics.block_height.clone(); let gas_price_gwei = self.metrics.gas_price_gwei.clone(); @@ -595,9 +527,9 @@ impl PrometheusMiddleware { if block_height.is_some() || gas_price_gwei.is_some() { Self::update_block_details(&*client, chain, block_height, gas_price_gwei).await; } - if let Some(wallet_balance) = wallet_balance { - Self::update_wallet_balances(client.clone(), &data, chain, wallet_balance).await; - } + // if let Some(wallet_balance) = wallet_balance { + // Self::update_wallet_balances(client.clone(), &data, chain, wallet_balance).await; + // } // more metrics to come... } @@ -609,9 +541,7 @@ impl PrometheusMiddleware { block_height: Option, gas_price_gwei: Option, ) { - let current_block = if let Ok(Some(b)) = client.get_block(BlockNumber::Latest).await { - b - } else { + let Ok(Some(current_block)) = client.get_block(BlockNumber::Latest).await else { return; }; @@ -635,63 +565,6 @@ impl PrometheusMiddleware { } } } - - async fn update_wallet_balances( - client: Arc, - data: &PrometheusMiddlewareConf, - chain: &str, - wallet_balance_metric: GaugeVec, - ) { - for (wallet_addr, wallet_info) in data.wallets.iter() { - let wallet_addr_str: String = wallet_addr.encode_hex(); - let wallet_name = wallet_info.name.as_deref().unwrap_or("none"); - - match client.get_balance(*wallet_addr, None).await { - Ok(balance) => { - // Okay, so the native type is not a token, but whatever, close enough. - // Note: This is ETH for many chains, but not all so that is why we use `N` and `Native` - // TODO: can we get away with scaling as 18 in all cases here? I am guessing not. - let balance = u256_as_scaled_f64(balance, 18); - trace!("Wallet {wallet_name} ({wallet_addr_str}) on chain {chain} balance is {balance} of the native currency"); - wallet_balance_metric - .with(&hashmap! { - "chain" => chain, - "wallet_address" => wallet_addr_str.as_str(), - "wallet_name" => wallet_name, - "token_address" => "none", - "token_symbol" => "Native", - "token_name" => "Native" - }).set(balance) - }, - Err(e) => warn!("Metric update failed for wallet {wallet_name} ({wallet_addr_str}) on chain {chain} balance for native currency; {e}") - } - for (token_addr, token) in data.tokens.iter() { - let token_addr_str: String = token_addr.encode_hex(); - let balance = match Erc20::new(*token_addr, client.clone()) - .balance_of(*wallet_addr) - .call() - .await - { - Ok(b) => u256_as_scaled_f64(b, token.decimals), - Err(e) => { - warn!("Metric update failed for wallet {wallet_name} ({wallet_addr_str}) on chain {chain} balance for {name}; {e}", name=token.name); - continue; - } - }; - trace!("Wallet {wallet_name} ({wallet_addr_str}) on chain {chain} balance is {balance}{}", token.symbol); - wallet_balance_metric - .with(&hashmap! { - "chain" => chain, - "wallet_address" => wallet_addr_str.as_str(), - "wallet_name" => wallet_name, - "token_address" => token_addr_str.as_str(), - "token_symbol" => token.symbol.as_str(), - "token_name" => token.symbol.as_str() - }) - .set(balance); - } - } - } } impl Debug for PrometheusMiddleware { diff --git a/rust/hyperlane-base/Cargo.toml b/rust/hyperlane-base/Cargo.toml index 02d870e644..4e78c30243 100644 --- a/rust/hyperlane-base/Cargo.toml +++ b/rust/hyperlane-base/Cargo.toml @@ -15,6 +15,7 @@ bs58.workspace = true color-eyre = { workspace = true, optional = true } config.workspace = true convert_case.workspace = true +derive_builder.workspace = true derive-new.workspace = true ed25519-dalek.workspace = true ethers.workspace = true @@ -22,6 +23,7 @@ eyre.workspace = true fuels.workspace = true futures-util.workspace = true itertools.workspace = true +maplit.workspace = true paste.workspace = true prometheus.workspace = true rocksdb.workspace = true diff --git a/rust/hyperlane-base/src/metrics/agent.rs b/rust/hyperlane-base/src/metrics/agent.rs new file mode 100644 index 0000000000..a5542b7385 --- /dev/null +++ b/rust/hyperlane-base/src/metrics/agent.rs @@ -0,0 +1,112 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use derive_builder::Builder; +use derive_new::new; +use eyre::Result; +use hyperlane_core::{ + metrics::agent::AgenMetricsFetcher, ChainResult, HyperlaneDomain, H256, U256, +}; +use maplit::hashmap; +use prometheus::GaugeVec; +use tracing::{trace, warn}; + +use crate::CoreMetrics; + +/// Expected label names for the `wallet_balance` metric. +pub const WALLET_BALANCE_LABELS: &[&str] = &[ + "chain", + "wallet_address", + "wallet_name", + "token_address", + "token_symbol", + "token_name", +]; +/// Help string for the metric. +pub const WALLET_BALANCE_HELP: &str = + "Current native token balance for the wallet addresses in the `wallets` set"; + +#[derive(Clone, Builder)] +pub struct AgentMetrics { + /// Current balance of eth and other tokens in the `tokens` map for the + /// wallet addresses in the `wallets` set. + /// - `chain`: the chain name (or chain ID if the name is unknown) of the + /// chain the tx occurred on. + /// - `wallet_address`: Address of the wallet holding the funds. + /// - `wallet_name`: Name of the address holding the funds. + /// - `token_address`: Address of the token. + /// - `token_symbol`: Symbol of the token. + /// - `token_name`: Full name of the token. + #[builder(setter(into, strip_option), default)] + wallet_balance: Option, +} + +pub(crate) fn create_agent_metrics(metrics: &CoreMetrics) -> Result { + Ok(AgentMetricsBuilder::default() + .wallet_balance(metrics.new_gauge( + "wallet_balance", + WALLET_BALANCE_HELP, + WALLET_BALANCE_LABELS, + )?) + .build()?) +} + +/// Configuration for the prometheus middleware. This can be loaded via serde. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type", rename_all = "camelCase"))] +pub struct AgentMetricsConf { + /// The account to track + #[cfg_attr(feature = "serde", serde(default))] + pub address: Option, + + /// Information about the chain this metric is for + pub domain: HyperlaneDomain, + + pub name: String, +} + +#[derive(new)] +pub struct PrometheusAgent { + metrics: AgentMetrics, + conf: AgentMetricsConf, + fetcher: Box, +} + +impl PrometheusAgent { + async fn update_wallet_balances(&self) { + let Some(wallet_addr) = self.conf.address.clone() else { + return; + }; + let wallet_name = self.conf.name.clone(); + let Some(wallet_balance_metric) = self.metrics.wallet_balance.clone() else { + return; + }; + let chain = self.conf.domain.name(); + + match self.fetcher.get_balance().await { + Ok(balance) => { + // Okay, so the native type is not a token, but whatever, close enough. + // Note: This is ETH for many chains, but not all so that is why we use `N` and `Native` + // TODO: can we get away with scaling as 18 in all cases here? I am guessing not. + let balance = u256_as_scaled_f64(U256::from(balance), 18); + trace!("Wallet {wallet_name} ({wallet_addr}) on chain {chain} balance is {balance} of the native currency"); + wallet_balance_metric + .with(&hashmap! { + "chain" => chain, + "wallet_address" => wallet_addr.as_str(), + "wallet_name" => wallet_name.as_str(), + "token_address" => "none", + "token_symbol" => "Native", + "token_name" => "Native" + }).set(balance) + }, + Err(e) => warn!("Metric update failed for wallet {wallet_name} ({wallet_addr}) on chain {chain} balance for native currency; {e}") + } + } +} + +/// Convert a u256 scaled integer value into the corresponding f64 value. +fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { + value.to_f64_lossy() / (10u64.pow(decimals as u32) as f64) +} diff --git a/rust/hyperlane-base/src/metrics/mod.rs b/rust/hyperlane-base/src/metrics/mod.rs index ff30be6dc7..283b5474d9 100644 --- a/rust/hyperlane-base/src/metrics/mod.rs +++ b/rust/hyperlane-base/src/metrics/mod.rs @@ -6,5 +6,6 @@ pub const NAMESPACE: &str = "hyperlane"; mod core; pub use self::core::*; +pub(crate) mod agent; mod json_rpc_client; mod provider; diff --git a/rust/hyperlane-base/src/metrics/provider.rs b/rust/hyperlane-base/src/metrics/provider.rs index 86a9fd5602..54def51ae0 100644 --- a/rust/hyperlane-base/src/metrics/provider.rs +++ b/rust/hyperlane-base/src/metrics/provider.rs @@ -46,10 +46,5 @@ pub(crate) fn create_provider_metrics(metrics: &CoreMetrics) -> Result Result> { + let ctx = "Building Agent Metrics Fetcher"; + + match &self.connection { + ChainConnectionConf::Ethereum(conf) => { + Ok(Box::new(h_eth::agent_metrics::EthereumMetricsFetcher {}) + as Box) + } + ChainConnectionConf::Fuel(_) => todo!(), + ChainConnectionConf::Sealevel(_) => todo!(), + ChainConnectionConf::Cosmos(conf) => { + let signer = self + .cosmos_signer() + .await + .context(ctx)? + .ok_or(eyre!("No signer set")) + .context(ctx)?; + let address = CosmosAddress::from_pubkey(signer.public_key, &conf.get_prefix()) + .context(ctx)?; + let metrics_fetcher = CosmosMetricsFetcher::new( + conf.clone(), + self.locator(self.addresses.mailbox), + address, + )?; + Ok(Box::new(metrics_fetcher) as Box) + } + } + } + async fn signer(&self) -> Result> { if let Some(conf) = &self.signer { Ok(Some(conf.build::().await?)) @@ -639,13 +670,18 @@ impl ChainConf { self.signer().await } + pub async fn agent_metrics_conf(&self, agent_name: String) -> Result { + let chain_signer_address = self.chain_signer().await?.map(|s| s.address_string()); + Ok(AgentMetricsConf { + address: chain_signer_address, + domain: self.domain.clone(), + name: agent_name, + }) + } + /// Get a clone of the ethereum metrics conf with correctly configured /// contract information. - fn metrics_conf( - &self, - agent_name: &str, - signer: &Option, - ) -> PrometheusMiddlewareConf { + pub fn metrics_conf(&self, agent_name: &str) -> PrometheusMiddlewareConf { let mut cfg = self.metrics_conf.clone(); if cfg.chain.is_none() { @@ -654,14 +690,6 @@ impl ChainConf { }); } - if let Some(signer) = signer { - cfg.wallets - .entry(signer.eth_address().into()) - .or_insert_with(|| WalletInfo { - name: Some(agent_name.into()), - }); - } - let mut register_contract = |name: &str, address: H256, fns: HashMap, String>| { cfg.contracts .entry(address.into()) @@ -718,7 +746,7 @@ impl ChainConf { B: BuildableWithProvider + Sync, { let signer = self.ethereum_signer().await?; - let metrics_conf = self.metrics_conf(metrics.agent_name(), &signer); + let metrics_conf = self.metrics_conf(metrics.agent_name()); let rpc_metrics = Some(metrics.json_rpc_client_metrics()); let middleware_metrics = Some((metrics.provider_metrics(), metrics_conf)); let res = builder diff --git a/rust/hyperlane-core/src/chain.rs b/rust/hyperlane-core/src/chain.rs index 94c2c5c44e..01d5ec7c3d 100644 --- a/rust/hyperlane-core/src/chain.rs +++ b/rust/hyperlane-core/src/chain.rs @@ -5,6 +5,7 @@ use std::{ hash::{Hash, Hasher}, }; +use derive_new::new; use num_derive::FromPrimitive; use num_traits::FromPrimitive; #[cfg(feature = "strum")] @@ -18,7 +19,7 @@ pub struct Address(pub bytes::Bytes); #[derive(Debug, Clone)] pub struct Balance(pub num::BigInt); -#[derive(Debug, Clone)] +#[derive(Debug, Clone, new)] pub struct ContractLocator<'a> { pub domain: &'a HyperlaneDomain, pub address: H256, diff --git a/rust/hyperlane-core/src/lib.rs b/rust/hyperlane-core/src/lib.rs index 0e8349186b..3d91f35fcc 100644 --- a/rust/hyperlane-core/src/lib.rs +++ b/rust/hyperlane-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod utils; pub mod test_utils; pub mod config; +pub mod metrics; /// Core hyperlane system data structures mod types; diff --git a/rust/hyperlane-core/src/metrics/agent.rs b/rust/hyperlane-core/src/metrics/agent.rs new file mode 100644 index 0000000000..d124be66d2 --- /dev/null +++ b/rust/hyperlane-core/src/metrics/agent.rs @@ -0,0 +1,17 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use derive_new::new; +use eyre::Result; + +use crate::{ChainResult, U256}; + +#[async_trait] +pub trait AgenMetricsFetcher { + async fn get_balance(&self) -> ChainResult; +} + +// /// Convert a u256 scaled integer value into the corresponding f64 value. +// fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { +// value.to_f64_lossy() / (10u64.pow(decimals as u32) as f64) +// } diff --git a/rust/hyperlane-core/src/metrics/mod.rs b/rust/hyperlane-core/src/metrics/mod.rs new file mode 100644 index 0000000000..f17bc55db8 --- /dev/null +++ b/rust/hyperlane-core/src/metrics/mod.rs @@ -0,0 +1 @@ +pub mod agent; diff --git a/rust/hyperlane-core/src/traits/provider.rs b/rust/hyperlane-core/src/traits/provider.rs index 3f00e7a4ac..ec8cae7052 100644 --- a/rust/hyperlane-core/src/traits/provider.rs +++ b/rust/hyperlane-core/src/traits/provider.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use auto_impl::auto_impl; use thiserror::Error; -use crate::{BlockInfo, ChainResult, HyperlaneChain, TxnInfo, H256}; +use crate::{BlockInfo, ChainResult, HyperlaneChain, TxnInfo, H256, U256}; /// Interface for a provider. Allows abstraction over different provider types /// for different chains. @@ -24,6 +24,9 @@ pub trait HyperlaneProvider: HyperlaneChain + Send + Sync + Debug { /// Returns whether a contract exists at the provided address async fn is_contract(&self, address: &H256) -> ChainResult; + + /// Returns the native currency balance of the given address + async fn get_balance(&self, address: String) -> ChainResult; } /// Errors when querying for provider information. diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 1ecb26dc0c..3b3a907849 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -537,7 +537,7 @@ fn run_locally() { } fn termination_invariants_met(_messages_expected: u32) -> eyre::Result { - Ok(true) + Ok(false) // TODO: uncomment once CI passes consistently on Ubuntu // let gas_payments_scraped = fetch_metric( // "9093", From 42d887eeab0dd2c060577fd378865ecd82ff9160 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 28 Nov 2023 01:26:11 +0000 Subject: [PATCH 02/18] feat: add agent metrics fetchers --- rust/agents/relayer/src/relayer.rs | 69 +++++++++++++------ rust/agents/scraper/src/agent.rs | 29 +++++--- rust/agents/validator/src/validator.rs | 22 ++++-- .../hyperlane-ethereum/src/trait_builder.rs | 3 +- rust/hyperlane-base/src/agent.rs | 25 +++++-- rust/hyperlane-base/src/lib.rs | 2 +- rust/hyperlane-base/src/metrics/agent.rs | 28 ++++++-- rust/hyperlane-base/src/metrics/core.rs | 6 ++ rust/hyperlane-base/src/metrics/mod.rs | 9 ++- rust/hyperlane-core/src/metrics/agent.rs | 9 +-- 10 files changed, 145 insertions(+), 57 deletions(-) diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index 7badb1174a..02fd03b4df 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -9,11 +9,16 @@ use derive_more::AsRef; use eyre::Result; use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, + metrics::{ + self, + agent::{AgentMetrics, Metrics, MetricsFetcher}, + }, run_all, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, WatermarkContractSync, }; use hyperlane_core::{ - HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, MerkleTreeInsertion, U256, + metrics::agent::METRICS_SCRAPE_INTERVAL, HyperlaneDomain, HyperlaneMessage, + InterchainGasPayment, MerkleTreeInsertion, U256, }; use tokio::{ sync::{ @@ -92,11 +97,15 @@ impl BaseAgent for Relayer { type Settings = RelayerSettings; - async fn from_settings(settings: Self::Settings, metrics: Arc) -> Result + async fn from_settings( + settings: Self::Settings, + core_metrics: Arc, + agent_metrics: Metrics, + ) -> Result<(Self, Vec)> where Self: Sized, { - let core = settings.build_hyperlane_core(metrics.clone()); + let core = settings.build_hyperlane_core(core_metrics.clone()); let db = DB::from_path(&settings.db)?; let dbs = settings .origin_chains @@ -105,18 +114,18 @@ impl BaseAgent for Relayer { .collect::>(); let mailboxes = settings - .build_mailboxes(settings.destination_chains.iter(), &metrics) + .build_mailboxes(settings.destination_chains.iter(), &core_metrics) .await?; let validator_announces = settings - .build_validator_announces(settings.origin_chains.iter(), &metrics) + .build_validator_announces(settings.origin_chains.iter(), &core_metrics) .await?; - let contract_sync_metrics = Arc::new(ContractSyncMetrics::new(&metrics)); + let contract_sync_metrics = Arc::new(ContractSyncMetrics::new(&core_metrics)); let message_syncs = settings .build_message_indexers( settings.origin_chains.iter(), - &metrics, + &core_metrics, &contract_sync_metrics, dbs.iter() .map(|(d, db)| (d.clone(), Arc::new(db.clone()) as _)) @@ -126,7 +135,7 @@ impl BaseAgent for Relayer { let interchain_gas_payment_syncs = settings .build_interchain_gas_payment_indexers( settings.origin_chains.iter(), - &metrics, + &core_metrics, &contract_sync_metrics, dbs.iter() .map(|(d, db)| (d.clone(), Arc::new(db.clone()) as _)) @@ -136,7 +145,7 @@ impl BaseAgent for Relayer { let merkle_tree_hook_syncs = settings .build_merkle_tree_hook_indexers( settings.origin_chains.iter(), - &metrics, + &core_metrics, &contract_sync_metrics, dbs.iter() .map(|(d, db)| (d.clone(), Arc::new(db.clone()) as _)) @@ -188,20 +197,29 @@ impl BaseAgent for Relayer { .collect(); let mut msg_ctxs = HashMap::new(); + let mut metrics_fetchers = vec![]; // let mut custom_metrics = HashMap::new(); for destination in &settings.destination_chains { let destination_chain_setup = core.settings.chain_setup(destination).unwrap().clone(); let agent_metrics_conf = destination_chain_setup .agent_metrics_conf("relayer".to_owned()) .await?; - println!("~~~ agent metrics: {:?}", agent_metrics_conf); - println!("~~~ agent signer: {:?}", destination_chain_setup.signer); - // custom_metrics.insert( - // destination.id(), - // destination_chain_setup - // .metrics(destination) - // .expect("Missing metrics config"), - // ); + // PrometheusAgent + let metrics_fetcher = destination_chain_setup + .build_agent_metrics_fetcher() + .await?; + let agent_metrics = + AgentMetrics::new(agent_metrics.clone(), agent_metrics_conf, metrics_fetcher); + + let fetcher_task = tokio::spawn(async move { + agent_metrics + .start_updating_on_interval(METRICS_SCRAPE_INTERVAL) + .await; + Ok(()) + }) + .instrument(info_span!("AgentMetricsFetcher")); + metrics_fetchers.push(fetcher_task); + let transaction_gas_limit: Option = if skip_transaction_gas_limit_for.contains(&destination.id()) { None @@ -232,13 +250,13 @@ impl BaseAgent for Relayer { metadata_builder, origin_gas_payment_enforcer: gas_payment_enforcers[origin].clone(), transaction_gas_limit, - metrics: MessageSubmissionMetrics::new(&metrics, origin, destination), + metrics: MessageSubmissionMetrics::new(&core_metrics, origin, destination), }), ); } } - Ok(Self { + let relayer = Self { dbs, origin_chains: settings.origin_chains, destination_chains: settings.destination_chains, @@ -253,12 +271,19 @@ impl BaseAgent for Relayer { transaction_gas_limit, skip_transaction_gas_limit_for, allow_local_checkpoint_syncers: settings.allow_local_checkpoint_syncers, - }) + }; + + Ok((relayer, metrics_fetchers)) } #[allow(clippy::async_yields_async)] - async fn run(self) -> Instrumented>> { - let mut tasks = vec![]; + async fn run( + self, + metrics_fetchers: Vec, + ) -> Instrumented>> { + // The tasks vec is initially set to the metrics fetcher tasks, + // and is then extended with the rest of the tasks. + let mut tasks = metrics_fetchers; // send channels by destination chain let mut send_channels = HashMap::with_capacity(self.destination_chains.len()); diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index b582f8e2ef..1f7ab20919 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -3,8 +3,10 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; use derive_more::AsRef; use hyperlane_base::{ - run_all, settings::IndexSettings, BaseAgent, ContractSyncMetrics, CoreMetrics, - HyperlaneAgentCore, + metrics::agent::{Metrics as AgentMetrics, MetricsFetcher}, + run_all, + settings::IndexSettings, + BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, }; use hyperlane_core::HyperlaneDomain; use tokio::task::JoinHandle; @@ -38,7 +40,8 @@ impl BaseAgent for Scraper { async fn from_settings( settings: Self::Settings, metrics: Arc, - ) -> eyre::Result + agent_metrics: AgentMetrics, + ) -> eyre::Result<(Self, Vec)> where Self: Sized, { @@ -73,16 +76,22 @@ impl BaseAgent for Scraper { trace!(domain_count = scrapers.len(), "Created scrapers"); - Ok(Self { - core, - metrics, - contract_sync_metrics, - scrapers, - }) + Ok(( + Self { + core, + metrics, + contract_sync_metrics, + scrapers, + }, + Default::default(), + )) } #[allow(clippy::async_yields_async)] - async fn run(self) -> Instrumented>> { + async fn run( + self, + _metrics_fetchers: Vec, + ) -> Instrumented>> { let mut tasks = Vec::with_capacity(self.scrapers.len()); for domain in self.scrapers.keys() { tasks.push(self.scrape(*domain).await); diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index 96feb97b8f..cbf446947f 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -10,6 +10,7 @@ use tracing::{error, info, info_span, instrument::Instrumented, warn, Instrument use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, + metrics::agent::{Metrics as AgentMetrics, MetricsFetcher}, run_all, BaseAgent, CheckpointSyncer, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, }; @@ -51,7 +52,11 @@ impl BaseAgent for Validator { type Settings = ValidatorSettings; - async fn from_settings(settings: Self::Settings, metrics: Arc) -> Result + async fn from_settings( + settings: Self::Settings, + metrics: Arc, + agent_metrics: AgentMetrics, + ) -> Result<(Self, Vec)> where Self: Sized, { @@ -88,7 +93,7 @@ impl BaseAgent for Validator { .await? .into(); - Ok(Self { + let validator = Self { origin_chain: settings.origin_chain, core, db: msg_db, @@ -101,12 +106,19 @@ impl BaseAgent for Validator { reorg_period: settings.reorg_period, interval: settings.interval, checkpoint_syncer, - }) + }; + + Ok((validator, Default::default())) } #[allow(clippy::async_yields_async)] - async fn run(mut self) -> Instrumented>> { - let mut tasks = vec![]; + async fn run( + mut self, + metrics_fetchers: Vec, + ) -> Instrumented>> { + // The tasks vec is initially set to the metrics fetcher tasks, + // and is then extended with the rest of the tasks. + let mut tasks = metrics_fetchers; if let Some(signer_instance) = self.signer_instance.take() { tasks.push( diff --git a/rust/chains/hyperlane-ethereum/src/trait_builder.rs b/rust/chains/hyperlane-ethereum/src/trait_builder.rs index 03a33c2fdc..561be2928f 100644 --- a/rust/chains/hyperlane-ethereum/src/trait_builder.rs +++ b/rust/chains/hyperlane-ethereum/src/trait_builder.rs @@ -10,6 +10,7 @@ use ethers::prelude::{ Http, JsonRpcClient, Middleware, NonceManagerMiddleware, Provider, Quorum, QuorumProvider, SignerMiddleware, WeightedProvider, Ws, WsClientError, }; +use hyperlane_core::metrics::agent::METRICS_SCRAPE_INTERVAL; use reqwest::{Client, Url}; use thiserror::Error; @@ -27,7 +28,6 @@ use hyperlane_core::{ use crate::{signers::Signers, ConnectionConf, FallbackProvider, RetryingProvider}; // This should be whatever the prometheus scrape interval is -const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); const HTTP_CLIENT_TIMEOUT: Duration = Duration::from_secs(60); /// An error when connecting to an ethereum provider. @@ -194,6 +194,7 @@ pub trait BuildableWithProvider { Ok(if let Some(metrics) = metrics { let provider = Arc::new(PrometheusMiddleware::new(provider, metrics.0, metrics.1)); + // This has to be moved tokio::spawn(provider.start_updating_on_interval(METRICS_SCRAPE_INTERVAL)); self.build_with_signer(provider, locator, signer).await? } else { diff --git a/rust/hyperlane-base/src/agent.rs b/rust/hyperlane-base/src/agent.rs index 540a32254c..01e3ee6f4f 100644 --- a/rust/hyperlane-base/src/agent.rs +++ b/rust/hyperlane-base/src/agent.rs @@ -7,7 +7,13 @@ use hyperlane_core::config::*; use tokio::task::JoinHandle; use tracing::{debug_span, instrument::Instrumented, Instrument}; -use crate::{metrics::CoreMetrics, settings::Settings}; +use crate::{ + metrics::{ + agent::{create_agent_metrics, Metrics as AgentMetrics, MetricsFetcher}, + CoreMetrics, + }, + settings::Settings, +}; /// Properties shared across all hyperlane agents #[derive(Debug)] @@ -36,13 +42,20 @@ pub trait BaseAgent: Send + Sync + Debug { type Settings: LoadableFromSettings; /// Instantiate the agent from the standard settings object - async fn from_settings(settings: Self::Settings, metrics: Arc) -> Result + async fn from_settings( + settings: Self::Settings, + metrics: Arc, + agent_metrics: AgentMetrics, + ) -> Result<(Self, Vec)> where Self: Sized; /// Start running this agent. #[allow(clippy::async_yields_async)] - async fn run(self) -> Instrumented>>; + async fn run( + self, + metrics_fetchers: Vec, + ) -> Instrumented>>; } /// Call this from `main` to fully initialize and run the agent for its entire @@ -68,10 +81,12 @@ pub async fn agent_main() -> Result<()> { let metrics = settings.as_ref().metrics(A::AGENT_NAME)?; core_settings.tracing.start_tracing(&metrics)?; - let agent = A::from_settings(settings, metrics.clone()).await?; + let agent_metrics = create_agent_metrics(&metrics)?; + let (agent, metrics_fetchers) = + A::from_settings(settings, metrics.clone(), agent_metrics).await?; metrics.run_http_server(); - agent.run().await.await? + agent.run(metrics_fetchers).await.await? } /// Utility to run multiple tasks and shutdown if any one task ends. diff --git a/rust/hyperlane-base/src/lib.rs b/rust/hyperlane-base/src/lib.rs index eeb3e58c23..b4a0b1cf9c 100644 --- a/rust/hyperlane-base/src/lib.rs +++ b/rust/hyperlane-base/src/lib.rs @@ -12,7 +12,7 @@ pub mod settings; mod agent; pub use agent::*; -mod metrics; +pub mod metrics; pub use metrics::*; mod contract_sync; diff --git a/rust/hyperlane-base/src/metrics/agent.rs b/rust/hyperlane-base/src/metrics/agent.rs index a5542b7385..8fbb74fea2 100644 --- a/rust/hyperlane-base/src/metrics/agent.rs +++ b/rust/hyperlane-base/src/metrics/agent.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use std::future::Future; +use std::{collections::HashMap, time::Duration}; use async_trait::async_trait; use derive_builder::Builder; @@ -9,6 +10,8 @@ use hyperlane_core::{ }; use maplit::hashmap; use prometheus::GaugeVec; +use tokio::time::MissedTickBehavior; +use tracing::instrument::Instrumented; use tracing::{trace, warn}; use crate::CoreMetrics; @@ -26,8 +29,10 @@ pub const WALLET_BALANCE_LABELS: &[&str] = &[ pub const WALLET_BALANCE_HELP: &str = "Current native token balance for the wallet addresses in the `wallets` set"; +pub type MetricsFetcher = Instrumented>>; + #[derive(Clone, Builder)] -pub struct AgentMetrics { +pub struct Metrics { /// Current balance of eth and other tokens in the `tokens` map for the /// wallet addresses in the `wallets` set. /// - `chain`: the chain name (or chain ID if the name is unknown) of the @@ -41,8 +46,8 @@ pub struct AgentMetrics { wallet_balance: Option, } -pub(crate) fn create_agent_metrics(metrics: &CoreMetrics) -> Result { - Ok(AgentMetricsBuilder::default() +pub(crate) fn create_agent_metrics(metrics: &CoreMetrics) -> Result { + Ok(MetricsBuilder::default() .wallet_balance(metrics.new_gauge( "wallet_balance", WALLET_BALANCE_HELP, @@ -67,13 +72,13 @@ pub struct AgentMetricsConf { } #[derive(new)] -pub struct PrometheusAgent { - metrics: AgentMetrics, +pub struct AgentMetrics { + metrics: Metrics, conf: AgentMetricsConf, fetcher: Box, } -impl PrometheusAgent { +impl AgentMetrics { async fn update_wallet_balances(&self) { let Some(wallet_addr) = self.conf.address.clone() else { return; @@ -104,6 +109,15 @@ impl PrometheusAgent { Err(e) => warn!("Metric update failed for wallet {wallet_name} ({wallet_addr}) on chain {chain} balance for native currency; {e}") } } + + pub async fn start_updating_on_interval(self, period: Duration) { + let mut interval = tokio::time::interval(period); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + self.update_wallet_balances().await; + interval.tick().await; + } + } } /// Convert a u256 scaled integer value into the corresponding f64 value. diff --git a/rust/hyperlane-base/src/metrics/core.rs b/rust/hyperlane-base/src/metrics/core.rs index a0c40d8a8e..a11f58d30d 100644 --- a/rust/hyperlane-base/src/metrics/core.rs +++ b/rust/hyperlane-base/src/metrics/core.rs @@ -183,6 +183,12 @@ impl CoreMetrics { }) } + /// Get the prometheus registry this metrics instance is attached to. + /// This can be used to attach new metrics to it. + pub fn registry(&self) -> &Registry { + &self.registry + } + /// Create the provider metrics attached to this core metrics instance. pub fn provider_metrics(&self) -> MiddlewareMetrics { self.provider_metrics diff --git a/rust/hyperlane-base/src/metrics/mod.rs b/rust/hyperlane-base/src/metrics/mod.rs index 283b5474d9..1b032015e9 100644 --- a/rust/hyperlane-base/src/metrics/mod.rs +++ b/rust/hyperlane-base/src/metrics/mod.rs @@ -1,11 +1,16 @@ //! Useful metrics that all agents should track. +pub use self::core::*; + /// The metrics namespace prefix. All metric names will start with `{NAMESPACE}_`. pub const NAMESPACE: &str = "hyperlane"; +// This should be whatever the prometheus scrape interval is +const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); + mod core; -pub use self::core::*; +use std::time::Duration; -pub(crate) mod agent; +pub mod agent; mod json_rpc_client; mod provider; diff --git a/rust/hyperlane-core/src/metrics/agent.rs b/rust/hyperlane-core/src/metrics/agent.rs index d124be66d2..d0d2385d1f 100644 --- a/rust/hyperlane-core/src/metrics/agent.rs +++ b/rust/hyperlane-core/src/metrics/agent.rs @@ -1,13 +1,14 @@ -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; use async_trait::async_trait; -use derive_new::new; -use eyre::Result; use crate::{ChainResult, U256}; +// This should be whatever the prometheus scrape interval is +pub const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); + #[async_trait] -pub trait AgenMetricsFetcher { +pub trait AgenMetricsFetcher: Send + Sync { async fn get_balance(&self) -> ChainResult; } From a400dc86727545892338fbea007c12de9873e6d8 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:55:28 +0000 Subject: [PATCH 03/18] fix: cosmos balance denom usage --- rust/chains/hyperlane-cosmos/src/providers/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/mod.rs b/rust/chains/hyperlane-cosmos/src/providers/mod.rs index 77296ca756..620c97172b 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/mod.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/mod.rs @@ -18,6 +18,7 @@ pub mod rpc; #[derive(Debug, Clone)] pub struct CosmosProvider { domain: HyperlaneDomain, + canonical_asset: String, grpc_client: WasmGrpcProvider, _rpc_client: HttpClient, } @@ -45,6 +46,7 @@ impl CosmosProvider { domain, _rpc_client, grpc_client, + canonical_asset: conf.get_canonical_asset(), }) } @@ -82,7 +84,7 @@ impl HyperlaneProvider for CosmosProvider { // denom is of the form "untrn" Ok(self .grpc_client - .get_balance(address, "".to_string()) + .get_balance(address, self.canonical_asset.clone()) .await? .into()) } From 1b59e8dec8d31ca41031f63d595763c6eaa4f288 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 29 Nov 2023 12:14:08 +0000 Subject: [PATCH 04/18] feat: verify relayer balance in cosmos invariants --- .../hyperlane-cosmos/src/providers/grpc.rs | 7 - rust/utils/run-locally/src/cosmos/mod.rs | 122 ++++++++++++------ rust/utils/run-locally/src/metrics.rs | 17 ++- 3 files changed, 91 insertions(+), 55 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 3a23564fc3..40e5f27d82 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -237,17 +237,10 @@ impl WasmGrpcProvider { .await .map_err(ChainCommunicationError::from_other)? .into_inner(); - // let coin = u128::decode( - // response - // .balance - // .ok_or_else(|| ChainCommunicationError::from_other_str("account not present"))? - // .amount - // ); let balance = response .balance .ok_or_else(|| ChainCommunicationError::from_other_str("account not present"))?; - println!("~~~ relayer balance: {:?}", balance); Ok(balance.amount.parse()?) } diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index e60b98e01f..0ac5c9f69d 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -7,6 +7,7 @@ use std::{env, fs}; use cosmwasm_schema::cw_serde; use hpl_interface::types::bech32_decode; use macro_rules_attribute::apply; +use maplit::hashmap; use tempfile::tempdir; mod cli; @@ -26,7 +27,7 @@ use crate::cosmos::link::link_networks; use crate::logging::log; use crate::program::Program; use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle}; -use crate::AGENT_BIN_PATH; +use crate::{fetch_metric, AGENT_BIN_PATH}; use cli::{OsmosisCLI, OsmosisEndpoint}; use self::deploy::deploy_cw_hyperlane; @@ -460,6 +461,11 @@ fn run_locally() { debug, ); + // give things a chance to fully start. + sleep(Duration::from_secs(5)); + + let starting_relayer_balance: f64 = relayer_balance_sum(hpl_rly_metrics_port).unwrap(); + // dispatch messages let mut dispatched_messages = 0; @@ -517,12 +523,16 @@ fn run_locally() { // Mostly copy-pasta from `rust/utils/run-locally/src/main.rs` // TODO: refactor to share code let loop_start = Instant::now(); - // give things a chance to fully start. - sleep(Duration::from_secs(5)); let mut failure_occurred = false; loop { // look for the end condition. - if termination_invariants_met(hpl_rly_metrics_port, dispatched_messages).unwrap_or(false) { + if termination_invariants_met( + hpl_rly_metrics_port, + dispatched_messages, + starting_relayer_balance, + ) + .unwrap_or(false) + { // end condition reached successfully break; } else if (Instant::now() - loop_start).as_secs() > TIMEOUT_SECS { @@ -542,44 +552,74 @@ fn run_locally() { } } -fn termination_invariants_met(_metrics_port: u32, _messages_expected: u32) -> eyre::Result { - Ok(true) +fn relayer_balance_sum(metrics_port: u32) -> eyre::Result { + let balance = fetch_metric( + &metrics_port.to_string(), + "hyperlane_wallet_balance", + &hashmap! {}, + )? + .iter() + .sum(); + Ok(balance) +} + +fn termination_invariants_met( + relayer_metrics_port: u32, + messages_expected: u32, + starting_relayer_balance: f64, +) -> eyre::Result { // TODO: uncomment once CI passes consistently on Ubuntu - // let gas_payments_scraped = fetch_metric( - // "9093", - // "hyperlane_contract_sync_stored_events", - // &hashmap! {"data_type" => "gas_payment"}, - // )? - // .iter() - // .sum::(); - // let expected_gas_payments = messages_expected; - // if gas_payments_scraped != expected_gas_payments { - // log!( - // "Scraper has scraped {} gas payments, expected {}", - // gas_payments_scraped, - // expected_gas_payments - // ); - // return Ok(false); - // } - - // let delivered_messages_scraped = fetch_metric( - // "9093", - // "hyperlane_operations_processed_count", - // &hashmap! {"phase" => "confirmed"}, - // )? - // .iter() - // .sum::(); - // if delivered_messages_scraped != messages_expected { - // log!( - // "Relayer confirmed {} submitted messages, expected {}", - // delivered_messages_scraped, - // messages_expected - // ); - // return Ok(false); - // } - - // log!("Termination invariants have been meet"); - // Ok(true) + let gas_payments_scraped = fetch_metric( + &relayer_metrics_port.to_string(), + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payment"}, + )? + .iter() + .sum::(); + let expected_gas_payments = messages_expected; + if gas_payments_scraped != expected_gas_payments { + log!( + "Scraper has scraped {} gas payments, expected {}", + gas_payments_scraped, + expected_gas_payments + ); + return Ok(false); + } + + let delivered_messages_scraped = fetch_metric( + &relayer_metrics_port.to_string(), + "hyperlane_operations_processed_count", + &hashmap! {"phase" => "confirmed"}, + )? + .iter() + .sum::(); + if delivered_messages_scraped != messages_expected { + log!( + "Relayer confirmed {} submitted messages, expected {}", + delivered_messages_scraped, + messages_expected + ); + return Ok(false); + } + + let ending_relayer_balance: f64 = relayer_balance_sum(relayer_metrics_port).unwrap(); + // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: + // https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/c2288eb31734ba1f2f997e2c6ecb30176427bc2c/rust/utils/run-locally/src/cosmos/cli.rs#L55 + // What's stopping this is that the format returned by the `uosmo` balance query is a surprisingly low number (0.000003999999995184) + // but then maybe the gas_per_tx is just very low - how can we check that? (maybe by simulating said tx) + if starting_relayer_balance <= ending_relayer_balance { + // worth retrying this because metrics are polled every + // `METRICS_SCRAPE_INTERVAL` + log!( + "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", + starting_relayer_balance, + ending_relayer_balance + ); + return Ok(false); + } + + log!("Termination invariants have been meet"); + Ok(true) } #[cfg(test)] diff --git a/rust/utils/run-locally/src/metrics.rs b/rust/utils/run-locally/src/metrics.rs index 3ee1f17146..9aff78ea3a 100644 --- a/rust/utils/run-locally/src/metrics.rs +++ b/rust/utils/run-locally/src/metrics.rs @@ -1,8 +1,13 @@ -use std::collections::HashMap; +use std::{collections::HashMap, error::Error as StdError, str::FromStr}; -use eyre::{eyre, Result}; +use eyre::{eyre, ErrReport, Result}; -pub fn fetch_metric(port: &str, metric: &str, labels: &HashMap<&str, &str>) -> Result> { +/// Fetch a prometheus format metric, filtering by labels. +pub fn fetch_metric(port: &str, metric: &str, labels: &HashMap<&str, &str>) -> Result> +where + T: FromStr, + E: Into + StdError + Send + Sync + 'static, +{ let resp = ureq::get(&format!("http://127.0.0.1:{}/metrics", port)); resp.call()? .into_string()? @@ -16,10 +21,8 @@ pub fn fetch_metric(port: &str, metric: &str, labels: &HashMap<&str, &str>) -> R .all(|(k, v)| l.contains(&format!("{k}=\"{v}"))) }) .map(|l| { - Ok(l.rsplit_once(' ') - .ok_or(eyre!("Unknown metric format"))? - .1 - .parse::()?) + let value = l.rsplit_once(' ').ok_or(eyre!("Unknown metric format"))?.1; + Ok(value.parse::()?) }) .collect() } From 7d39a381952b8ad194efc49f95e5754ebbe7a737 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 29 Nov 2023 12:49:16 +0000 Subject: [PATCH 05/18] fix: cleanup --- rust/Cargo.lock | 1 + rust/agents/relayer/src/relayer.rs | 13 ++- rust/agents/scraper/src/agent.rs | 10 +- rust/agents/validator/src/validator.rs | 2 +- .../hyperlane-cosmos/src/providers/grpc.rs | 6 +- .../hyperlane-cosmos/src/providers/mod.rs | 13 ++- .../hyperlane-cosmos/src/providers/rpc.rs | 27 +++-- .../hyperlane-ethereum/src/trait_builder.rs | 4 +- rust/ethers-prometheus/Cargo.toml | 1 + rust/ethers-prometheus/src/lib.rs | 5 - rust/ethers-prometheus/src/middleware/mod.rs | 22 +--- rust/hyperlane-base/src/metrics/agent.rs | 6 +- rust/hyperlane-base/src/metrics/core.rs | 6 - rust/hyperlane-core/src/metrics/agent.rs | 9 +- rust/utils/run-locally/src/cosmos/mod.rs | 103 +++++++++--------- 15 files changed, 103 insertions(+), 125 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 73aa67c458..da803cccc2 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2775,6 +2775,7 @@ dependencies = [ "derive_builder", "ethers", "futures", + "hyperlane-core", "log", "maplit", "parking_lot 0.12.1", diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index 02fd03b4df..a779afe32b 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -198,18 +198,19 @@ impl BaseAgent for Relayer { let mut msg_ctxs = HashMap::new(); let mut metrics_fetchers = vec![]; - // let mut custom_metrics = HashMap::new(); for destination in &settings.destination_chains { let destination_chain_setup = core.settings.chain_setup(destination).unwrap().clone(); let agent_metrics_conf = destination_chain_setup .agent_metrics_conf("relayer".to_owned()) .await?; - // PrometheusAgent - let metrics_fetcher = destination_chain_setup + let agent_metrics_fetcher = destination_chain_setup .build_agent_metrics_fetcher() .await?; - let agent_metrics = - AgentMetrics::new(agent_metrics.clone(), agent_metrics_conf, metrics_fetcher); + let agent_metrics = AgentMetrics::new( + agent_metrics.clone(), + agent_metrics_conf, + agent_metrics_fetcher, + ); let fetcher_task = tokio::spawn(async move { agent_metrics @@ -281,7 +282,7 @@ impl BaseAgent for Relayer { self, metrics_fetchers: Vec, ) -> Instrumented>> { - // The tasks vec is initially set to the metrics fetcher tasks, + // The tasks vec is initialized with the metrics fetcher tasks, // and is then extended with the rest of the tasks. let mut tasks = metrics_fetchers; diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index 1f7ab20919..0264177e4c 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -87,12 +87,18 @@ impl BaseAgent for Scraper { )) } + /// Run the scraper + /// + /// * `metrics_fetchers` - A list of metrics fetchers to run. Currently this + /// only comprise #[allow(clippy::async_yields_async)] async fn run( self, - _metrics_fetchers: Vec, + metrics_fetchers: Vec, ) -> Instrumented>> { - let mut tasks = Vec::with_capacity(self.scrapers.len()); + // The tasks vec is initialized with the metrics fetcher tasks, + // and is then extended with the rest of the tasks. + let mut tasks = metrics_fetchers; for domain in self.scrapers.keys() { tasks.push(self.scrape(*domain).await); } diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index cbf446947f..46ffb1051c 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -116,7 +116,7 @@ impl BaseAgent for Validator { mut self, metrics_fetchers: Vec, ) -> Instrumented>> { - // The tasks vec is initially set to the metrics fetcher tasks, + // The tasks vec is initialized with the metrics fetcher tasks, // and is then extended with the rest of the tasks. let mut tasks = metrics_fetchers; diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 40e5f27d82..77b5eabc2a 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -227,11 +227,7 @@ impl WasmGrpcProvider { pub async fn get_balance(&self, address: String, denom: String) -> ChainResult { let mut client = QueryBalanceClient::new(self.channel.clone()); - let balance_request = tonic::Request::new(QueryBalanceRequest { - address, - /// denom is the coin denom to query balances for. - denom, - }); + let balance_request = tonic::Request::new(QueryBalanceRequest { address, denom }); let response = client .balance(balance_request) .await diff --git a/rust/chains/hyperlane-cosmos/src/providers/mod.rs b/rust/chains/hyperlane-cosmos/src/providers/mod.rs index 620c97172b..527f06d2bd 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/mod.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/mod.rs @@ -14,13 +14,13 @@ pub mod grpc; /// cosmos rpc provider pub mod rpc; -/// A reference to a Cosmos chain +/// Abstraction over a connection to a Cosmos chain #[derive(Debug, Clone)] pub struct CosmosProvider { domain: HyperlaneDomain, canonical_asset: String, grpc_client: WasmGrpcProvider, - _rpc_client: HttpClient, + rpc_client: HttpClient, } impl CosmosProvider { @@ -32,7 +32,7 @@ impl CosmosProvider { signer: Option, ) -> ChainResult { let grpc_client = WasmGrpcProvider::new(conf.clone(), locator, signer)?; - let _rpc_client = HttpClient::builder( + let rpc_client = HttpClient::builder( conf.get_rpc_url() .parse() .map_err(Into::::into)?, @@ -44,7 +44,7 @@ impl CosmosProvider { Ok(Self { domain, - _rpc_client, + rpc_client, grpc_client, canonical_asset: conf.get_canonical_asset(), }) @@ -53,6 +53,10 @@ impl CosmosProvider { pub fn grpc(&self) -> WasmGrpcProvider { self.grpc_client.clone() } + + pub fn rpc(&self) -> &HttpClient { + &self.rpc_client + } } impl HyperlaneChain for CosmosProvider { @@ -81,7 +85,6 @@ impl HyperlaneProvider for CosmosProvider { } async fn get_balance(&self, address: String) -> ChainResult { - // denom is of the form "untrn" Ok(self .grpc_client .get_balance(address, self.canonical_asset.clone()) diff --git a/rust/chains/hyperlane-cosmos/src/providers/rpc.rs b/rust/chains/hyperlane-cosmos/src/providers/rpc.rs index 88c5ded065..e8fe8ba48a 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/rpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/rpc.rs @@ -10,7 +10,7 @@ use hyperlane_core::{ChainCommunicationError, ChainResult, ContractLocator, LogM use tracing::{instrument, trace}; use crate::address::CosmosAddress; -use crate::{ConnectionConf, HyperlaneCosmosError}; +use crate::{ConnectionConf, CosmosProvider, HyperlaneCosmosError}; const PAGINATION_LIMIT: u8 = 100; @@ -50,7 +50,7 @@ impl ParsedEvent { #[derive(Debug)] /// Cosmwasm RPC Provider pub struct CosmosWasmIndexer { - client: HttpClient, + provider: CosmosProvider, contract_address: CosmosAddress, target_event_kind: String, reorg_period: u32, @@ -66,17 +66,14 @@ impl CosmosWasmIndexer { event_type: String, reorg_period: u32, ) -> ChainResult { - let client = HttpClient::builder( - conf.get_rpc_url() - .parse() - .map_err(Into::::into)?, - ) - // Consider supporting different compatibility modes. - .compat_mode(CompatMode::latest()) - .build() - .map_err(Into::::into)?; + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + None, + )?; Ok(Self { - client, + provider, contract_address: CosmosAddress::from_h256( locator.address, conf.get_prefix().as_str(), @@ -91,7 +88,8 @@ impl CosmosWasmIndexer { #[instrument(level = "trace", err, skip(self))] async fn tx_search(&self, query: Query, page: u32) -> ChainResult { Ok(self - .client + .provider + .rpc() .tx_search(query, false, page, PAGINATION_LIMIT, Order::Ascending) .await .map_err(Into::::into)?) @@ -176,7 +174,8 @@ impl CosmosWasmIndexer { impl WasmIndexer for CosmosWasmIndexer { async fn get_finalized_block_number(&self) -> ChainResult { let latest_height: u32 = self - .client + .provider + .rpc() .latest_block() .await .map_err(Into::::into)? diff --git a/rust/chains/hyperlane-ethereum/src/trait_builder.rs b/rust/chains/hyperlane-ethereum/src/trait_builder.rs index 561be2928f..89e4f31d4f 100644 --- a/rust/chains/hyperlane-ethereum/src/trait_builder.rs +++ b/rust/chains/hyperlane-ethereum/src/trait_builder.rs @@ -194,7 +194,9 @@ pub trait BuildableWithProvider { Ok(if let Some(metrics) = metrics { let provider = Arc::new(PrometheusMiddleware::new(provider, metrics.0, metrics.1)); - // This has to be moved + // TODO: This task is spawned each time `.build_ethereum(...)` is called, which is about 15 times, + // in spite of it doing the same thing, wasting resources. + // Only spawn this once along with the other agent tasks. tokio::spawn(provider.start_updating_on_interval(METRICS_SCRAPE_INTERVAL)); self.build_with_signer(provider, locator, signer).await? } else { diff --git a/rust/ethers-prometheus/Cargo.toml b/rust/ethers-prometheus/Cargo.toml index 73dc90bd23..92e304d6fa 100644 --- a/rust/ethers-prometheus/Cargo.toml +++ b/rust/ethers-prometheus/Cargo.toml @@ -26,6 +26,7 @@ tokio = { workspace = true, features = ["time", "sync", "parking_lot"] } # enable feature for this crate that is imported by ethers-rs primitive-types = { workspace = true, features = ["fp-conversion"] } +hyperlane-core = { path = "../hyperlane-core", features = ["agent", "float"] } [build-dependencies] abigen = { path = "../utils/abigen", features = ["ethers"] } diff --git a/rust/ethers-prometheus/src/lib.rs b/rust/ethers-prometheus/src/lib.rs index 26a50ecb8b..6268aee6dd 100644 --- a/rust/ethers-prometheus/src/lib.rs +++ b/rust/ethers-prometheus/src/lib.rs @@ -19,8 +19,3 @@ pub struct ChainInfo { /// "kovan". pub name: Option, } - -/// Convert a u256 scaled integer value into the corresponding f64 value. -fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { - value.to_f64_lossy() / (10u64.pow(decimals as u32) as f64) -} diff --git a/rust/ethers-prometheus/src/middleware/mod.rs b/rust/ethers-prometheus/src/middleware/mod.rs index 19129af612..b536a1b13f 100644 --- a/rust/ethers-prometheus/src/middleware/mod.rs +++ b/rust/ethers-prometheus/src/middleware/mod.rs @@ -14,6 +14,7 @@ use ethers::abi::AbiEncode; use ethers::prelude::*; use ethers::types::transaction::eip2718::TypedTransaction; use ethers::utils::hex::ToHex; +use hyperlane_core::metrics::agent::u256_as_scaled_f64; use log::{debug, trace}; use maplit::hashmap; use prometheus::{CounterVec, GaugeVec, IntCounterVec, IntGaugeVec}; @@ -23,7 +24,6 @@ use tokio::time::MissedTickBehavior; pub use error::PrometheusMiddlewareError; -use crate::u256_as_scaled_f64; pub use crate::ChainInfo; mod error; @@ -208,17 +208,6 @@ pub struct MiddlewareMetrics { // /// - `address_to`: destination address of the transaction. // #[builder(setter(into, strip_option), default)] // transaction_send_gas_eth_total: Option, - // /// Current balance of eth and other tokens in the `tokens` map for the - // /// wallet addresses in the `wallets` set. - // /// - `chain`: the chain name (or chain ID if the name is unknown) of the - // /// chain the tx occurred on. - // /// - `wallet_address`: Address of the wallet holding the funds. - // /// - `wallet_name`: Name of the address holding the funds. - // /// - `token_address`: Address of the token. - // /// - `token_symbol`: Symbol of the token. - // /// - `token_name`: Full name of the token. - // #[builder(setter(into, strip_option), default)] - // wallet_balance: Option, } /// An ethers-rs middleware that instruments calls with prometheus metrics. To @@ -236,9 +225,6 @@ pub struct PrometheusMiddleware { #[cfg_attr(feature = "serde", derive(serde::Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type", rename_all = "camelCase"))] pub struct PrometheusMiddlewareConf { - // /// The wallets to track and identifying info - // #[cfg_attr(feature = "serde", serde(default))] - // pub wallets: HashMap, /// Contract info for more useful metrics #[cfg_attr(feature = "serde", serde(default))] pub contracts: HashMap, @@ -512,7 +498,6 @@ impl PrometheusMiddleware { /// prometheus scrape interval. pub fn update(&self) -> impl Future { // all metrics are Arcs internally so just clone the ones we want to report for. - // let wallet_balance = self.metrics.wallet_balance.clone(); let block_height = self.metrics.block_height.clone(); let gas_price_gwei = self.metrics.gas_price_gwei.clone(); @@ -527,9 +512,6 @@ impl PrometheusMiddleware { if block_height.is_some() || gas_price_gwei.is_some() { Self::update_block_details(&*client, chain, block_height, gas_price_gwei).await; } - // if let Some(wallet_balance) = wallet_balance { - // Self::update_wallet_balances(client.clone(), &data, chain, wallet_balance).await; - // } // more metrics to come... } @@ -557,7 +539,7 @@ impl PrometheusMiddleware { } if let Some(gas_price_gwei) = gas_price_gwei { if let Some(london_fee) = current_block.base_fee_per_gas { - let gas = u256_as_scaled_f64(london_fee, 18) * 1e9; + let gas = u256_as_scaled_f64(london_fee.into(), 18) * 1e9; trace!("Gas price for chain {chain} is {gas:.1}gwei"); gas_price_gwei.with(&hashmap! { "chain" => chain }).set(gas); } else { diff --git a/rust/hyperlane-base/src/metrics/agent.rs b/rust/hyperlane-base/src/metrics/agent.rs index 8fbb74fea2..37e21ab3cc 100644 --- a/rust/hyperlane-base/src/metrics/agent.rs +++ b/rust/hyperlane-base/src/metrics/agent.rs @@ -5,6 +5,7 @@ use async_trait::async_trait; use derive_builder::Builder; use derive_new::new; use eyre::Result; +use hyperlane_core::metrics::agent::u256_as_scaled_f64; use hyperlane_core::{ metrics::agent::AgenMetricsFetcher, ChainResult, HyperlaneDomain, H256, U256, }; @@ -119,8 +120,3 @@ impl AgentMetrics { } } } - -/// Convert a u256 scaled integer value into the corresponding f64 value. -fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { - value.to_f64_lossy() / (10u64.pow(decimals as u32) as f64) -} diff --git a/rust/hyperlane-base/src/metrics/core.rs b/rust/hyperlane-base/src/metrics/core.rs index a11f58d30d..a0c40d8a8e 100644 --- a/rust/hyperlane-base/src/metrics/core.rs +++ b/rust/hyperlane-base/src/metrics/core.rs @@ -183,12 +183,6 @@ impl CoreMetrics { }) } - /// Get the prometheus registry this metrics instance is attached to. - /// This can be used to attach new metrics to it. - pub fn registry(&self) -> &Registry { - &self.registry - } - /// Create the provider metrics attached to this core metrics instance. pub fn provider_metrics(&self) -> MiddlewareMetrics { self.provider_metrics diff --git a/rust/hyperlane-core/src/metrics/agent.rs b/rust/hyperlane-core/src/metrics/agent.rs index d0d2385d1f..59349eec4b 100644 --- a/rust/hyperlane-core/src/metrics/agent.rs +++ b/rust/hyperlane-core/src/metrics/agent.rs @@ -12,7 +12,8 @@ pub trait AgenMetricsFetcher: Send + Sync { async fn get_balance(&self) -> ChainResult; } -// /// Convert a u256 scaled integer value into the corresponding f64 value. -// fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { -// value.to_f64_lossy() / (10u64.pow(decimals as u32) as f64) -// } +/// Convert a u256 scaled integer value into the corresponding f64 value. +#[cfg(feature = "float")] +pub fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { + value.to_f64_lossy() / (10u64.pow(decimals as u32) as f64) +} diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 0ac5c9f69d..de20beb631 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -568,58 +568,59 @@ fn termination_invariants_met( messages_expected: u32, starting_relayer_balance: f64, ) -> eyre::Result { - // TODO: uncomment once CI passes consistently on Ubuntu - let gas_payments_scraped = fetch_metric( - &relayer_metrics_port.to_string(), - "hyperlane_contract_sync_stored_events", - &hashmap! {"data_type" => "gas_payment"}, - )? - .iter() - .sum::(); - let expected_gas_payments = messages_expected; - if gas_payments_scraped != expected_gas_payments { - log!( - "Scraper has scraped {} gas payments, expected {}", - gas_payments_scraped, - expected_gas_payments - ); - return Ok(false); - } - - let delivered_messages_scraped = fetch_metric( - &relayer_metrics_port.to_string(), - "hyperlane_operations_processed_count", - &hashmap! {"phase" => "confirmed"}, - )? - .iter() - .sum::(); - if delivered_messages_scraped != messages_expected { - log!( - "Relayer confirmed {} submitted messages, expected {}", - delivered_messages_scraped, - messages_expected - ); - return Ok(false); - } - - let ending_relayer_balance: f64 = relayer_balance_sum(relayer_metrics_port).unwrap(); - // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: - // https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/c2288eb31734ba1f2f997e2c6ecb30176427bc2c/rust/utils/run-locally/src/cosmos/cli.rs#L55 - // What's stopping this is that the format returned by the `uosmo` balance query is a surprisingly low number (0.000003999999995184) - // but then maybe the gas_per_tx is just very low - how can we check that? (maybe by simulating said tx) - if starting_relayer_balance <= ending_relayer_balance { - // worth retrying this because metrics are polled every - // `METRICS_SCRAPE_INTERVAL` - log!( - "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", - starting_relayer_balance, - ending_relayer_balance - ); - return Ok(false); - } - - log!("Termination invariants have been meet"); Ok(true) + // TODO: uncomment once CI passes consistently on Ubuntu + // let gas_payments_scraped = fetch_metric( + // &relayer_metrics_port.to_string(), + // "hyperlane_contract_sync_stored_events", + // &hashmap! {"data_type" => "gas_payment"}, + // )? + // .iter() + // .sum::(); + // let expected_gas_payments = messages_expected; + // if gas_payments_scraped != expected_gas_payments { + // log!( + // "Scraper has scraped {} gas payments, expected {}", + // gas_payments_scraped, + // expected_gas_payments + // ); + // return Ok(false); + // } + + // let delivered_messages_scraped = fetch_metric( + // &relayer_metrics_port.to_string(), + // "hyperlane_operations_processed_count", + // &hashmap! {"phase" => "confirmed"}, + // )? + // .iter() + // .sum::(); + // if delivered_messages_scraped != messages_expected { + // log!( + // "Relayer confirmed {} submitted messages, expected {}", + // delivered_messages_scraped, + // messages_expected + // ); + // return Ok(false); + // } + + // let ending_relayer_balance: f64 = relayer_balance_sum(relayer_metrics_port).unwrap(); + // // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: + // // https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/c2288eb31734ba1f2f997e2c6ecb30176427bc2c/rust/utils/run-locally/src/cosmos/cli.rs#L55 + // // What's stopping this is that the format returned by the `uosmo` balance query is a surprisingly low number (0.000003999999995184) + // // but then maybe the gas_per_tx is just very low - how can we check that? (maybe by simulating said tx) + // if starting_relayer_balance <= ending_relayer_balance { + // // worth retrying this because metrics are polled every + // // `METRICS_SCRAPE_INTERVAL` + // log!( + // "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", + // starting_relayer_balance, + // ending_relayer_balance + // ); + // return Ok(false); + // } + + // log!("Termination invariants have been meet"); + // Ok(true) } #[cfg(test)] From 2b949f413e583e62073f4183575b8ed28dd57d97 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 29 Nov 2023 19:31:15 +0000 Subject: [PATCH 06/18] fix: clippy --- rust/agents/relayer/src/relayer.rs | 11 +++---- rust/agents/scraper/src/agent.rs | 8 ++--- rust/agents/validator/src/validator.rs | 8 ++--- .../{metrics_fetcher.rs => agent_metrics.rs} | 2 ++ .../hyperlane-cosmos/src/aggregation_ism.rs | 2 +- .../hyperlane-cosmos/src/interchain_gas.rs | 1 - .../src/interchain_security_module.rs | 2 +- rust/chains/hyperlane-cosmos/src/lib.rs | 10 +++---- rust/chains/hyperlane-cosmos/src/mailbox.rs | 5 +--- .../hyperlane-cosmos/src/merkle_tree_hook.rs | 2 +- .../hyperlane-cosmos/src/providers/mod.rs | 2 ++ .../hyperlane-cosmos/src/providers/rpc.rs | 2 +- .../hyperlane-cosmos/src/routing_ism.rs | 2 +- .../hyperlane-ethereum/src/agent_metrics.rs | 1 + rust/chains/hyperlane-ethereum/src/lib.rs | 4 ++- .../chains/hyperlane-ethereum/src/provider.rs | 2 +- rust/ethers-prometheus/src/lib.rs | 2 -- rust/ethers-prometheus/src/middleware/mod.rs | 2 +- rust/hyperlane-base/src/agent.rs | 7 ++--- .../metrics/{agent.rs => agent_metrics.rs} | 17 ++++++----- rust/hyperlane-base/src/metrics/mod.rs | 8 ++--- rust/hyperlane-base/src/settings/chains.rs | 29 +++++++++---------- rust/hyperlane-core/src/lib.rs | 1 + rust/hyperlane-core/src/metrics/agent.rs | 8 +++-- rust/hyperlane-core/src/metrics/mod.rs | 1 + rust/utils/run-locally/Cargo.toml | 2 +- rust/utils/run-locally/src/cosmos/mod.rs | 6 ++-- rust/utils/run-locally/src/main.rs | 2 +- 28 files changed, 75 insertions(+), 74 deletions(-) rename rust/chains/hyperlane-cosmos/src/{metrics_fetcher.rs => agent_metrics.rs} (90%) rename rust/hyperlane-base/src/metrics/{agent.rs => agent_metrics.rs} (89%) diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index a779afe32b..af7df960ae 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -9,10 +9,7 @@ use derive_more::AsRef; use eyre::Result; use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, - metrics::{ - self, - agent::{AgentMetrics, Metrics, MetricsFetcher}, - }, + metrics::{AgentMetrics, InstrumentedFallibleTask, Metrics}, run_all, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, WatermarkContractSync, }; @@ -101,7 +98,7 @@ impl BaseAgent for Relayer { settings: Self::Settings, core_metrics: Arc, agent_metrics: Metrics, - ) -> Result<(Self, Vec)> + ) -> Result<(Self, Vec>)> where Self: Sized, { @@ -201,7 +198,7 @@ impl BaseAgent for Relayer { for destination in &settings.destination_chains { let destination_chain_setup = core.settings.chain_setup(destination).unwrap().clone(); let agent_metrics_conf = destination_chain_setup - .agent_metrics_conf("relayer".to_owned()) + .agent_metrics_conf(Self::AGENT_NAME.to_string()) .await?; let agent_metrics_fetcher = destination_chain_setup .build_agent_metrics_fetcher() @@ -280,7 +277,7 @@ impl BaseAgent for Relayer { #[allow(clippy::async_yields_async)] async fn run( self, - metrics_fetchers: Vec, + metrics_fetchers: Vec>, ) -> Instrumented>> { // The tasks vec is initialized with the metrics fetcher tasks, // and is then extended with the rest of the tasks. diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index 0264177e4c..24549b3765 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; use derive_more::AsRef; use hyperlane_base::{ - metrics::agent::{Metrics as AgentMetrics, MetricsFetcher}, + metrics::{InstrumentedFallibleTask, Metrics as AgentMetrics}, run_all, settings::IndexSettings, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, @@ -40,8 +40,8 @@ impl BaseAgent for Scraper { async fn from_settings( settings: Self::Settings, metrics: Arc, - agent_metrics: AgentMetrics, - ) -> eyre::Result<(Self, Vec)> + _agent_metrics: AgentMetrics, + ) -> eyre::Result<(Self, Vec>)> where Self: Sized, { @@ -94,7 +94,7 @@ impl BaseAgent for Scraper { #[allow(clippy::async_yields_async)] async fn run( self, - metrics_fetchers: Vec, + metrics_fetchers: Vec>, ) -> Instrumented>> { // The tasks vec is initialized with the metrics fetcher tasks, // and is then extended with the rest of the tasks. diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index 46ffb1051c..2fd90b4edb 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -10,7 +10,7 @@ use tracing::{error, info, info_span, instrument::Instrumented, warn, Instrument use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, - metrics::agent::{Metrics as AgentMetrics, MetricsFetcher}, + metrics::{InstrumentedFallibleTask, Metrics as AgentMetrics}, run_all, BaseAgent, CheckpointSyncer, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, }; @@ -55,8 +55,8 @@ impl BaseAgent for Validator { async fn from_settings( settings: Self::Settings, metrics: Arc, - agent_metrics: AgentMetrics, - ) -> Result<(Self, Vec)> + _agent_metrics: AgentMetrics, + ) -> Result<(Self, Vec>)> where Self: Sized, { @@ -114,7 +114,7 @@ impl BaseAgent for Validator { #[allow(clippy::async_yields_async)] async fn run( mut self, - metrics_fetchers: Vec, + metrics_fetchers: Vec>, ) -> Instrumented>> { // The tasks vec is initialized with the metrics fetcher tasks, // and is then extended with the rest of the tasks. diff --git a/rust/chains/hyperlane-cosmos/src/metrics_fetcher.rs b/rust/chains/hyperlane-cosmos/src/agent_metrics.rs similarity index 90% rename from rust/chains/hyperlane-cosmos/src/metrics_fetcher.rs rename to rust/chains/hyperlane-cosmos/src/agent_metrics.rs index 3cb20419c0..30c15f98f3 100644 --- a/rust/chains/hyperlane-cosmos/src/metrics_fetcher.rs +++ b/rust/chains/hyperlane-cosmos/src/agent_metrics.rs @@ -6,6 +6,7 @@ use hyperlane_core::{ use crate::{address::CosmosAddress, ConnectionConf, CosmosProvider}; +/// Concrete struct for implementing the AgenMetricsFetcher and HyperlaneChain traits for Cosmos #[derive(Debug)] pub struct CosmosMetricsFetcher { address: CosmosAddress, @@ -14,6 +15,7 @@ pub struct CosmosMetricsFetcher { } impl CosmosMetricsFetcher { + /// Instiante a new CosmosMetricsFetcher pub fn new( conf: ConnectionConf, locator: ContractLocator, diff --git a/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs b/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs index a2439f92a3..49e5d60397 100644 --- a/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use crate::{ address::CosmosAddress, - grpc::{WasmGrpcProvider, WasmProvider}, + grpc::WasmProvider, payloads::aggregate_ism::{ModulesAndThresholdRequest, ModulesAndThresholdResponse}, ConnectionConf, CosmosProvider, Signer, }; diff --git a/rust/chains/hyperlane-cosmos/src/interchain_gas.rs b/rust/chains/hyperlane-cosmos/src/interchain_gas.rs index 83044913b4..491274c846 100644 --- a/rust/chains/hyperlane-cosmos/src/interchain_gas.rs +++ b/rust/chains/hyperlane-cosmos/src/interchain_gas.rs @@ -10,7 +10,6 @@ use once_cell::sync::Lazy; use std::ops::RangeInclusive; use crate::{ - grpc::WasmGrpcProvider, rpc::{CosmosWasmIndexer, ParsedEvent, WasmIndexer}, signers::Signer, utils::{CONTRACT_ADDRESS_ATTRIBUTE_KEY, CONTRACT_ADDRESS_ATTRIBUTE_KEY_BASE64}, diff --git a/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs b/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs index f4c6932cf5..9e726e5628 100644 --- a/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs +++ b/rust/chains/hyperlane-cosmos/src/interchain_security_module.rs @@ -5,7 +5,7 @@ use hyperlane_core::{ }; use crate::{ - grpc::{WasmGrpcProvider, WasmProvider}, + grpc::WasmProvider, payloads::{ general::EmptyStruct, ism_routes::{QueryIsmGeneralRequest, QueryIsmModuleTypeRequest}, diff --git a/rust/chains/hyperlane-cosmos/src/lib.rs b/rust/chains/hyperlane-cosmos/src/lib.rs index 92b3a56730..c61f1de261 100644 --- a/rust/chains/hyperlane-cosmos/src/lib.rs +++ b/rust/chains/hyperlane-cosmos/src/lib.rs @@ -5,6 +5,7 @@ // TODO: Remove once we start filling things in #![allow(unused_variables)] +mod agent_metrics; mod aggregation_ism; mod error; mod interchain_gas; @@ -12,7 +13,6 @@ mod interchain_security_module; mod libs; mod mailbox; mod merkle_tree_hook; -mod metrics_fetcher; mod multisig_ism; mod payloads; mod providers; @@ -24,8 +24,8 @@ mod utils; mod validator_announce; pub use self::{ - aggregation_ism::*, error::*, interchain_gas::*, interchain_security_module::*, libs::*, - mailbox::*, merkle_tree_hook::*, metrics_fetcher::*, multisig_ism::*, providers::*, - routing_ism::*, signers::*, trait_builder::*, trait_builder::*, validator_announce::*, - validator_announce::*, + agent_metrics::*, aggregation_ism::*, error::*, interchain_gas::*, + interchain_security_module::*, libs::*, mailbox::*, merkle_tree_hook::*, multisig_ism::*, + providers::*, routing_ism::*, signers::*, trait_builder::*, trait_builder::*, + validator_announce::*, validator_announce::*, }; diff --git a/rust/chains/hyperlane-cosmos/src/mailbox.rs b/rust/chains/hyperlane-cosmos/src/mailbox.rs index 51e8529bc3..2249546b77 100644 --- a/rust/chains/hyperlane-cosmos/src/mailbox.rs +++ b/rust/chains/hyperlane-cosmos/src/mailbox.rs @@ -14,10 +14,7 @@ use crate::payloads::{general, mailbox}; use crate::rpc::{CosmosWasmIndexer, ParsedEvent, WasmIndexer}; use crate::CosmosProvider; use crate::{address::CosmosAddress, types::tx_response_to_outcome}; -use crate::{ - grpc::{WasmGrpcProvider, WasmProvider}, - HyperlaneCosmosError, -}; +use crate::{grpc::WasmProvider, HyperlaneCosmosError}; use crate::{signers::Signer, utils::get_block_height_for_lag, ConnectionConf}; use async_trait::async_trait; use cosmrs::proto::cosmos::base::abci::v1beta1::TxResponse; diff --git a/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs b/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs index f18f5a2a83..40a0a2ab55 100644 --- a/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs +++ b/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs @@ -12,7 +12,7 @@ use once_cell::sync::Lazy; use tracing::instrument; use crate::{ - grpc::{WasmGrpcProvider, WasmProvider}, + grpc::WasmProvider, payloads::{ general::{self}, merkle_tree_hook, diff --git a/rust/chains/hyperlane-cosmos/src/providers/mod.rs b/rust/chains/hyperlane-cosmos/src/providers/mod.rs index 527f06d2bd..4bb960b536 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/mod.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/mod.rs @@ -50,10 +50,12 @@ impl CosmosProvider { }) } + /// Get a grpc client pub fn grpc(&self) -> WasmGrpcProvider { self.grpc_client.clone() } + /// Get an rpc client pub fn rpc(&self) -> &HttpClient { &self.rpc_client } diff --git a/rust/chains/hyperlane-cosmos/src/providers/rpc.rs b/rust/chains/hyperlane-cosmos/src/providers/rpc.rs index e8fe8ba48a..1f0d2a24a1 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/rpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/rpc.rs @@ -1,7 +1,7 @@ use std::ops::RangeInclusive; use async_trait::async_trait; -use cosmrs::rpc::client::{Client, CompatMode, HttpClient}; +use cosmrs::rpc::client::Client; use cosmrs::rpc::endpoint::{tx, tx_search::Response as TxSearchResponse}; use cosmrs::rpc::query::Query; use cosmrs::rpc::Order; diff --git a/rust/chains/hyperlane-cosmos/src/routing_ism.rs b/rust/chains/hyperlane-cosmos/src/routing_ism.rs index 97360546f4..63b759f1b9 100644 --- a/rust/chains/hyperlane-cosmos/src/routing_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/routing_ism.rs @@ -9,7 +9,7 @@ use hyperlane_core::{ use crate::{ address::CosmosAddress, - grpc::{WasmGrpcProvider, WasmProvider}, + grpc::WasmProvider, payloads::ism_routes::{ IsmRouteRequest, IsmRouteRequestInner, IsmRouteRespnose, QueryRoutingIsmGeneralRequest, }, diff --git a/rust/chains/hyperlane-ethereum/src/agent_metrics.rs b/rust/chains/hyperlane-ethereum/src/agent_metrics.rs index a78528ef05..f1bd5065ba 100644 --- a/rust/chains/hyperlane-ethereum/src/agent_metrics.rs +++ b/rust/chains/hyperlane-ethereum/src/agent_metrics.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use hyperlane_core::{metrics::agent::AgenMetricsFetcher, ChainResult, U256}; +/// Concrete struct for implementing the AgenMetricsFetcher trait for Ethereum pub struct EthereumMetricsFetcher {} #[async_trait] diff --git a/rust/chains/hyperlane-ethereum/src/lib.rs b/rust/chains/hyperlane-ethereum/src/lib.rs index 27d75b3d57..6f3d9b268b 100644 --- a/rust/chains/hyperlane-ethereum/src/lib.rs +++ b/rust/chains/hyperlane-ethereum/src/lib.rs @@ -74,9 +74,11 @@ mod signers; #[cfg(not(doctest))] mod singleton_signer; -pub mod agent_metrics; +mod agent_metrics; mod config; +pub use self::agent_metrics::*; + fn extract_fn_map(abi: &'static Lazy) -> HashMap, &'static str> { abi.functions() .map(|f| (f.selector().to_vec(), f.name.as_str())) diff --git a/rust/chains/hyperlane-ethereum/src/provider.rs b/rust/chains/hyperlane-ethereum/src/provider.rs index 871fc2a464..e23b19a779 100644 --- a/rust/chains/hyperlane-ethereum/src/provider.rs +++ b/rust/chains/hyperlane-ethereum/src/provider.rs @@ -106,7 +106,7 @@ where Ok(!code.is_empty()) } - async fn get_balance(&self, address: String) -> ChainResult { + async fn get_balance(&self, _address: String) -> ChainResult { todo!() } } diff --git a/rust/ethers-prometheus/src/lib.rs b/rust/ethers-prometheus/src/lib.rs index 6268aee6dd..8cf57329f0 100644 --- a/rust/ethers-prometheus/src/lib.rs +++ b/rust/ethers-prometheus/src/lib.rs @@ -3,8 +3,6 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] -use ethers::prelude::U256; - mod contracts; pub mod json_rpc_client; diff --git a/rust/ethers-prometheus/src/middleware/mod.rs b/rust/ethers-prometheus/src/middleware/mod.rs index b536a1b13f..1526f84e72 100644 --- a/rust/ethers-prometheus/src/middleware/mod.rs +++ b/rust/ethers-prometheus/src/middleware/mod.rs @@ -20,9 +20,9 @@ use maplit::hashmap; use prometheus::{CounterVec, GaugeVec, IntCounterVec, IntGaugeVec}; use static_assertions::assert_impl_all; use tokio::sync::RwLock; -use tokio::time::MissedTickBehavior; pub use error::PrometheusMiddlewareError; +use tokio::time::MissedTickBehavior; pub use crate::ChainInfo; diff --git a/rust/hyperlane-base/src/agent.rs b/rust/hyperlane-base/src/agent.rs index 01e3ee6f4f..e638108dcf 100644 --- a/rust/hyperlane-base/src/agent.rs +++ b/rust/hyperlane-base/src/agent.rs @@ -9,8 +9,7 @@ use tracing::{debug_span, instrument::Instrumented, Instrument}; use crate::{ metrics::{ - agent::{create_agent_metrics, Metrics as AgentMetrics, MetricsFetcher}, - CoreMetrics, + create_agent_metrics, CoreMetrics, InstrumentedFallibleTask, Metrics as AgentMetrics, }, settings::Settings, }; @@ -46,7 +45,7 @@ pub trait BaseAgent: Send + Sync + Debug { settings: Self::Settings, metrics: Arc, agent_metrics: AgentMetrics, - ) -> Result<(Self, Vec)> + ) -> Result<(Self, Vec>)> where Self: Sized; @@ -54,7 +53,7 @@ pub trait BaseAgent: Send + Sync + Debug { #[allow(clippy::async_yields_async)] async fn run( self, - metrics_fetchers: Vec, + metrics_fetchers: Vec>, ) -> Instrumented>>; } diff --git a/rust/hyperlane-base/src/metrics/agent.rs b/rust/hyperlane-base/src/metrics/agent_metrics.rs similarity index 89% rename from rust/hyperlane-base/src/metrics/agent.rs rename to rust/hyperlane-base/src/metrics/agent_metrics.rs index 37e21ab3cc..267197e9b4 100644 --- a/rust/hyperlane-base/src/metrics/agent.rs +++ b/rust/hyperlane-base/src/metrics/agent_metrics.rs @@ -1,14 +1,10 @@ -use std::future::Future; -use std::{collections::HashMap, time::Duration}; +use std::time::Duration; -use async_trait::async_trait; use derive_builder::Builder; use derive_new::new; use eyre::Result; use hyperlane_core::metrics::agent::u256_as_scaled_f64; -use hyperlane_core::{ - metrics::agent::AgenMetricsFetcher, ChainResult, HyperlaneDomain, H256, U256, -}; +use hyperlane_core::{metrics::agent::AgenMetricsFetcher, HyperlaneDomain}; use maplit::hashmap; use prometheus::GaugeVec; use tokio::time::MissedTickBehavior; @@ -30,8 +26,10 @@ pub const WALLET_BALANCE_LABELS: &[&str] = &[ pub const WALLET_BALANCE_HELP: &str = "Current native token balance for the wallet addresses in the `wallets` set"; -pub type MetricsFetcher = Instrumented>>; +/// Instrumented fallible task alias +pub type InstrumentedFallibleTask = Instrumented>>; +/// Agent-specific metrics #[derive(Clone, Builder)] pub struct Metrics { /// Current balance of eth and other tokens in the `tokens` map for the @@ -69,9 +67,11 @@ pub struct AgentMetricsConf { /// Information about the chain this metric is for pub domain: HyperlaneDomain, + /// Name of the agent the metrics are about pub name: String, } +/// Utility struct to update agent metrics #[derive(new)] pub struct AgentMetrics { metrics: Metrics, @@ -95,7 +95,7 @@ impl AgentMetrics { // Okay, so the native type is not a token, but whatever, close enough. // Note: This is ETH for many chains, but not all so that is why we use `N` and `Native` // TODO: can we get away with scaling as 18 in all cases here? I am guessing not. - let balance = u256_as_scaled_f64(U256::from(balance), 18); + let balance = u256_as_scaled_f64(balance, 18); trace!("Wallet {wallet_name} ({wallet_addr}) on chain {chain} balance is {balance} of the native currency"); wallet_balance_metric .with(&hashmap! { @@ -111,6 +111,7 @@ impl AgentMetrics { } } + /// Periodically updates the metrics pub async fn start_updating_on_interval(self, period: Duration) { let mut interval = tokio::time::interval(period); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); diff --git a/rust/hyperlane-base/src/metrics/mod.rs b/rust/hyperlane-base/src/metrics/mod.rs index 1b032015e9..b2b1c6acd9 100644 --- a/rust/hyperlane-base/src/metrics/mod.rs +++ b/rust/hyperlane-base/src/metrics/mod.rs @@ -5,12 +5,10 @@ pub use self::core::*; /// The metrics namespace prefix. All metric names will start with `{NAMESPACE}_`. pub const NAMESPACE: &str = "hyperlane"; -// This should be whatever the prometheus scrape interval is -const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); - mod core; -use std::time::Duration; -pub mod agent; +mod agent_metrics; mod json_rpc_client; mod provider; + +pub use self::agent_metrics::*; diff --git a/rust/hyperlane-base/src/settings/chains.rs b/rust/hyperlane-base/src/settings/chains.rs index a266658b49..a40abeb531 100644 --- a/rust/hyperlane-base/src/settings/chains.rs +++ b/rust/hyperlane-base/src/settings/chains.rs @@ -1,18 +1,16 @@ use ethers::prelude::Selector; -use h_cosmos::{address::CosmosAddress, CosmosMetricsFetcher, CosmosProvider}; +use h_cosmos::{address::CosmosAddress, CosmosMetricsFetcher}; use std::collections::HashMap; use eyre::{eyre, Context, Result}; -use ethers_prometheus::middleware::{ - ChainInfo, ContractInfo, PrometheusMiddlewareConf, WalletInfo, -}; +use ethers_prometheus::middleware::{ChainInfo, ContractInfo, PrometheusMiddlewareConf}; use hyperlane_core::{ - metrics::agent::AgenMetricsFetcher, AggregationIsm, CcipReadIsm, ChainResult, ContractLocator, - HyperlaneAbi, HyperlaneDomain, HyperlaneDomainProtocol, HyperlaneMessage, HyperlaneProvider, - HyperlaneSigner, IndexMode, InterchainGasPaymaster, InterchainGasPayment, - InterchainSecurityModule, Mailbox, MerkleTreeHook, MerkleTreeInsertion, MultisigIsm, - RoutingIsm, SequenceIndexer, ValidatorAnnounce, H256, + metrics::agent::AgenMetricsFetcher, AggregationIsm, CcipReadIsm, ContractLocator, HyperlaneAbi, + HyperlaneDomain, HyperlaneDomainProtocol, HyperlaneMessage, HyperlaneProvider, IndexMode, + InterchainGasPaymaster, InterchainGasPayment, InterchainSecurityModule, Mailbox, + MerkleTreeHook, MerkleTreeInsertion, MultisigIsm, RoutingIsm, SequenceIndexer, + ValidatorAnnounce, H256, }; use hyperlane_cosmos as h_cosmos; use hyperlane_ethereum::{ @@ -23,7 +21,7 @@ use hyperlane_fuel as h_fuel; use hyperlane_sealevel as h_sealevel; use crate::{ - metrics::agent::AgentMetricsConf, + metrics::AgentMetricsConf, settings::signers::{BuildableWithSignerConf, ChainSigner, SignerConf}, CoreMetrics, }; @@ -596,13 +594,13 @@ impl ChainConf { .context(ctx) } + /// Try to convert the chain setting into a trait object for fetching agent metrics pub async fn build_agent_metrics_fetcher(&self) -> Result> { let ctx = "Building Agent Metrics Fetcher"; match &self.connection { - ChainConnectionConf::Ethereum(conf) => { - Ok(Box::new(h_eth::agent_metrics::EthereumMetricsFetcher {}) - as Box) + ChainConnectionConf::Ethereum(_conf) => { + Ok(Box::new(h_eth::EthereumMetricsFetcher {}) as Box) } ChainConnectionConf::Fuel(_) => todo!(), ChainConnectionConf::Sealevel(_) => todo!(), @@ -670,6 +668,7 @@ impl ChainConf { self.signer().await } + /// Try to build an agent metrics configuration from the chain config pub async fn agent_metrics_conf(&self, agent_name: String) -> Result { let chain_signer_address = self.chain_signer().await?.map(|s| s.address_string()); Ok(AgentMetricsConf { @@ -681,7 +680,7 @@ impl ChainConf { /// Get a clone of the ethereum metrics conf with correctly configured /// contract information. - pub fn metrics_conf(&self, agent_name: &str) -> PrometheusMiddlewareConf { + pub fn metrics_conf(&self) -> PrometheusMiddlewareConf { let mut cfg = self.metrics_conf.clone(); if cfg.chain.is_none() { @@ -746,7 +745,7 @@ impl ChainConf { B: BuildableWithProvider + Sync, { let signer = self.ethereum_signer().await?; - let metrics_conf = self.metrics_conf(metrics.agent_name()); + let metrics_conf = self.metrics_conf(); let rpc_metrics = Some(metrics.json_rpc_client_metrics()); let middleware_metrics = Some((metrics.provider_metrics(), metrics_conf)); let res = builder diff --git a/rust/hyperlane-core/src/lib.rs b/rust/hyperlane-core/src/lib.rs index 3d91f35fcc..6834df3951 100644 --- a/rust/hyperlane-core/src/lib.rs +++ b/rust/hyperlane-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod utils; pub mod test_utils; pub mod config; +/// Prometheus metrics traits / utilities pub mod metrics; /// Core hyperlane system data structures diff --git a/rust/hyperlane-core/src/metrics/agent.rs b/rust/hyperlane-core/src/metrics/agent.rs index 59349eec4b..6134e2b8e5 100644 --- a/rust/hyperlane-core/src/metrics/agent.rs +++ b/rust/hyperlane-core/src/metrics/agent.rs @@ -1,14 +1,18 @@ -use std::{collections::HashMap, time::Duration}; +use std::time::Duration; use async_trait::async_trait; use crate::{ChainResult, U256}; -// This should be whatever the prometheus scrape interval is +/// Interval for querying the prometheus metrics endpoint. +/// This should be whatever the prometheus scrape interval is pub const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); +/// Trait to be implemented by all chain-specific agent implementations, +/// to support gathering agent metrics. #[async_trait] pub trait AgenMetricsFetcher: Send + Sync { + /// Fetch the balance of the wallet address associated with the chain provider. async fn get_balance(&self) -> ChainResult; } diff --git a/rust/hyperlane-core/src/metrics/mod.rs b/rust/hyperlane-core/src/metrics/mod.rs index f17bc55db8..4c82be31d7 100644 --- a/rust/hyperlane-core/src/metrics/mod.rs +++ b/rust/hyperlane-core/src/metrics/mod.rs @@ -1 +1,2 @@ +/// Agent metrics utils pub mod agent; diff --git a/rust/utils/run-locally/Cargo.toml b/rust/utils/run-locally/Cargo.toml index 1f59fb2fb3..66e86a92bc 100644 --- a/rust/utils/run-locally/Cargo.toml +++ b/rust/utils/run-locally/Cargo.toml @@ -10,7 +10,7 @@ publish.workspace = true version.workspace = true [dependencies] -hyperlane-core = { path = "../../hyperlane-core" } +hyperlane-core = { path = "../../hyperlane-core", features = ["float"]} toml_edit.workspace = true k256.workspace = true ripemd.workspace = true diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index de20beb631..5ff4402527 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -564,9 +564,9 @@ fn relayer_balance_sum(metrics_port: u32) -> eyre::Result { } fn termination_invariants_met( - relayer_metrics_port: u32, - messages_expected: u32, - starting_relayer_balance: f64, + _relayer_metrics_port: u32, + _messages_expected: u32, + _starting_relayer_balance: f64, ) -> eyre::Result { Ok(true) // TODO: uncomment once CI passes consistently on Ubuntu diff --git a/rust/utils/run-locally/src/main.rs b/rust/utils/run-locally/src/main.rs index 0ee63b02d4..0910d14710 100644 --- a/rust/utils/run-locally/src/main.rs +++ b/rust/utils/run-locally/src/main.rs @@ -388,7 +388,7 @@ fn main() -> ExitCode { let loop_start = Instant::now(); // give things a chance to fully start. - sleep(Duration::from_secs(5)); + sleep(Duration::from_secs(10)); let mut failure_occurred = false; while !SHUTDOWN.load(Ordering::Relaxed) { if config.ci_mode { From d091ced5b3835706bfda07bbacfb22f1558631ca Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 29 Nov 2023 23:25:52 +0000 Subject: [PATCH 07/18] fix: wait for longer before querying metrics in cosmos e2e --- rust/utils/run-locally/src/cosmos/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 5ff4402527..78c5e1d6bd 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -462,7 +462,7 @@ fn run_locally() { ); // give things a chance to fully start. - sleep(Duration::from_secs(5)); + sleep(Duration::from_secs(10)); let starting_relayer_balance: f64 = relayer_balance_sum(hpl_rly_metrics_port).unwrap(); From 4bafa7678ccb42b111c2cd14fe896c02a7321136 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Thu, 30 Nov 2023 00:14:48 +0000 Subject: [PATCH 08/18] chore: use a paranoid sleep in cosmos e2e because wasm init may be slow --- rust/utils/run-locally/src/cosmos/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 78c5e1d6bd..d60c65c9cd 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -462,7 +462,9 @@ fn run_locally() { ); // give things a chance to fully start. - sleep(Duration::from_secs(10)); + println!("sleeping for 100s"); + sleep(Duration::from_secs(100)); + println!("done sleeping for 100s"); let starting_relayer_balance: f64 = relayer_balance_sum(hpl_rly_metrics_port).unwrap(); From bbecadb0fc6942b3c415782d5ec435f5728db716 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:34:48 +0000 Subject: [PATCH 09/18] fix(cosmos): agent reorg period --- rust/utils/run-locally/src/cosmos/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index d60c65c9cd..938fcbf475 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -258,7 +258,7 @@ fn launch_cosmos_validator( .hyp_env("CHECKPOINTSYNCER_PATH", checkpoint_path.to_str().unwrap()) .hyp_env("CHECKPOINTSYNCER_TYPE", "localStorage") .hyp_env("ORIGINCHAINNAME", agent_config.name) - .hyp_env("REORGPERIOD", "100") + .hyp_env("REORGPERIOD", "1") .hyp_env("DB", validator_base_db.to_str().unwrap()) .hyp_env("METRICSPORT", agent_config.metrics_port.to_string()) .hyp_env("VALIDATOR_SIGNER_TYPE", agent_config.signer.typ) @@ -288,7 +288,7 @@ fn launch_cosmos_relayer( .env("CONFIG_FILES", agent_config_path.to_str().unwrap()) .env("RUST_BACKTRACE", "1") .hyp_env("RELAYCHAINS", relay_chains.join(",")) - .hyp_env("REORGPERIOD", "100") + .hyp_env("REORGPERIOD", "1") .hyp_env("DB", relayer_base.as_ref().to_str().unwrap()) .hyp_env("ALLOWLOCALCHECKPOINTSYNCERS", "true") .hyp_env("TRACING_LEVEL", if debug { "debug" } else { "info" }) From 67d22f82e823bb87840b576621f61b87ed35514f Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:36:48 +0000 Subject: [PATCH 10/18] chore(cosmos): re-enable e2e invariants --- rust/utils/run-locally/src/cosmos/mod.rs | 108 +++++++++++------------ 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 938fcbf475..573b29f1b0 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -566,63 +566,61 @@ fn relayer_balance_sum(metrics_port: u32) -> eyre::Result { } fn termination_invariants_met( - _relayer_metrics_port: u32, - _messages_expected: u32, - _starting_relayer_balance: f64, + relayer_metrics_port: u32, + messages_expected: u32, + starting_relayer_balance: f64, ) -> eyre::Result { + let gas_payments_scraped = fetch_metric( + &relayer_metrics_port.to_string(), + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payment"}, + )? + .iter() + .sum::(); + let expected_gas_payments = messages_expected; + if gas_payments_scraped != expected_gas_payments { + log!( + "Scraper has scraped {} gas payments, expected {}", + gas_payments_scraped, + expected_gas_payments + ); + return Ok(false); + } + + let delivered_messages_scraped = fetch_metric( + &relayer_metrics_port.to_string(), + "hyperlane_operations_processed_count", + &hashmap! {"phase" => "confirmed"}, + )? + .iter() + .sum::(); + if delivered_messages_scraped != messages_expected { + log!( + "Relayer confirmed {} submitted messages, expected {}", + delivered_messages_scraped, + messages_expected + ); + return Ok(false); + } + + let ending_relayer_balance: f64 = relayer_balance_sum(relayer_metrics_port).unwrap(); + // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: + // https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/c2288eb31734ba1f2f997e2c6ecb30176427bc2c/rust/utils/run-locally/src/cosmos/cli.rs#L55 + // What's stopping this is that the format returned by the `uosmo` balance query is a surprisingly low number (0.000003999999995184) + // but then maybe the gas_per_tx is just very low - how can we check that? (maybe by simulating said tx) + if starting_relayer_balance <= ending_relayer_balance { + // worth retrying this because metrics are polled every + // `METRICS_SCRAPE_INTERVAL` + log!( + "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", + starting_relayer_balance, + ending_relayer_balance + ); + return Ok(false); + } + + log!("Termination invariants have been meet"); Ok(true) - // TODO: uncomment once CI passes consistently on Ubuntu - // let gas_payments_scraped = fetch_metric( - // &relayer_metrics_port.to_string(), - // "hyperlane_contract_sync_stored_events", - // &hashmap! {"data_type" => "gas_payment"}, - // )? - // .iter() - // .sum::(); - // let expected_gas_payments = messages_expected; - // if gas_payments_scraped != expected_gas_payments { - // log!( - // "Scraper has scraped {} gas payments, expected {}", - // gas_payments_scraped, - // expected_gas_payments - // ); - // return Ok(false); - // } - - // let delivered_messages_scraped = fetch_metric( - // &relayer_metrics_port.to_string(), - // "hyperlane_operations_processed_count", - // &hashmap! {"phase" => "confirmed"}, - // )? - // .iter() - // .sum::(); - // if delivered_messages_scraped != messages_expected { - // log!( - // "Relayer confirmed {} submitted messages, expected {}", - // delivered_messages_scraped, - // messages_expected - // ); - // return Ok(false); - // } - - // let ending_relayer_balance: f64 = relayer_balance_sum(relayer_metrics_port).unwrap(); - // // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: - // // https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/c2288eb31734ba1f2f997e2c6ecb30176427bc2c/rust/utils/run-locally/src/cosmos/cli.rs#L55 - // // What's stopping this is that the format returned by the `uosmo` balance query is a surprisingly low number (0.000003999999995184) - // // but then maybe the gas_per_tx is just very low - how can we check that? (maybe by simulating said tx) - // if starting_relayer_balance <= ending_relayer_balance { - // // worth retrying this because metrics are polled every - // // `METRICS_SCRAPE_INTERVAL` - // log!( - // "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", - // starting_relayer_balance, - // ending_relayer_balance - // ); - // return Ok(false); - // } - - // log!("Termination invariants have been meet"); - // Ok(true) } #[cfg(test)] From dd0e1a331b6266c0ce8fdd7245064d38be9d412f Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Thu, 30 Nov 2023 18:15:47 +0000 Subject: [PATCH 11/18] fix: upgrade osmosis version for non-mac OSs --- rust/utils/run-locally/src/cosmos/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 573b29f1b0..6027611ddc 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -34,7 +34,7 @@ use self::deploy::deploy_cw_hyperlane; use self::source::{CLISource, CodeSource}; const OSMOSIS_CLI_GIT: &str = "https://github.com/osmosis-labs/osmosis"; -const OSMOSIS_CLI_VERSION: &str = "19.0.0"; +const OSMOSIS_CLI_VERSION: &str = "20.5.0"; const KEY_HPL_VALIDATOR: (&str,&str) = ("hpl-validator", "guard evolve region sentence danger sort despair eye deputy brave trim actor left recipe debate document upgrade sustain bus cage afford half demand pigeon"); const KEY_HPL_RELAYER: (&str,&str) = ("hpl-relayer", "moral item damp melt gloom vendor notice head assume balance doctor retire fashion trim find biology saddle undo switch fault cattle toast drip empty"); From b1ccae1a787ee01435505d0f44c0e065318dec27 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:57:22 +0000 Subject: [PATCH 12/18] feat: query evm balance, clean up --- rust/agents/relayer/src/relayer.rs | 86 ++++++++++--------- rust/agents/scraper/src/agent.rs | 32 +++---- rust/agents/validator/src/validator.rs | 15 ++-- .../hyperlane-cosmos/src/agent_metrics.rs | 54 ------------ .../hyperlane-cosmos/src/aggregation_ism.rs | 2 +- rust/chains/hyperlane-cosmos/src/lib.rs | 8 +- .../hyperlane-cosmos/src/providers/mod.rs | 23 ++--- .../hyperlane-ethereum/src/agent_metrics.rs | 12 --- rust/chains/hyperlane-ethereum/src/lib.rs | 3 - .../chains/hyperlane-ethereum/src/provider.rs | 45 ++++++++-- rust/chains/hyperlane-fuel/src/provider.rs | 14 +-- .../chains/hyperlane-sealevel/src/provider.rs | 14 +-- rust/hyperlane-base/src/agent.rs | 16 ++-- .../src/metrics/agent_metrics.rs | 10 +-- rust/hyperlane-base/src/settings/chains.rs | 50 ++++++----- rust/hyperlane-core/src/metrics/agent.rs | 4 +- rust/hyperlane-core/src/traits/provider.rs | 9 +- rust/utils/run-locally/src/cosmos/mod.rs | 22 +---- rust/utils/run-locally/src/invariants.rs | 14 +++ rust/utils/run-locally/src/main.rs | 4 +- rust/utils/run-locally/src/metrics.rs | 12 +++ 21 files changed, 211 insertions(+), 238 deletions(-) delete mode 100644 rust/chains/hyperlane-cosmos/src/agent_metrics.rs delete mode 100644 rust/chains/hyperlane-ethereum/src/agent_metrics.rs diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index af7df960ae..0256b77329 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -9,9 +9,11 @@ use derive_more::AsRef; use eyre::Result; use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, - metrics::{AgentMetrics, InstrumentedFallibleTask, Metrics}, - run_all, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, - SequencedDataContractSync, WatermarkContractSync, + metrics::{AgentMetrics, Metrics}, + run_all, + settings::ChainConf, + BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, + WatermarkContractSync, }; use hyperlane_core::{ metrics::agent::METRICS_SCRAPE_INTERVAL, HyperlaneDomain, HyperlaneMessage, @@ -51,7 +53,7 @@ struct ContextKey { #[derive(AsRef)] pub struct Relayer { origin_chains: HashSet, - destination_chains: HashSet, + destination_chains: HashMap, #[as_ref] core: HyperlaneAgentCore, message_syncs: HashMap>>, @@ -69,6 +71,8 @@ pub struct Relayer { transaction_gas_limit: Option, skip_transaction_gas_limit_for: HashSet, allow_local_checkpoint_syncers: bool, + core_metrics: Arc, + agent_metrics: Metrics, } impl Debug for Relayer { @@ -98,7 +102,7 @@ impl BaseAgent for Relayer { settings: Self::Settings, core_metrics: Arc, agent_metrics: Metrics, - ) -> Result<(Self, Vec>)> + ) -> Result where Self: Sized, { @@ -194,30 +198,10 @@ impl BaseAgent for Relayer { .collect(); let mut msg_ctxs = HashMap::new(); - let mut metrics_fetchers = vec![]; + let mut destination_chains = HashMap::new(); for destination in &settings.destination_chains { let destination_chain_setup = core.settings.chain_setup(destination).unwrap().clone(); - let agent_metrics_conf = destination_chain_setup - .agent_metrics_conf(Self::AGENT_NAME.to_string()) - .await?; - let agent_metrics_fetcher = destination_chain_setup - .build_agent_metrics_fetcher() - .await?; - let agent_metrics = AgentMetrics::new( - agent_metrics.clone(), - agent_metrics_conf, - agent_metrics_fetcher, - ); - - let fetcher_task = tokio::spawn(async move { - agent_metrics - .start_updating_on_interval(METRICS_SCRAPE_INTERVAL) - .await; - Ok(()) - }) - .instrument(info_span!("AgentMetricsFetcher")); - metrics_fetchers.push(fetcher_task); - + destination_chains.insert(destination.clone(), destination_chain_setup.clone()); let transaction_gas_limit: Option = if skip_transaction_gas_limit_for.contains(&destination.id()) { None @@ -257,7 +241,7 @@ impl BaseAgent for Relayer { let relayer = Self { dbs, origin_chains: settings.origin_chains, - destination_chains: settings.destination_chains, + destination_chains, msg_ctxs, core, message_syncs, @@ -269,28 +253,48 @@ impl BaseAgent for Relayer { transaction_gas_limit, skip_transaction_gas_limit_for, allow_local_checkpoint_syncers: settings.allow_local_checkpoint_syncers, + core_metrics, + agent_metrics, }; - Ok((relayer, metrics_fetchers)) + Ok(relayer) } #[allow(clippy::async_yields_async)] - async fn run( - self, - metrics_fetchers: Vec>, - ) -> Instrumented>> { - // The tasks vec is initialized with the metrics fetcher tasks, - // and is then extended with the rest of the tasks. - let mut tasks = metrics_fetchers; + async fn run(self) -> Instrumented>> { + let mut tasks = vec![]; // send channels by destination chain let mut send_channels = HashMap::with_capacity(self.destination_chains.len()); - for destination in &self.destination_chains { + for (dest_domain, dest_conf) in &self.destination_chains { let (send_channel, receive_channel) = mpsc::unbounded_channel::>(); - send_channels.insert(destination.id(), send_channel); + send_channels.insert(dest_domain.id(), send_channel); + + tasks.push(self.run_destination_submitter(dest_domain, receive_channel)); - tasks.push(self.run_destination_submitter(destination, receive_channel)); + let agent_metrics_conf = dest_conf + .agent_metrics_conf(Self::AGENT_NAME.to_string()) + .await + .unwrap(); + let agent_metrics_fetcher = dest_conf + .build_agent_metrics_fetcher(&self.core_metrics) + .await + .unwrap(); + let agent_metrics = AgentMetrics::new( + self.agent_metrics.clone(), + agent_metrics_conf, + agent_metrics_fetcher, + ); + + let fetcher_task = tokio::spawn(async move { + agent_metrics + .start_updating_on_interval(METRICS_SCRAPE_INTERVAL) + .await; + Ok(()) + }) + .instrument(info_span!("AgentMetricsFetcher")); + tasks.push(fetcher_task); } for origin in &self.origin_chains { @@ -364,11 +368,11 @@ impl Relayer { let metrics = MessageProcessorMetrics::new( &self.core.metrics, origin, - self.destination_chains.iter(), + self.destination_chains.keys(), ); let destination_ctxs = self .destination_chains - .iter() + .keys() .filter(|&destination| destination != origin) .map(|destination| { ( diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index 24549b3765..69b9d750ea 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -3,10 +3,8 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; use derive_more::AsRef; use hyperlane_base::{ - metrics::{InstrumentedFallibleTask, Metrics as AgentMetrics}, - run_all, - settings::IndexSettings, - BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, + metrics::Metrics as AgentMetrics, run_all, settings::IndexSettings, BaseAgent, + ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, }; use hyperlane_core::HyperlaneDomain; use tokio::task::JoinHandle; @@ -41,7 +39,7 @@ impl BaseAgent for Scraper { settings: Self::Settings, metrics: Arc, _agent_metrics: AgentMetrics, - ) -> eyre::Result<(Self, Vec>)> + ) -> eyre::Result where Self: Sized, { @@ -76,15 +74,12 @@ impl BaseAgent for Scraper { trace!(domain_count = scrapers.len(), "Created scrapers"); - Ok(( - Self { - core, - metrics, - contract_sync_metrics, - scrapers, - }, - Default::default(), - )) + Ok(Self { + core, + metrics, + contract_sync_metrics, + scrapers, + }) } /// Run the scraper @@ -92,13 +87,8 @@ impl BaseAgent for Scraper { /// * `metrics_fetchers` - A list of metrics fetchers to run. Currently this /// only comprise #[allow(clippy::async_yields_async)] - async fn run( - self, - metrics_fetchers: Vec>, - ) -> Instrumented>> { - // The tasks vec is initialized with the metrics fetcher tasks, - // and is then extended with the rest of the tasks. - let mut tasks = metrics_fetchers; + async fn run(self) -> Instrumented>> { + let mut tasks = Vec::with_capacity(self.scrapers.len()); for domain in self.scrapers.keys() { tasks.push(self.scrape(*domain).await); } diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index 2fd90b4edb..7633decbc2 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -10,7 +10,7 @@ use tracing::{error, info, info_span, instrument::Instrumented, warn, Instrument use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, - metrics::{InstrumentedFallibleTask, Metrics as AgentMetrics}, + metrics::Metrics as AgentMetrics, run_all, BaseAgent, CheckpointSyncer, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, }; @@ -56,7 +56,7 @@ impl BaseAgent for Validator { settings: Self::Settings, metrics: Arc, _agent_metrics: AgentMetrics, - ) -> Result<(Self, Vec>)> + ) -> Result where Self: Sized, { @@ -108,17 +108,12 @@ impl BaseAgent for Validator { checkpoint_syncer, }; - Ok((validator, Default::default())) + Ok(validator) } #[allow(clippy::async_yields_async)] - async fn run( - mut self, - metrics_fetchers: Vec>, - ) -> Instrumented>> { - // The tasks vec is initialized with the metrics fetcher tasks, - // and is then extended with the rest of the tasks. - let mut tasks = metrics_fetchers; + async fn run(mut self) -> Instrumented>> { + let mut tasks = vec![]; if let Some(signer_instance) = self.signer_instance.take() { tasks.push( diff --git a/rust/chains/hyperlane-cosmos/src/agent_metrics.rs b/rust/chains/hyperlane-cosmos/src/agent_metrics.rs deleted file mode 100644 index 30c15f98f3..0000000000 --- a/rust/chains/hyperlane-cosmos/src/agent_metrics.rs +++ /dev/null @@ -1,54 +0,0 @@ -use async_trait::async_trait; -use hyperlane_core::{ - metrics::agent::AgenMetricsFetcher, ChainResult, ContractLocator, HyperlaneChain, - HyperlaneDomain, HyperlaneProvider, U256, -}; - -use crate::{address::CosmosAddress, ConnectionConf, CosmosProvider}; - -/// Concrete struct for implementing the AgenMetricsFetcher and HyperlaneChain traits for Cosmos -#[derive(Debug)] -pub struct CosmosMetricsFetcher { - address: CosmosAddress, - provider: CosmosProvider, - domain: HyperlaneDomain, -} - -impl CosmosMetricsFetcher { - /// Instiante a new CosmosMetricsFetcher - pub fn new( - conf: ConnectionConf, - locator: ContractLocator, - address: CosmosAddress, - ) -> ChainResult { - let provider = CosmosProvider::new( - locator.domain.clone(), - conf.clone(), - Some(locator.clone()), - None, - )?; - - Ok(Self { - address, - provider, - domain: locator.domain.clone(), - }) - } -} - -impl HyperlaneChain for CosmosMetricsFetcher { - fn domain(&self) -> &HyperlaneDomain { - &self.domain - } - - fn provider(&self) -> Box { - Box::new(self.provider.clone()) - } -} - -#[async_trait] -impl AgenMetricsFetcher for CosmosMetricsFetcher { - async fn get_balance(&self) -> ChainResult { - self.provider.get_balance(self.address.address()).await - } -} diff --git a/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs b/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs index 49e5d60397..a17a4ba3af 100644 --- a/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/aggregation_ism.rs @@ -55,7 +55,7 @@ impl HyperlaneChain for CosmosAggregationIsm { } fn provider(&self) -> Box { - Box::new(self.provider.clone()) + self.provider.clone() } } diff --git a/rust/chains/hyperlane-cosmos/src/lib.rs b/rust/chains/hyperlane-cosmos/src/lib.rs index c61f1de261..82a4a0ece1 100644 --- a/rust/chains/hyperlane-cosmos/src/lib.rs +++ b/rust/chains/hyperlane-cosmos/src/lib.rs @@ -5,7 +5,6 @@ // TODO: Remove once we start filling things in #![allow(unused_variables)] -mod agent_metrics; mod aggregation_ism; mod error; mod interchain_gas; @@ -24,8 +23,7 @@ mod utils; mod validator_announce; pub use self::{ - agent_metrics::*, aggregation_ism::*, error::*, interchain_gas::*, - interchain_security_module::*, libs::*, mailbox::*, merkle_tree_hook::*, multisig_ism::*, - providers::*, routing_ism::*, signers::*, trait_builder::*, trait_builder::*, - validator_announce::*, validator_announce::*, + aggregation_ism::*, error::*, interchain_gas::*, interchain_security_module::*, libs::*, + mailbox::*, merkle_tree_hook::*, multisig_ism::*, providers::*, routing_ism::*, signers::*, + trait_builder::*, trait_builder::*, validator_announce::*, validator_announce::*, }; diff --git a/rust/chains/hyperlane-cosmos/src/providers/mod.rs b/rust/chains/hyperlane-cosmos/src/providers/mod.rs index 4bb960b536..66763509b5 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/mod.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/mod.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use hyperlane_core::{ - BlockInfo, ChainResult, ContractLocator, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, - TxnInfo, H256, U256, + metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, ContractLocator, HyperlaneChain, + HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, }; use tendermint_rpc::{client::CompatMode, HttpClient}; @@ -71,6 +71,17 @@ impl HyperlaneChain for CosmosProvider { } } +#[async_trait] +impl AgentMetricsFetcher for CosmosProvider { + async fn get_balance(&self, address: String) -> ChainResult { + Ok(self + .grpc_client + .get_balance(address, self.canonical_asset.clone()) + .await? + .into()) + } +} + #[async_trait] impl HyperlaneProvider for CosmosProvider { async fn get_block_by_hash(&self, _hash: &H256) -> ChainResult { @@ -85,12 +96,4 @@ impl HyperlaneProvider for CosmosProvider { // FIXME Ok(true) } - - async fn get_balance(&self, address: String) -> ChainResult { - Ok(self - .grpc_client - .get_balance(address, self.canonical_asset.clone()) - .await? - .into()) - } } diff --git a/rust/chains/hyperlane-ethereum/src/agent_metrics.rs b/rust/chains/hyperlane-ethereum/src/agent_metrics.rs deleted file mode 100644 index f1bd5065ba..0000000000 --- a/rust/chains/hyperlane-ethereum/src/agent_metrics.rs +++ /dev/null @@ -1,12 +0,0 @@ -use async_trait::async_trait; -use hyperlane_core::{metrics::agent::AgenMetricsFetcher, ChainResult, U256}; - -/// Concrete struct for implementing the AgenMetricsFetcher trait for Ethereum -pub struct EthereumMetricsFetcher {} - -#[async_trait] -impl AgenMetricsFetcher for EthereumMetricsFetcher { - async fn get_balance(&self) -> ChainResult { - Ok(0.into()) - } -} diff --git a/rust/chains/hyperlane-ethereum/src/lib.rs b/rust/chains/hyperlane-ethereum/src/lib.rs index 6f3d9b268b..2d42850bc4 100644 --- a/rust/chains/hyperlane-ethereum/src/lib.rs +++ b/rust/chains/hyperlane-ethereum/src/lib.rs @@ -74,11 +74,8 @@ mod signers; #[cfg(not(doctest))] mod singleton_signer; -mod agent_metrics; mod config; -pub use self::agent_metrics::*; - fn extract_fn_map(abi: &'static Lazy) -> HashMap, &'static str> { abi.functions() .map(|f| (f.selector().to_vec(), f.name.as_str())) diff --git a/rust/chains/hyperlane-ethereum/src/provider.rs b/rust/chains/hyperlane-ethereum/src/provider.rs index e23b19a779..620ae9aabc 100644 --- a/rust/chains/hyperlane-ethereum/src/provider.rs +++ b/rust/chains/hyperlane-ethereum/src/provider.rs @@ -6,7 +6,7 @@ use std::time::Duration; use async_trait::async_trait; use derive_new::new; use ethers::prelude::Middleware; -use hyperlane_core::{ethers_core_types, U256}; +use hyperlane_core::{ethers_core_types, metrics::agent::AgentMetricsFetcher, U256}; use tokio::time::sleep; use tracing::instrument; @@ -44,6 +44,22 @@ where } } +#[async_trait] +impl AgentMetricsFetcher for EthereumProvider +where + M: Middleware + 'static, +{ + #[instrument(err, skip(self))] + async fn get_balance(&self, address: String) -> ChainResult { + let balance = self + .provider + .get_balance(address, None) + .await + .map_err(ChainCommunicationError::from_other)?; + Ok(balance.into()) + } +} + #[async_trait] impl HyperlaneProvider for EthereumProvider where @@ -105,10 +121,6 @@ where .map_err(ChainCommunicationError::from_other)?; Ok(!code.is_empty()) } - - async fn get_balance(&self, _address: String) -> ChainResult { - todo!() - } } impl EthereumProvider @@ -149,6 +161,29 @@ impl BuildableWithProvider for HyperlaneProviderBuilder { } } +/// Builder for the Agent Metrics Fetcher. +// TODO: Remove this when trait upcasting is stabilized and Box can be used +// as Box +// Tracking issue: +// https://github.com/rust-lang/rust/issues/65991 +pub struct AgentMetricsFetcherBuilder {} + +#[async_trait] +impl BuildableWithProvider for AgentMetricsFetcherBuilder { + type Output = Box; + + async fn build_with_provider( + &self, + provider: M, + locator: &ContractLocator, + ) -> Self::Output { + Box::new(EthereumProvider::new( + Arc::new(provider), + locator.domain.clone(), + )) + } +} + /// Call a get function that returns a Result> and retry if the inner /// option is None. This can happen because the provider has not discovered the /// object we are looking for yet. diff --git a/rust/chains/hyperlane-fuel/src/provider.rs b/rust/chains/hyperlane-fuel/src/provider.rs index 8048076e04..54c38e850c 100644 --- a/rust/chains/hyperlane-fuel/src/provider.rs +++ b/rust/chains/hyperlane-fuel/src/provider.rs @@ -1,7 +1,8 @@ use async_trait::async_trait; use hyperlane_core::{ - BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, + metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, + HyperlaneProvider, TxnInfo, H256, U256, }; /// A wrapper around a fuel provider to get generic blockchain information. @@ -18,6 +19,13 @@ impl HyperlaneChain for FuelProvider { } } +#[async_trait] +impl AgentMetricsFetcher for FuelProvider { + async fn get_balance(&self, address: String) -> ChainResult { + todo!() + } +} + #[async_trait] impl HyperlaneProvider for FuelProvider { async fn get_block_by_hash(&self, hash: &H256) -> ChainResult { @@ -31,8 +39,4 @@ impl HyperlaneProvider for FuelProvider { async fn is_contract(&self, address: &H256) -> ChainResult { todo!() } - - async fn get_balance(&self, address: String) -> ChainResult { - todo!() - } } diff --git a/rust/chains/hyperlane-sealevel/src/provider.rs b/rust/chains/hyperlane-sealevel/src/provider.rs index 3f7449aef2..fdd20795a8 100644 --- a/rust/chains/hyperlane-sealevel/src/provider.rs +++ b/rust/chains/hyperlane-sealevel/src/provider.rs @@ -1,7 +1,8 @@ use async_trait::async_trait; use hyperlane_core::{ - BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, + metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, + HyperlaneProvider, TxnInfo, H256, U256, }; /// A wrapper around a Sealevel provider to get generic blockchain information. @@ -29,6 +30,13 @@ impl HyperlaneChain for SealevelProvider { } } +#[async_trait] +impl AgentMetricsFetcher for SealevelProvider { + async fn get_balance(&self, _address: String) -> ChainResult { + todo!() // FIXME + } +} + #[async_trait] impl HyperlaneProvider for SealevelProvider { async fn get_block_by_hash(&self, _hash: &H256) -> ChainResult { @@ -43,8 +51,4 @@ impl HyperlaneProvider for SealevelProvider { // FIXME Ok(true) } - - async fn get_balance(&self, _address: String) -> ChainResult { - todo!() - } } diff --git a/rust/hyperlane-base/src/agent.rs b/rust/hyperlane-base/src/agent.rs index e638108dcf..40fa639051 100644 --- a/rust/hyperlane-base/src/agent.rs +++ b/rust/hyperlane-base/src/agent.rs @@ -8,9 +8,7 @@ use tokio::task::JoinHandle; use tracing::{debug_span, instrument::Instrumented, Instrument}; use crate::{ - metrics::{ - create_agent_metrics, CoreMetrics, InstrumentedFallibleTask, Metrics as AgentMetrics, - }, + metrics::{create_agent_metrics, CoreMetrics, Metrics as AgentMetrics}, settings::Settings, }; @@ -45,16 +43,13 @@ pub trait BaseAgent: Send + Sync + Debug { settings: Self::Settings, metrics: Arc, agent_metrics: AgentMetrics, - ) -> Result<(Self, Vec>)> + ) -> Result where Self: Sized; /// Start running this agent. #[allow(clippy::async_yields_async)] - async fn run( - self, - metrics_fetchers: Vec>, - ) -> Instrumented>>; + async fn run(self) -> Instrumented>>; } /// Call this from `main` to fully initialize and run the agent for its entire @@ -81,11 +76,10 @@ pub async fn agent_main() -> Result<()> { let metrics = settings.as_ref().metrics(A::AGENT_NAME)?; core_settings.tracing.start_tracing(&metrics)?; let agent_metrics = create_agent_metrics(&metrics)?; - let (agent, metrics_fetchers) = - A::from_settings(settings, metrics.clone(), agent_metrics).await?; + let agent = A::from_settings(settings, metrics.clone(), agent_metrics).await?; metrics.run_http_server(); - agent.run(metrics_fetchers).await.await? + agent.run().await.await? } /// Utility to run multiple tasks and shutdown if any one task ends. diff --git a/rust/hyperlane-base/src/metrics/agent_metrics.rs b/rust/hyperlane-base/src/metrics/agent_metrics.rs index 267197e9b4..868de5b619 100644 --- a/rust/hyperlane-base/src/metrics/agent_metrics.rs +++ b/rust/hyperlane-base/src/metrics/agent_metrics.rs @@ -4,11 +4,10 @@ use derive_builder::Builder; use derive_new::new; use eyre::Result; use hyperlane_core::metrics::agent::u256_as_scaled_f64; -use hyperlane_core::{metrics::agent::AgenMetricsFetcher, HyperlaneDomain}; +use hyperlane_core::{metrics::agent::AgentMetricsFetcher, HyperlaneDomain}; use maplit::hashmap; use prometheus::GaugeVec; use tokio::time::MissedTickBehavior; -use tracing::instrument::Instrumented; use tracing::{trace, warn}; use crate::CoreMetrics; @@ -26,9 +25,6 @@ pub const WALLET_BALANCE_LABELS: &[&str] = &[ pub const WALLET_BALANCE_HELP: &str = "Current native token balance for the wallet addresses in the `wallets` set"; -/// Instrumented fallible task alias -pub type InstrumentedFallibleTask = Instrumented>>; - /// Agent-specific metrics #[derive(Clone, Builder)] pub struct Metrics { @@ -76,7 +72,7 @@ pub struct AgentMetricsConf { pub struct AgentMetrics { metrics: Metrics, conf: AgentMetricsConf, - fetcher: Box, + fetcher: Box, } impl AgentMetrics { @@ -90,7 +86,7 @@ impl AgentMetrics { }; let chain = self.conf.domain.name(); - match self.fetcher.get_balance().await { + match self.fetcher.get_balance(wallet_addr.clone()).await { Ok(balance) => { // Okay, so the native type is not a token, but whatever, close enough. // Note: This is ETH for many chains, but not all so that is why we use `N` and `Native` diff --git a/rust/hyperlane-base/src/settings/chains.rs b/rust/hyperlane-base/src/settings/chains.rs index a40abeb531..99add295c2 100644 --- a/rust/hyperlane-base/src/settings/chains.rs +++ b/rust/hyperlane-base/src/settings/chains.rs @@ -1,14 +1,14 @@ use ethers::prelude::Selector; -use h_cosmos::{address::CosmosAddress, CosmosMetricsFetcher}; +use h_cosmos::CosmosProvider; use std::collections::HashMap; use eyre::{eyre, Context, Result}; use ethers_prometheus::middleware::{ChainInfo, ContractInfo, PrometheusMiddlewareConf}; use hyperlane_core::{ - metrics::agent::AgenMetricsFetcher, AggregationIsm, CcipReadIsm, ContractLocator, HyperlaneAbi, - HyperlaneDomain, HyperlaneDomainProtocol, HyperlaneMessage, HyperlaneProvider, IndexMode, - InterchainGasPaymaster, InterchainGasPayment, InterchainSecurityModule, Mailbox, + metrics::agent::AgentMetricsFetcher, AggregationIsm, CcipReadIsm, ContractLocator, + HyperlaneAbi, HyperlaneDomain, HyperlaneDomainProtocol, HyperlaneMessage, HyperlaneProvider, + IndexMode, InterchainGasPaymaster, InterchainGasPayment, InterchainSecurityModule, Mailbox, MerkleTreeHook, MerkleTreeInsertion, MultisigIsm, RoutingIsm, SequenceIndexer, ValidatorAnnounce, H256, }; @@ -22,10 +22,12 @@ use hyperlane_sealevel as h_sealevel; use crate::{ metrics::AgentMetricsConf, - settings::signers::{BuildableWithSignerConf, ChainSigner, SignerConf}, + settings::signers::{BuildableWithSignerConf, SignerConf}, CoreMetrics, }; +use super::ChainSigner; + /// A chain setup is a domain ID, an address on that chain (where the mailbox is /// deployed) and details for connecting to the chain API. #[derive(Clone, Debug)] @@ -595,30 +597,34 @@ impl ChainConf { } /// Try to convert the chain setting into a trait object for fetching agent metrics - pub async fn build_agent_metrics_fetcher(&self) -> Result> { - let ctx = "Building Agent Metrics Fetcher"; - + pub async fn build_agent_metrics_fetcher( + &self, + metrics: &CoreMetrics, + ) -> Result> { match &self.connection { - ChainConnectionConf::Ethereum(_conf) => { - Ok(Box::new(h_eth::EthereumMetricsFetcher {}) as Box) + ChainConnectionConf::Ethereum(conf) => { + let locator = self.locator(H256::zero()); + let provider = self + .build_ethereum( + conf, + &locator, + metrics, + h_eth::AgentMetricsFetcherBuilder {}, + ) + .await?; + Ok(provider) } ChainConnectionConf::Fuel(_) => todo!(), ChainConnectionConf::Sealevel(_) => todo!(), ChainConnectionConf::Cosmos(conf) => { - let signer = self - .cosmos_signer() - .await - .context(ctx)? - .ok_or(eyre!("No signer set")) - .context(ctx)?; - let address = CosmosAddress::from_pubkey(signer.public_key, &conf.get_prefix()) - .context(ctx)?; - let metrics_fetcher = CosmosMetricsFetcher::new( + let locator = self.locator(H256::zero()); + let provider = CosmosProvider::new( + locator.domain.clone(), conf.clone(), - self.locator(self.addresses.mailbox), - address, + Some(locator.clone()), + None, )?; - Ok(Box::new(metrics_fetcher) as Box) + Ok(Box::new(provider) as Box) } } } diff --git a/rust/hyperlane-core/src/metrics/agent.rs b/rust/hyperlane-core/src/metrics/agent.rs index 6134e2b8e5..e7e77c9c6d 100644 --- a/rust/hyperlane-core/src/metrics/agent.rs +++ b/rust/hyperlane-core/src/metrics/agent.rs @@ -11,9 +11,9 @@ pub const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); /// Trait to be implemented by all chain-specific agent implementations, /// to support gathering agent metrics. #[async_trait] -pub trait AgenMetricsFetcher: Send + Sync { +pub trait AgentMetricsFetcher: Send + Sync { /// Fetch the balance of the wallet address associated with the chain provider. - async fn get_balance(&self) -> ChainResult; + async fn get_balance(&self, address: String) -> ChainResult; } /// Convert a u256 scaled integer value into the corresponding f64 value. diff --git a/rust/hyperlane-core/src/traits/provider.rs b/rust/hyperlane-core/src/traits/provider.rs index ec8cae7052..4d0d127467 100644 --- a/rust/hyperlane-core/src/traits/provider.rs +++ b/rust/hyperlane-core/src/traits/provider.rs @@ -4,7 +4,9 @@ use async_trait::async_trait; use auto_impl::auto_impl; use thiserror::Error; -use crate::{BlockInfo, ChainResult, HyperlaneChain, TxnInfo, H256, U256}; +use crate::{ + metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, HyperlaneChain, TxnInfo, H256, +}; /// Interface for a provider. Allows abstraction over different provider types /// for different chains. @@ -15,7 +17,7 @@ use crate::{BlockInfo, ChainResult, HyperlaneChain, TxnInfo, H256, U256}; /// the context of a contract. #[async_trait] #[auto_impl(&, Box, Arc)] -pub trait HyperlaneProvider: HyperlaneChain + Send + Sync + Debug { +pub trait HyperlaneProvider: HyperlaneChain + AgentMetricsFetcher + Send + Sync + Debug { /// Get block info for a given block hash async fn get_block_by_hash(&self, hash: &H256) -> ChainResult; @@ -24,9 +26,6 @@ pub trait HyperlaneProvider: HyperlaneChain + Send + Sync + Debug { /// Returns whether a contract exists at the provided address async fn is_contract(&self, address: &H256) -> ChainResult; - - /// Returns the native currency balance of the given address - async fn get_balance(&self, address: String) -> ChainResult; } /// Errors when querying for provider information. diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 6027611ddc..1bf109e15f 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -25,6 +25,7 @@ use utils::*; use crate::cosmos::link::link_networks; use crate::logging::log; +use crate::metrics::agent_balance_sum; use crate::program::Program; use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle}; use crate::{fetch_metric, AGENT_BIN_PATH}; @@ -258,7 +259,6 @@ fn launch_cosmos_validator( .hyp_env("CHECKPOINTSYNCER_PATH", checkpoint_path.to_str().unwrap()) .hyp_env("CHECKPOINTSYNCER_TYPE", "localStorage") .hyp_env("ORIGINCHAINNAME", agent_config.name) - .hyp_env("REORGPERIOD", "1") .hyp_env("DB", validator_base_db.to_str().unwrap()) .hyp_env("METRICSPORT", agent_config.metrics_port.to_string()) .hyp_env("VALIDATOR_SIGNER_TYPE", agent_config.signer.typ) @@ -288,7 +288,6 @@ fn launch_cosmos_relayer( .env("CONFIG_FILES", agent_config_path.to_str().unwrap()) .env("RUST_BACKTRACE", "1") .hyp_env("RELAYCHAINS", relay_chains.join(",")) - .hyp_env("REORGPERIOD", "1") .hyp_env("DB", relayer_base.as_ref().to_str().unwrap()) .hyp_env("ALLOWLOCALCHECKPOINTSYNCERS", "true") .hyp_env("TRACING_LEVEL", if debug { "debug" } else { "info" }) @@ -462,11 +461,9 @@ fn run_locally() { ); // give things a chance to fully start. - println!("sleeping for 100s"); - sleep(Duration::from_secs(100)); - println!("done sleeping for 100s"); + sleep(Duration::from_secs(10)); - let starting_relayer_balance: f64 = relayer_balance_sum(hpl_rly_metrics_port).unwrap(); + let starting_relayer_balance: f64 = agent_balance_sum(hpl_rly_metrics_port).unwrap(); // dispatch messages let mut dispatched_messages = 0; @@ -554,17 +551,6 @@ fn run_locally() { } } -fn relayer_balance_sum(metrics_port: u32) -> eyre::Result { - let balance = fetch_metric( - &metrics_port.to_string(), - "hyperlane_wallet_balance", - &hashmap! {}, - )? - .iter() - .sum(); - Ok(balance) -} - fn termination_invariants_met( relayer_metrics_port: u32, messages_expected: u32, @@ -603,7 +589,7 @@ fn termination_invariants_met( return Ok(false); } - let ending_relayer_balance: f64 = relayer_balance_sum(relayer_metrics_port).unwrap(); + let ending_relayer_balance: f64 = agent_balance_sum(relayer_metrics_port).unwrap(); // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: // https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/c2288eb31734ba1f2f997e2c6ecb30176427bc2c/rust/utils/run-locally/src/cosmos/cli.rs#L55 // What's stopping this is that the format returned by the `uosmo` balance query is a surprisingly low number (0.000003999999995184) diff --git a/rust/utils/run-locally/src/invariants.rs b/rust/utils/run-locally/src/invariants.rs index f1fb725959..4e76c9473c 100644 --- a/rust/utils/run-locally/src/invariants.rs +++ b/rust/utils/run-locally/src/invariants.rs @@ -1,6 +1,7 @@ // use std::path::Path; use crate::config::Config; +use crate::metrics::agent_balance_sum; use maplit::hashmap; use crate::logging::log; @@ -15,6 +16,7 @@ pub const SOL_MESSAGES_EXPECTED: u32 = 0; /// number of messages have been sent. pub fn termination_invariants_met( config: &Config, + starting_relayer_balance: f64, // solana_cli_tools_path: &Path, // solana_config_path: &Path, ) -> eyre::Result { @@ -129,6 +131,18 @@ pub fn termination_invariants_met( return Ok(false); } + let ending_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); + if starting_relayer_balance <= ending_relayer_balance { + // worth retrying this because metrics are polled every + // `METRICS_SCRAPE_INTERVAL` + log!( + "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", + starting_relayer_balance, + ending_relayer_balance + ); + return Ok(false); + } + log!("Termination invariants have been meet"); Ok(true) } diff --git a/rust/utils/run-locally/src/main.rs b/rust/utils/run-locally/src/main.rs index 0910d14710..52d56ed5d4 100644 --- a/rust/utils/run-locally/src/main.rs +++ b/rust/utils/run-locally/src/main.rs @@ -30,6 +30,7 @@ use crate::{ config::Config, ethereum::start_anvil, invariants::termination_invariants_met, + metrics::agent_balance_sum, solana::*, utils::{concat_path, make_static, stop_child, AgentHandles, ArbitraryData, TaskHandle}, }; @@ -390,10 +391,11 @@ fn main() -> ExitCode { // give things a chance to fully start. sleep(Duration::from_secs(10)); let mut failure_occurred = false; + let starting_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); while !SHUTDOWN.load(Ordering::Relaxed) { if config.ci_mode { // for CI we have to look for the end condition. - if termination_invariants_met(&config) + if termination_invariants_met(&config, starting_relayer_balance) // if termination_invariants_met(&config, &solana_path, &solana_config_path) .unwrap_or(false) { diff --git a/rust/utils/run-locally/src/metrics.rs b/rust/utils/run-locally/src/metrics.rs index 9aff78ea3a..aad0f626d9 100644 --- a/rust/utils/run-locally/src/metrics.rs +++ b/rust/utils/run-locally/src/metrics.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, error::Error as StdError, str::FromStr}; use eyre::{eyre, ErrReport, Result}; +use maplit::hashmap; /// Fetch a prometheus format metric, filtering by labels. pub fn fetch_metric(port: &str, metric: &str, labels: &HashMap<&str, &str>) -> Result> @@ -26,3 +27,14 @@ where }) .collect() } + +pub fn agent_balance_sum(metrics_port: u32) -> eyre::Result { + let balance = fetch_metric( + &metrics_port.to_string(), + "hyperlane_wallet_balance", + &hashmap! {}, + )? + .iter() + .sum(); + Ok(balance) +} From cc8578a30de2cfee23890c83782dd8a3eebd515d Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:34:52 +0000 Subject: [PATCH 13/18] fix: evm balance metrics querying --- rust/chains/hyperlane-ethereum/src/provider.rs | 6 +++++- rust/hyperlane-base/src/settings/signers.rs | 3 ++- rust/utils/run-locally/src/cosmos/types.rs | 2 -- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/rust/chains/hyperlane-ethereum/src/provider.rs b/rust/chains/hyperlane-ethereum/src/provider.rs index 620ae9aabc..c73c1d8da5 100644 --- a/rust/chains/hyperlane-ethereum/src/provider.rs +++ b/rust/chains/hyperlane-ethereum/src/provider.rs @@ -6,6 +6,7 @@ use std::time::Duration; use async_trait::async_trait; use derive_new::new; use ethers::prelude::Middleware; +use ethers_core::types::Address; use hyperlane_core::{ethers_core_types, metrics::agent::AgentMetricsFetcher, U256}; use tokio::time::sleep; use tracing::instrument; @@ -51,9 +52,12 @@ where { #[instrument(err, skip(self))] async fn get_balance(&self, address: String) -> ChainResult { + // Can't use the address directly as a string, because ethers interprets it + // as an ENS name rather than an address. + let addr: Address = address.parse()?; let balance = self .provider - .get_balance(address, None) + .get_balance(addr, None) .await .map_err(ChainCommunicationError::from_other)?; Ok(balance.into()) diff --git a/rust/hyperlane-base/src/settings/signers.rs b/rust/hyperlane-base/src/settings/signers.rs index 2979488544..c48de65f73 100644 --- a/rust/hyperlane-base/src/settings/signers.rs +++ b/rust/hyperlane-base/src/settings/signers.rs @@ -3,6 +3,7 @@ use std::time::Duration; use async_trait::async_trait; use ed25519_dalek::SecretKey; use ethers::prelude::{AwsSigner, LocalWallet}; +use ethers::utils::hex::ToHex; use eyre::{bail, Context, Report}; use hyperlane_core::H256; use hyperlane_sealevel::Keypair; @@ -96,7 +97,7 @@ impl BuildableWithSignerConf for hyperlane_ethereum::Signers { impl ChainSigner for hyperlane_ethereum::Signers { fn address_string(&self) -> String { - ethers::abi::AbiEncode::encode_hex(ethers::signers::Signer::address(self)) + ethers::signers::Signer::address(self).encode_hex() } } diff --git a/rust/utils/run-locally/src/cosmos/types.rs b/rust/utils/run-locally/src/cosmos/types.rs index 138cd3522d..f50986b600 100644 --- a/rust/utils/run-locally/src/cosmos/types.rs +++ b/rust/utils/run-locally/src/cosmos/types.rs @@ -119,7 +119,6 @@ pub struct AgentConfig { pub validator_announce: String, pub merkle_tree_hook: String, pub protocol: String, - pub finality_blocks: u32, pub chain_id: String, pub rpc_urls: Vec, pub grpc_url: String, @@ -151,7 +150,6 @@ impl AgentConfig { validator_announce: to_hex_addr(&network.deployments.va), merkle_tree_hook: to_hex_addr(&network.deployments.hook_merkle), protocol: "cosmos".to_string(), - finality_blocks: 1, chain_id: format!("cosmos-test-{}", network.domain), rpc_urls: vec![AgentUrl { http: format!( From e160a20d9157a33c79e8852d870c7e00724197ac Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:35:54 +0000 Subject: [PATCH 14/18] fix: seaorm connection timeout hiccups --- rust/agents/scraper/migration/bin/common.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rust/agents/scraper/migration/bin/common.rs b/rust/agents/scraper/migration/bin/common.rs index 096f7628c1..df7173fc0e 100644 --- a/rust/agents/scraper/migration/bin/common.rs +++ b/rust/agents/scraper/migration/bin/common.rs @@ -1,9 +1,11 @@ -use std::env; +use std::{env, time::Duration}; use migration::sea_orm::{Database, DatabaseConnection}; pub use migration::{DbErr, Migrator, MigratorTrait as _}; +use sea_orm::ConnectOptions; const LOCAL_DATABASE_URL: &str = "postgresql://postgres:47221c18c610@localhost:5432/postgres"; +const CONNECT_TIMEOUT: u64 = 20; pub fn url() -> String { env::var("DATABASE_URL").unwrap_or_else(|_| LOCAL_DATABASE_URL.into()) @@ -16,6 +18,8 @@ pub async fn init() -> Result { .init(); let url = url(); + let mut options: ConnectOptions = url.clone().into(); + options.connect_timeout(Duration::from_secs(CONNECT_TIMEOUT)); println!("Connecting to {url}"); - Database::connect(url).await + Database::connect(options).await } From c555e6af2dd12230dc404f5a69f1360029d65b49 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:44:25 +0000 Subject: [PATCH 15/18] cleanup --- rust/agents/relayer/src/relayer.rs | 6 ++---- rust/agents/scraper/src/agent.rs | 4 ---- rust/agents/validator/src/validator.rs | 6 ++---- rust/hyperlane-base/src/metrics/agent_metrics.rs | 4 ++-- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index 0256b77329..459661c7bf 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -238,7 +238,7 @@ impl BaseAgent for Relayer { } } - let relayer = Self { + Ok(Self { dbs, origin_chains: settings.origin_chains, destination_chains, @@ -255,9 +255,7 @@ impl BaseAgent for Relayer { allow_local_checkpoint_syncers: settings.allow_local_checkpoint_syncers, core_metrics, agent_metrics, - }; - - Ok(relayer) + }) } #[allow(clippy::async_yields_async)] diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index 69b9d750ea..7178ee7706 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -82,10 +82,6 @@ impl BaseAgent for Scraper { }) } - /// Run the scraper - /// - /// * `metrics_fetchers` - A list of metrics fetchers to run. Currently this - /// only comprise #[allow(clippy::async_yields_async)] async fn run(self) -> Instrumented>> { let mut tasks = Vec::with_capacity(self.scrapers.len()); diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index 7633decbc2..1a27a31968 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -93,7 +93,7 @@ impl BaseAgent for Validator { .await? .into(); - let validator = Self { + Ok(Self { origin_chain: settings.origin_chain, core, db: msg_db, @@ -106,9 +106,7 @@ impl BaseAgent for Validator { reorg_period: settings.reorg_period, interval: settings.interval, checkpoint_syncer, - }; - - Ok(validator) + }) } #[allow(clippy::async_yields_async)] diff --git a/rust/hyperlane-base/src/metrics/agent_metrics.rs b/rust/hyperlane-base/src/metrics/agent_metrics.rs index 868de5b619..87d10993f9 100644 --- a/rust/hyperlane-base/src/metrics/agent_metrics.rs +++ b/rust/hyperlane-base/src/metrics/agent_metrics.rs @@ -28,8 +28,8 @@ pub const WALLET_BALANCE_HELP: &str = /// Agent-specific metrics #[derive(Clone, Builder)] pub struct Metrics { - /// Current balance of eth and other tokens in the `tokens` map for the - /// wallet addresses in the `wallets` set. + /// Current balance of native tokens for the + /// wallet address. /// - `chain`: the chain name (or chain ID if the name is unknown) of the /// chain the tx occurred on. /// - `wallet_address`: Address of the wallet holding the funds. From fb614f86281b0622243ab1c4110b0b83199993f0 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 4 Dec 2023 12:30:42 +0000 Subject: [PATCH 16/18] outdated comment --- rust/chains/hyperlane-cosmos/src/providers/grpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 77b5eabc2a..12825cb42f 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -223,7 +223,7 @@ impl WasmGrpcProvider { Ok(gas_estimate) } - /// Estimates gas for a transaction containing `msgs`. + /// Fetches balance for a given `address` and `denom` pub async fn get_balance(&self, address: String, denom: String) -> ChainResult { let mut client = QueryBalanceClient::new(self.channel.clone()); From 5de6a05bede227877fdea42095a23d39fda36b2a Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 4 Dec 2023 18:06:38 +0000 Subject: [PATCH 17/18] fix: review comments --- rust/agents/relayer/src/relayer.rs | 15 ++--- rust/agents/scraper/src/agent.rs | 4 +- rust/agents/validator/src/validator.rs | 2 +- .../hyperlane-cosmos/src/libs/address.rs | 5 ++ rust/chains/hyperlane-cosmos/src/mailbox.rs | 2 +- .../hyperlane-cosmos/src/merkle_tree_hook.rs | 6 +- .../hyperlane-cosmos/src/providers/mod.rs | 27 ++++----- .../chains/hyperlane-ethereum/src/provider.rs | 59 +++++-------------- rust/chains/hyperlane-fuel/src/provider.rs | 14 ++--- .../chains/hyperlane-sealevel/src/provider.rs | 14 ++--- rust/hyperlane-base/src/agent.rs | 2 +- .../src/metrics/agent_metrics.rs | 21 +++---- rust/hyperlane-base/src/settings/chains.rs | 50 ++++------------ rust/hyperlane-core/src/metrics/agent.rs | 12 +--- rust/hyperlane-core/src/traits/provider.rs | 9 +-- rust/utils/run-locally/src/cosmos/mod.rs | 4 +- rust/utils/run-locally/src/invariants.rs | 3 +- 17 files changed, 89 insertions(+), 160 deletions(-) diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index 459661c7bf..ba4ac30cdb 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -9,7 +9,7 @@ use derive_more::AsRef; use eyre::Result; use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, - metrics::{AgentMetrics, Metrics}, + metrics::{AgentMetrics, AgentMetricsUpdater}, run_all, settings::ChainConf, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, @@ -72,7 +72,7 @@ pub struct Relayer { skip_transaction_gas_limit_for: HashSet, allow_local_checkpoint_syncers: bool, core_metrics: Arc, - agent_metrics: Metrics, + agent_metrics: AgentMetrics, } impl Debug for Relayer { @@ -101,7 +101,7 @@ impl BaseAgent for Relayer { async fn from_settings( settings: Self::Settings, core_metrics: Arc, - agent_metrics: Metrics, + agent_metrics: AgentMetrics, ) -> Result where Self: Sized, @@ -275,11 +275,8 @@ impl BaseAgent for Relayer { .agent_metrics_conf(Self::AGENT_NAME.to_string()) .await .unwrap(); - let agent_metrics_fetcher = dest_conf - .build_agent_metrics_fetcher(&self.core_metrics) - .await - .unwrap(); - let agent_metrics = AgentMetrics::new( + let agent_metrics_fetcher = dest_conf.build_provider(&self.core_metrics).await.unwrap(); + let agent_metrics = AgentMetricsUpdater::new( self.agent_metrics.clone(), agent_metrics_conf, agent_metrics_fetcher, @@ -291,7 +288,7 @@ impl BaseAgent for Relayer { .await; Ok(()) }) - .instrument(info_span!("AgentMetricsFetcher")); + .instrument(info_span!("AgentMetrics")); tasks.push(fetcher_task); } diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index 7178ee7706..9941c6a806 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -3,8 +3,8 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; use derive_more::AsRef; use hyperlane_base::{ - metrics::Metrics as AgentMetrics, run_all, settings::IndexSettings, BaseAgent, - ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, + metrics::AgentMetrics, run_all, settings::IndexSettings, BaseAgent, ContractSyncMetrics, + CoreMetrics, HyperlaneAgentCore, }; use hyperlane_core::HyperlaneDomain; use tokio::task::JoinHandle; diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index 1a27a31968..42474030ed 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -10,7 +10,7 @@ use tracing::{error, info, info_span, instrument::Instrumented, warn, Instrument use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, - metrics::Metrics as AgentMetrics, + metrics::AgentMetrics, run_all, BaseAgent, CheckpointSyncer, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, SequencedDataContractSync, }; diff --git a/rust/chains/hyperlane-cosmos/src/libs/address.rs b/rust/chains/hyperlane-cosmos/src/libs/address.rs index 7cfdb83caf..507bd24172 100644 --- a/rust/chains/hyperlane-cosmos/src/libs/address.rs +++ b/rust/chains/hyperlane-cosmos/src/libs/address.rs @@ -132,6 +132,11 @@ pub mod test { addr.address(), "neutron1kknekjxg0ear00dky5ykzs8wwp2gz62z9s6aaj" ); + // TODO: watch out for this edge case. This check will fail unless + // the first 12 bytes are removed from the digest. + // let digest = addr.digest(); + // let addr2 = CosmosAddress::from_h256(digest, prefix).expect("Cosmos address creation failed"); + // assert_eq!(addr.address(), addr2.address()); } #[test] diff --git a/rust/chains/hyperlane-cosmos/src/mailbox.rs b/rust/chains/hyperlane-cosmos/src/mailbox.rs index 2249546b77..4aafd29c87 100644 --- a/rust/chains/hyperlane-cosmos/src/mailbox.rs +++ b/rust/chains/hyperlane-cosmos/src/mailbox.rs @@ -96,7 +96,7 @@ impl Debug for CosmosMailbox { impl Mailbox for CosmosMailbox { #[instrument(level = "debug", err, ret, skip(self))] async fn count(&self, lag: Option) -> ChainResult { - let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; + let block_height = get_block_height_for_lag(self.provider.grpc(), lag).await?; self.nonce_at_block(block_height).await } diff --git a/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs b/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs index 40a0a2ab55..3de0821dff 100644 --- a/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs +++ b/rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs @@ -83,7 +83,7 @@ impl MerkleTreeHook for CosmosMerkleTreeHook { tree: general::EmptyStruct {}, }; - let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; + let block_height = get_block_height_for_lag(self.provider.grpc(), lag).await?; let data = self .provider @@ -117,7 +117,7 @@ impl MerkleTreeHook for CosmosMerkleTreeHook { count: general::EmptyStruct {}, }; - let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; + let block_height = get_block_height_for_lag(self.provider.grpc(), lag).await?; self.count_at_block(block_height).await } @@ -128,7 +128,7 @@ impl MerkleTreeHook for CosmosMerkleTreeHook { check_point: general::EmptyStruct {}, }; - let block_height = get_block_height_for_lag(&self.provider.grpc(), lag).await?; + let block_height = get_block_height_for_lag(self.provider.grpc(), lag).await?; let data = self .provider diff --git a/rust/chains/hyperlane-cosmos/src/providers/mod.rs b/rust/chains/hyperlane-cosmos/src/providers/mod.rs index 66763509b5..1ac97f1e75 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/mod.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/mod.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use hyperlane_core::{ - metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, ContractLocator, HyperlaneChain, - HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, + BlockInfo, ChainResult, ContractLocator, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, + TxnInfo, H256, U256, }; use tendermint_rpc::{client::CompatMode, HttpClient}; @@ -51,8 +51,8 @@ impl CosmosProvider { } /// Get a grpc client - pub fn grpc(&self) -> WasmGrpcProvider { - self.grpc_client.clone() + pub fn grpc(&self) -> &WasmGrpcProvider { + &self.grpc_client } /// Get an rpc client @@ -71,17 +71,6 @@ impl HyperlaneChain for CosmosProvider { } } -#[async_trait] -impl AgentMetricsFetcher for CosmosProvider { - async fn get_balance(&self, address: String) -> ChainResult { - Ok(self - .grpc_client - .get_balance(address, self.canonical_asset.clone()) - .await? - .into()) - } -} - #[async_trait] impl HyperlaneProvider for CosmosProvider { async fn get_block_by_hash(&self, _hash: &H256) -> ChainResult { @@ -96,4 +85,12 @@ impl HyperlaneProvider for CosmosProvider { // FIXME Ok(true) } + + async fn get_balance(&self, address: String) -> ChainResult { + Ok(self + .grpc_client + .get_balance(address, self.canonical_asset.clone()) + .await? + .into()) + } } diff --git a/rust/chains/hyperlane-ethereum/src/provider.rs b/rust/chains/hyperlane-ethereum/src/provider.rs index c73c1d8da5..5fced1aaf4 100644 --- a/rust/chains/hyperlane-ethereum/src/provider.rs +++ b/rust/chains/hyperlane-ethereum/src/provider.rs @@ -6,8 +6,8 @@ use std::time::Duration; use async_trait::async_trait; use derive_new::new; use ethers::prelude::Middleware; -use ethers_core::types::Address; -use hyperlane_core::{ethers_core_types, metrics::agent::AgentMetricsFetcher, U256}; +use ethers_core::abi::Address; +use hyperlane_core::{ethers_core_types, U256}; use tokio::time::sleep; use tracing::instrument; @@ -45,25 +45,6 @@ where } } -#[async_trait] -impl AgentMetricsFetcher for EthereumProvider -where - M: Middleware + 'static, -{ - #[instrument(err, skip(self))] - async fn get_balance(&self, address: String) -> ChainResult { - // Can't use the address directly as a string, because ethers interprets it - // as an ENS name rather than an address. - let addr: Address = address.parse()?; - let balance = self - .provider - .get_balance(addr, None) - .await - .map_err(ChainCommunicationError::from_other)?; - Ok(balance.into()) - } -} - #[async_trait] impl HyperlaneProvider for EthereumProvider where @@ -125,6 +106,19 @@ where .map_err(ChainCommunicationError::from_other)?; Ok(!code.is_empty()) } + + #[instrument(err, skip(self))] + async fn get_balance(&self, address: String) -> ChainResult { + // Can't use the address directly as a string, because ethers interprets it + // as an ENS name rather than an address. + let addr: Address = address.parse()?; + let balance = self + .provider + .get_balance(addr, None) + .await + .map_err(ChainCommunicationError::from_other)?; + Ok(balance.into()) + } } impl EthereumProvider @@ -165,29 +159,6 @@ impl BuildableWithProvider for HyperlaneProviderBuilder { } } -/// Builder for the Agent Metrics Fetcher. -// TODO: Remove this when trait upcasting is stabilized and Box can be used -// as Box -// Tracking issue: -// https://github.com/rust-lang/rust/issues/65991 -pub struct AgentMetricsFetcherBuilder {} - -#[async_trait] -impl BuildableWithProvider for AgentMetricsFetcherBuilder { - type Output = Box; - - async fn build_with_provider( - &self, - provider: M, - locator: &ContractLocator, - ) -> Self::Output { - Box::new(EthereumProvider::new( - Arc::new(provider), - locator.domain.clone(), - )) - } -} - /// Call a get function that returns a Result> and retry if the inner /// option is None. This can happen because the provider has not discovered the /// object we are looking for yet. diff --git a/rust/chains/hyperlane-fuel/src/provider.rs b/rust/chains/hyperlane-fuel/src/provider.rs index 54c38e850c..8048076e04 100644 --- a/rust/chains/hyperlane-fuel/src/provider.rs +++ b/rust/chains/hyperlane-fuel/src/provider.rs @@ -1,8 +1,7 @@ use async_trait::async_trait; use hyperlane_core::{ - metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, - HyperlaneProvider, TxnInfo, H256, U256, + BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, }; /// A wrapper around a fuel provider to get generic blockchain information. @@ -19,13 +18,6 @@ impl HyperlaneChain for FuelProvider { } } -#[async_trait] -impl AgentMetricsFetcher for FuelProvider { - async fn get_balance(&self, address: String) -> ChainResult { - todo!() - } -} - #[async_trait] impl HyperlaneProvider for FuelProvider { async fn get_block_by_hash(&self, hash: &H256) -> ChainResult { @@ -39,4 +31,8 @@ impl HyperlaneProvider for FuelProvider { async fn is_contract(&self, address: &H256) -> ChainResult { todo!() } + + async fn get_balance(&self, address: String) -> ChainResult { + todo!() + } } diff --git a/rust/chains/hyperlane-sealevel/src/provider.rs b/rust/chains/hyperlane-sealevel/src/provider.rs index fdd20795a8..47be23014c 100644 --- a/rust/chains/hyperlane-sealevel/src/provider.rs +++ b/rust/chains/hyperlane-sealevel/src/provider.rs @@ -1,8 +1,7 @@ use async_trait::async_trait; use hyperlane_core::{ - metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, - HyperlaneProvider, TxnInfo, H256, U256, + BlockInfo, ChainResult, HyperlaneChain, HyperlaneDomain, HyperlaneProvider, TxnInfo, H256, U256, }; /// A wrapper around a Sealevel provider to get generic blockchain information. @@ -30,13 +29,6 @@ impl HyperlaneChain for SealevelProvider { } } -#[async_trait] -impl AgentMetricsFetcher for SealevelProvider { - async fn get_balance(&self, _address: String) -> ChainResult { - todo!() // FIXME - } -} - #[async_trait] impl HyperlaneProvider for SealevelProvider { async fn get_block_by_hash(&self, _hash: &H256) -> ChainResult { @@ -51,4 +43,8 @@ impl HyperlaneProvider for SealevelProvider { // FIXME Ok(true) } + + async fn get_balance(&self, _address: String) -> ChainResult { + todo!() // FIXME + } } diff --git a/rust/hyperlane-base/src/agent.rs b/rust/hyperlane-base/src/agent.rs index 40fa639051..5df9250889 100644 --- a/rust/hyperlane-base/src/agent.rs +++ b/rust/hyperlane-base/src/agent.rs @@ -8,7 +8,7 @@ use tokio::task::JoinHandle; use tracing::{debug_span, instrument::Instrumented, Instrument}; use crate::{ - metrics::{create_agent_metrics, CoreMetrics, Metrics as AgentMetrics}, + metrics::{create_agent_metrics, AgentMetrics, CoreMetrics}, settings::Settings, }; diff --git a/rust/hyperlane-base/src/metrics/agent_metrics.rs b/rust/hyperlane-base/src/metrics/agent_metrics.rs index 87d10993f9..bf60cc9afc 100644 --- a/rust/hyperlane-base/src/metrics/agent_metrics.rs +++ b/rust/hyperlane-base/src/metrics/agent_metrics.rs @@ -4,7 +4,8 @@ use derive_builder::Builder; use derive_new::new; use eyre::Result; use hyperlane_core::metrics::agent::u256_as_scaled_f64; -use hyperlane_core::{metrics::agent::AgentMetricsFetcher, HyperlaneDomain}; +use hyperlane_core::HyperlaneDomain; +use hyperlane_core::HyperlaneProvider; use maplit::hashmap; use prometheus::GaugeVec; use tokio::time::MissedTickBehavior; @@ -27,7 +28,7 @@ pub const WALLET_BALANCE_HELP: &str = /// Agent-specific metrics #[derive(Clone, Builder)] -pub struct Metrics { +pub struct AgentMetrics { /// Current balance of native tokens for the /// wallet address. /// - `chain`: the chain name (or chain ID if the name is unknown) of the @@ -41,8 +42,8 @@ pub struct Metrics { wallet_balance: Option, } -pub(crate) fn create_agent_metrics(metrics: &CoreMetrics) -> Result { - Ok(MetricsBuilder::default() +pub(crate) fn create_agent_metrics(metrics: &CoreMetrics) -> Result { + Ok(AgentMetricsBuilder::default() .wallet_balance(metrics.new_gauge( "wallet_balance", WALLET_BALANCE_HELP, @@ -67,15 +68,15 @@ pub struct AgentMetricsConf { pub name: String, } -/// Utility struct to update agent metrics +/// Utility struct to update agent metrics for a given chain #[derive(new)] -pub struct AgentMetrics { - metrics: Metrics, +pub struct AgentMetricsUpdater { + metrics: AgentMetrics, conf: AgentMetricsConf, - fetcher: Box, + provider: Box, } -impl AgentMetrics { +impl AgentMetricsUpdater { async fn update_wallet_balances(&self) { let Some(wallet_addr) = self.conf.address.clone() else { return; @@ -86,7 +87,7 @@ impl AgentMetrics { }; let chain = self.conf.domain.name(); - match self.fetcher.get_balance(wallet_addr.clone()).await { + match self.provider.get_balance(wallet_addr.clone()).await { Ok(balance) => { // Okay, so the native type is not a token, but whatever, close enough. // Note: This is ETH for many chains, but not all so that is why we use `N` and `Native` diff --git a/rust/hyperlane-base/src/settings/chains.rs b/rust/hyperlane-base/src/settings/chains.rs index 99add295c2..a56ad31afb 100644 --- a/rust/hyperlane-base/src/settings/chains.rs +++ b/rust/hyperlane-base/src/settings/chains.rs @@ -6,9 +6,9 @@ use eyre::{eyre, Context, Result}; use ethers_prometheus::middleware::{ChainInfo, ContractInfo, PrometheusMiddlewareConf}; use hyperlane_core::{ - metrics::agent::AgentMetricsFetcher, AggregationIsm, CcipReadIsm, ContractLocator, - HyperlaneAbi, HyperlaneDomain, HyperlaneDomainProtocol, HyperlaneMessage, HyperlaneProvider, - IndexMode, InterchainGasPaymaster, InterchainGasPayment, InterchainSecurityModule, Mailbox, + AggregationIsm, CcipReadIsm, ContractLocator, HyperlaneAbi, HyperlaneDomain, + HyperlaneDomainProtocol, HyperlaneMessage, HyperlaneProvider, IndexMode, + InterchainGasPaymaster, InterchainGasPayment, InterchainSecurityModule, Mailbox, MerkleTreeHook, MerkleTreeInsertion, MultisigIsm, RoutingIsm, SequenceIndexer, ValidatorAnnounce, H256, }; @@ -119,7 +119,16 @@ impl ChainConf { } ChainConnectionConf::Fuel(_) => todo!(), ChainConnectionConf::Sealevel(_) => todo!(), - ChainConnectionConf::Cosmos(_) => todo!(), + ChainConnectionConf::Cosmos(conf) => { + let locator = self.locator(H256::zero()); + let provider = CosmosProvider::new( + locator.domain.clone(), + conf.clone(), + Some(locator.clone()), + None, + )?; + Ok(Box::new(provider) as Box) + } } .context(ctx) } @@ -596,39 +605,6 @@ impl ChainConf { .context(ctx) } - /// Try to convert the chain setting into a trait object for fetching agent metrics - pub async fn build_agent_metrics_fetcher( - &self, - metrics: &CoreMetrics, - ) -> Result> { - match &self.connection { - ChainConnectionConf::Ethereum(conf) => { - let locator = self.locator(H256::zero()); - let provider = self - .build_ethereum( - conf, - &locator, - metrics, - h_eth::AgentMetricsFetcherBuilder {}, - ) - .await?; - Ok(provider) - } - ChainConnectionConf::Fuel(_) => todo!(), - ChainConnectionConf::Sealevel(_) => todo!(), - ChainConnectionConf::Cosmos(conf) => { - let locator = self.locator(H256::zero()); - let provider = CosmosProvider::new( - locator.domain.clone(), - conf.clone(), - Some(locator.clone()), - None, - )?; - Ok(Box::new(provider) as Box) - } - } - } - async fn signer(&self) -> Result> { if let Some(conf) = &self.signer { Ok(Some(conf.build::().await?)) diff --git a/rust/hyperlane-core/src/metrics/agent.rs b/rust/hyperlane-core/src/metrics/agent.rs index e7e77c9c6d..1aee160740 100644 --- a/rust/hyperlane-core/src/metrics/agent.rs +++ b/rust/hyperlane-core/src/metrics/agent.rs @@ -1,21 +1,11 @@ use std::time::Duration; -use async_trait::async_trait; - -use crate::{ChainResult, U256}; +use crate::U256; /// Interval for querying the prometheus metrics endpoint. /// This should be whatever the prometheus scrape interval is pub const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); -/// Trait to be implemented by all chain-specific agent implementations, -/// to support gathering agent metrics. -#[async_trait] -pub trait AgentMetricsFetcher: Send + Sync { - /// Fetch the balance of the wallet address associated with the chain provider. - async fn get_balance(&self, address: String) -> ChainResult; -} - /// Convert a u256 scaled integer value into the corresponding f64 value. #[cfg(feature = "float")] pub fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { diff --git a/rust/hyperlane-core/src/traits/provider.rs b/rust/hyperlane-core/src/traits/provider.rs index 4d0d127467..7b2c930926 100644 --- a/rust/hyperlane-core/src/traits/provider.rs +++ b/rust/hyperlane-core/src/traits/provider.rs @@ -4,9 +4,7 @@ use async_trait::async_trait; use auto_impl::auto_impl; use thiserror::Error; -use crate::{ - metrics::agent::AgentMetricsFetcher, BlockInfo, ChainResult, HyperlaneChain, TxnInfo, H256, -}; +use crate::{BlockInfo, ChainResult, HyperlaneChain, TxnInfo, H256, U256}; /// Interface for a provider. Allows abstraction over different provider types /// for different chains. @@ -17,7 +15,7 @@ use crate::{ /// the context of a contract. #[async_trait] #[auto_impl(&, Box, Arc)] -pub trait HyperlaneProvider: HyperlaneChain + AgentMetricsFetcher + Send + Sync + Debug { +pub trait HyperlaneProvider: HyperlaneChain + Send + Sync + Debug { /// Get block info for a given block hash async fn get_block_by_hash(&self, hash: &H256) -> ChainResult; @@ -26,6 +24,9 @@ pub trait HyperlaneProvider: HyperlaneChain + AgentMetricsFetcher + Send + Sync /// Returns whether a contract exists at the provided address async fn is_contract(&self, address: &H256) -> ChainResult; + + /// Fetch the balance of the wallet address associated with the chain provider. + async fn get_balance(&self, address: String) -> ChainResult; } /// Errors when querying for provider information. diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 1bf109e15f..28c9515137 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -590,13 +590,13 @@ fn termination_invariants_met( } let ending_relayer_balance: f64 = agent_balance_sum(relayer_metrics_port).unwrap(); + + // Make sure the balance was correctly updated in the metrics. // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: // https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/c2288eb31734ba1f2f997e2c6ecb30176427bc2c/rust/utils/run-locally/src/cosmos/cli.rs#L55 // What's stopping this is that the format returned by the `uosmo` balance query is a surprisingly low number (0.000003999999995184) // but then maybe the gas_per_tx is just very low - how can we check that? (maybe by simulating said tx) if starting_relayer_balance <= ending_relayer_balance { - // worth retrying this because metrics are polled every - // `METRICS_SCRAPE_INTERVAL` log!( "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", starting_relayer_balance, diff --git a/rust/utils/run-locally/src/invariants.rs b/rust/utils/run-locally/src/invariants.rs index 4e76c9473c..6fe857a436 100644 --- a/rust/utils/run-locally/src/invariants.rs +++ b/rust/utils/run-locally/src/invariants.rs @@ -132,9 +132,8 @@ pub fn termination_invariants_met( } let ending_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); + // Make sure the balance was correctly updated in the metrics. if starting_relayer_balance <= ending_relayer_balance { - // worth retrying this because metrics are polled every - // `METRICS_SCRAPE_INTERVAL` log!( "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", starting_relayer_balance, From 2b4ae363d518b9dee371fc508db486febece1f84 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 5 Dec 2023 11:07:51 +0000 Subject: [PATCH 18/18] fix: cosmos metrics balance --- .../hyperlane-cosmos/src/providers/grpc.rs | 2 +- .../hyperlane-cosmos/src/providers/mod.rs | 3 +-- rust/ethers-prometheus/src/middleware/mod.rs | 4 +++- .../src/metrics/agent_metrics.rs | 2 +- rust/hyperlane-core/src/metrics/agent.rs | 18 +++++++++++++++++- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 12825cb42f..0cbc7f2aff 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -224,7 +224,7 @@ impl WasmGrpcProvider { } /// Fetches balance for a given `address` and `denom` - pub async fn get_balance(&self, address: String, denom: String) -> ChainResult { + pub async fn get_balance(&self, address: String, denom: String) -> ChainResult { let mut client = QueryBalanceClient::new(self.channel.clone()); let balance_request = tonic::Request::new(QueryBalanceRequest { address, denom }); diff --git a/rust/chains/hyperlane-cosmos/src/providers/mod.rs b/rust/chains/hyperlane-cosmos/src/providers/mod.rs index 1ac97f1e75..973a886a3c 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/mod.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/mod.rs @@ -90,7 +90,6 @@ impl HyperlaneProvider for CosmosProvider { Ok(self .grpc_client .get_balance(address, self.canonical_asset.clone()) - .await? - .into()) + .await?) } } diff --git a/rust/ethers-prometheus/src/middleware/mod.rs b/rust/ethers-prometheus/src/middleware/mod.rs index 1526f84e72..c63447484e 100644 --- a/rust/ethers-prometheus/src/middleware/mod.rs +++ b/rust/ethers-prometheus/src/middleware/mod.rs @@ -15,6 +15,7 @@ use ethers::prelude::*; use ethers::types::transaction::eip2718::TypedTransaction; use ethers::utils::hex::ToHex; use hyperlane_core::metrics::agent::u256_as_scaled_f64; +use hyperlane_core::HyperlaneDomainProtocol; use log::{debug, trace}; use maplit::hashmap; use prometheus::{CounterVec, GaugeVec, IntCounterVec, IntGaugeVec}; @@ -539,7 +540,8 @@ impl PrometheusMiddleware { } if let Some(gas_price_gwei) = gas_price_gwei { if let Some(london_fee) = current_block.base_fee_per_gas { - let gas = u256_as_scaled_f64(london_fee.into(), 18) * 1e9; + let gas = + u256_as_scaled_f64(london_fee.into(), HyperlaneDomainProtocol::Ethereum) * 1e9; trace!("Gas price for chain {chain} is {gas:.1}gwei"); gas_price_gwei.with(&hashmap! { "chain" => chain }).set(gas); } else { diff --git a/rust/hyperlane-base/src/metrics/agent_metrics.rs b/rust/hyperlane-base/src/metrics/agent_metrics.rs index bf60cc9afc..ab4e01f513 100644 --- a/rust/hyperlane-base/src/metrics/agent_metrics.rs +++ b/rust/hyperlane-base/src/metrics/agent_metrics.rs @@ -92,7 +92,7 @@ impl AgentMetricsUpdater { // Okay, so the native type is not a token, but whatever, close enough. // Note: This is ETH for many chains, but not all so that is why we use `N` and `Native` // TODO: can we get away with scaling as 18 in all cases here? I am guessing not. - let balance = u256_as_scaled_f64(balance, 18); + let balance = u256_as_scaled_f64(balance, self.conf.domain.domain_protocol()); trace!("Wallet {wallet_name} ({wallet_addr}) on chain {chain} balance is {balance} of the native currency"); wallet_balance_metric .with(&hashmap! { diff --git a/rust/hyperlane-core/src/metrics/agent.rs b/rust/hyperlane-core/src/metrics/agent.rs index 1aee160740..02c2795e59 100644 --- a/rust/hyperlane-core/src/metrics/agent.rs +++ b/rust/hyperlane-core/src/metrics/agent.rs @@ -1,13 +1,29 @@ +use crate::HyperlaneDomainProtocol; use std::time::Duration; use crate::U256; +const ETHEREUM_DECIMALS: u8 = 18; +const COSMOS_DECIMALS: u8 = 6; +const SOLANA_DECIMALS: u8 = 9; + /// Interval for querying the prometheus metrics endpoint. /// This should be whatever the prometheus scrape interval is pub const METRICS_SCRAPE_INTERVAL: Duration = Duration::from_secs(60); /// Convert a u256 scaled integer value into the corresponding f64 value. #[cfg(feature = "float")] -pub fn u256_as_scaled_f64(value: U256, decimals: u8) -> f64 { +pub fn u256_as_scaled_f64(value: U256, domain: HyperlaneDomainProtocol) -> f64 { + let decimals = decimals_by_protocol(domain); value.to_f64_lossy() / (10u64.pow(decimals as u32) as f64) } + +/// Get the decimals each protocol typically uses for its lowest denomination +/// of the native token +pub fn decimals_by_protocol(protocol: HyperlaneDomainProtocol) -> u8 { + match protocol { + HyperlaneDomainProtocol::Cosmos => COSMOS_DECIMALS, + HyperlaneDomainProtocol::Sealevel => SOLANA_DECIMALS, + _ => ETHEREUM_DECIMALS, + } +}