Merge pull request #883 from near/xiangyi/verify_log

add timeout to requests to nodes and add metric for iter cnts
near · Oct 10, 2024 · c08f6f0 · c08f6f0
2 parents b0a09ec + d798f96
commit c08f6f0
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 2 deletions.
diff --git a/chain-signatures/node/src/cli.rs b/chain-signatures/node/src/cli.rs
@@ -8,6 +8,7 @@ use local_ip_address::local_ip;
 use near_account_id::AccountId;
 use near_crypto::{InMemorySigner, SecretKey};
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::{mpsc, RwLock};
 use tracing_stackdriver::layer as stackdriver_layer;
 use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
@@ -135,6 +136,7 @@ fn is_running_on_gcp() -> bool {
     let resp = reqwest::blocking::Client::new()
         .get("http://metadata.google.internal/computeMetadata/v1/instance/id")
         .header("Metadata-Flavor", "Google")
+        .timeout(Duration::from_secs(1))
         .send();
 
     match resp {

diff --git a/chain-signatures/node/src/http_client.rs b/chain-signatures/node/src/http_client.rs
@@ -46,6 +46,7 @@ async fn send_encrypted<U: IntoUrl>(
             .post(url.clone())
             .header("content-type", "application/json")
             .json(&message)
+            .timeout(Duration::from_secs(2))
             .send()
             .await
             .map_err(SendError::ReqwestClientError)?;

diff --git a/chain-signatures/node/src/mesh/connection.rs b/chain-signatures/node/src/mesh/connection.rs
@@ -49,7 +49,13 @@ impl Pool {
                 continue;
             };
 
-            let Ok(resp) = self.http.get(url.clone()).send().await else {
+            let Ok(resp) = self
+                .http
+                .get(url.clone())
+                .timeout(Duration::from_secs(1))
+                .send()
+                .await
+            else {
                 tracing::warn!(
                     "Pool.ping resp err participant {:?} url {}",
                     participant,
@@ -93,7 +99,13 @@ impl Pool {
                 continue;
             };
 
-            let Ok(resp) = self.http.get(url).send().await else {
+            let Ok(resp) = self
+                .http
+                .get(url)
+                .timeout(Duration::from_secs(1))
+                .send()
+                .await
+            else {
                 continue;
             };
 

diff --git a/chain-signatures/node/src/metrics.rs b/chain-signatures/node/src/metrics.rs
@@ -398,6 +398,15 @@ pub(crate) static SIGNATURE_PUBLISH_RESPONSE_ERRORS: Lazy<CounterVec> = Lazy::ne
     .unwrap()
 });
 
+pub(crate) static PROTOCOL_ITER_CNT: Lazy<CounterVec> = Lazy::new(|| {
+    try_create_counter_vec(
+        "multichain_protocol_iter_count",
+        "Count of multichain protocol iter",
+        &["node_account_id"],
+    )
+    .unwrap()
+});
+
 pub fn try_create_int_gauge_vec(name: &str, help: &str, labels: &[&str]) -> Result<IntGaugeVec> {
     check_metric_multichain_prefix(name)?;
     let opts = Opts::new(name, help);

diff --git a/chain-signatures/node/src/protocol/mod.rs b/chain-signatures/node/src/protocol/mod.rs
@@ -227,6 +227,10 @@ impl MpcSignProtocol {
         loop {
             let protocol_time = Instant::now();
             tracing::debug!("trying to advance chain signatures protocol");
+            crate::metrics::PROTOCOL_ITER_CNT
+                .with_label_values(&[my_account_id.as_str()])
+                .inc();
+
             loop {
                 let msg_result = self.receiver.try_recv();
                 match msg_result {