From 6e8e60d921af5d5172fb67358a5a8b8d03964763 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Fri, 3 Nov 2023 13:15:23 +1100 Subject: [PATCH] feat: use Grafana OTLP collector (#345) --- Cargo.lock | 15 ++++++++++++ infra/main.tf | 3 +++ infra/modules/leader/main.tf | 9 ++++++- infra/modules/leader/variables.tf | 8 +++++++ infra/terraform-dev.tfvars | 2 ++ infra/variables.tf | 10 ++++++++ mpc-recovery/Cargo.toml | 5 +++- mpc-recovery/src/logging.rs | 40 +++++++++++++++++++++++-------- 8 files changed, 80 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11fa28653..c85ebd9dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4677,6 +4677,19 @@ dependencies = [ "opentelemetry_sdk", ] +[[package]] +name = "opentelemetry-http" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry_api", + "reqwest", +] + [[package]] name = "opentelemetry-otlp" version = "0.10.0" @@ -4704,11 +4717,13 @@ dependencies = [ "async-trait", "futures-core", "http", + "opentelemetry-http", "opentelemetry-proto", "opentelemetry-semantic-conventions 0.12.0", "opentelemetry_api", "opentelemetry_sdk", "prost 0.11.9", + "reqwest", "thiserror", "tokio", "tonic 0.9.2", diff --git a/infra/main.tf b/infra/main.tf index 57a0b69a5..6f32cce90 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -148,6 +148,9 @@ module "leader" { jwt_signature_pk_url = var.jwt_signature_pk_url + otlp_endpoint = var.otlp_endpoint + opentelemetry_level = var.opentelemetry_level + depends_on = [ google_secret_manager_secret_iam_member.account_creator_secret_access, google_secret_manager_secret_iam_member.fast_auth_partners_secret_access, diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf index 0254bfc5b..608f6c606 100644 --- a/infra/modules/leader/main.tf +++ b/infra/modules/leader/main.tf @@ -61,11 +61,18 @@ resource "google_cloud_run_v2_service" "leader" { } } } - env { name = "MPC_RECOVERY_JWT_SIGNATURE_PK_URL" value = var.jwt_signature_pk_url } + env { + name = "MPC_RECOVERY_OTLP_ENDPOINT" + value = var.otlp_endpoint + } + env { + name = "MPC_RECOVERY_OPENTELEMETRY_LEVEL" + value = var.opentelemetry_level + } env { name = "RUST_LOG" diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf index 6955b0c2e..6e0655e58 100644 --- a/infra/modules/leader/variables.tf +++ b/infra/modules/leader/variables.tf @@ -51,3 +51,11 @@ variable "fast_auth_partners_secret_id" { variable "jwt_signature_pk_url" { type = string } + +variable "otlp_endpoint" { + type = string +} + +variable "opentelemetry_level" { + type = string +} diff --git a/infra/terraform-dev.tfvars b/infra/terraform-dev.tfvars index f22843b8a..0ab14fe3e 100644 --- a/infra/terraform-dev.tfvars +++ b/infra/terraform-dev.tfvars @@ -20,3 +20,5 @@ signer_configs = [ } ] jwt_signature_pk_url = "https://www.googleapis.com/robot/v1/metadata/x509/securetoken@system.gserviceaccount.com" +otlp_endpoint = "https://otel.dev.api.pagoda.co:443/v1/traces" +opentelemetry_level = "debug" diff --git a/infra/variables.tf b/infra/variables.tf index 9d5a8bae4..5917e1d50 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -53,3 +53,13 @@ variable "signer_configs" { variable "jwt_signature_pk_url" { type = string } + +variable "otlp_endpoint" { + type = string + default = "http://localhost:4317" +} + +variable "opentelemetry_level" { + type = string + default = "off" +} diff --git a/mpc-recovery/Cargo.toml b/mpc-recovery/Cargo.toml index 604f6ffb1..77e3962df 100644 --- a/mpc-recovery/Cargo.toml +++ b/mpc-recovery/Cargo.toml @@ -28,7 +28,10 @@ hyper-rustls = { version = "=0.23", features = ["http2"] } jsonwebtoken = "8.3.0" lazy_static = "1.4.0" opentelemetry = { version = "0.20.0", features = ["rt-tokio", "trace"] } -opentelemetry-otlp = "0.13.0" +opentelemetry-otlp = { version = "0.13.0", features = [ + "http-proto", + "reqwest-client", +] } opentelemetry-semantic-conventions = "0.12.0" prometheus = { version = "0.13.3", features = ["process"] } rand = "0.7" diff --git a/mpc-recovery/src/logging.rs b/mpc-recovery/src/logging.rs index 87ea39563..e2813dd3b 100644 --- a/mpc-recovery/src/logging.rs +++ b/mpc-recovery/src/logging.rs @@ -79,14 +79,23 @@ impl Display for ColorOutput { } /// Configures exporter of span and trace data. -#[derive(Debug, Default, clap::Parser)] +#[derive(Debug, clap::Parser)] pub struct Options { /// Enables export of span data using opentelemetry exporters. - #[clap(long, value_enum, default_value = "off")] - opentelemetry: OpenTelemetryLevel, + #[clap( + long, + env("MPC_RECOVERY_OPENTELEMETRY_LEVEL"), + value_enum, + default_value = "off" + )] + opentelemetry_level: OpenTelemetryLevel, /// Opentelemetry gRPC collector endpoint. - #[clap(long, default_value = "http://localhost:4317")] + #[clap( + long, + env("MPC_RECOVERY_OTLP_ENDPOINT"), + default_value = "http://localhost:4317" + )] otlp_endpoint: String, /// Whether the log needs to be colored. @@ -99,11 +108,22 @@ pub struct Options { log_span_events: bool, } +impl Default for Options { + fn default() -> Self { + Self { + opentelemetry_level: OpenTelemetryLevel::OFF, + otlp_endpoint: "http://localhost:4317".to_string(), + color: ColorOutput::Auto, + log_span_events: false, + } + } +} + impl Options { pub fn into_str_args(self) -> Vec { let mut buf = vec![ - "--opentelemetry".to_string(), - self.opentelemetry.to_string(), + "--opentelemetry-level".to_string(), + self.opentelemetry_level.to_string(), "--otlp-endpoint".to_string(), self.otlp_endpoint, "--color".to_string(), @@ -184,7 +204,7 @@ where let (filter, handle) = reload::Layer::::new(filter); let resource = vec![ - KeyValue::new(SERVICE_NAME, format!("mpc:{}", node_id)), + KeyValue::new(SERVICE_NAME, format!("mpc:{}:{}", env, node_id)), KeyValue::new("env", env), KeyValue::new("node_id", node_id), ]; @@ -193,7 +213,7 @@ where .tracing() .with_exporter( opentelemetry_otlp::new_exporter() - .tonic() + .http() .with_endpoint(otlp_endpoint), ) .with_trace_config( @@ -213,7 +233,7 @@ where fn set_default_otlp_level(options: &Options) { // Record the initial tracing level specified as a command-line flag. Use this recorded value to // reset opentelemetry filter when the LogConfig file gets deleted. - DEFAULT_OTLP_LEVEL.set(options.opentelemetry).unwrap(); + DEFAULT_OTLP_LEVEL.set(options.opentelemetry_level).unwrap(); } /// The resource representing a registered subscriber. @@ -287,7 +307,7 @@ pub async fn default_subscriber_with_opentelemetry( .unwrap_or_else(|_| panic!("Failed to set Log Layer Filter")); let (subscriber, handle) = add_opentelemetry_layer( - options.opentelemetry, + options.opentelemetry_level, &options.otlp_endpoint, env, node_id,