From de29799ae029d9d6156a94f90f7fa8e4a4166083 Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Tue, 14 May 2024 08:58:58 -0400 Subject: [PATCH 1/8] Remove UUID from attrs sent_to metrics. --- app/services/metrics_service.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/services/metrics_service.rb b/app/services/metrics_service.rb index 7e201be75a5..08c2dbb68c4 100644 --- a/app/services/metrics_service.rb +++ b/app/services/metrics_service.rb @@ -93,8 +93,7 @@ def self.record(description, service: nil, name: "unknown", caller: nil) app_name: app, attrs: { service: service, - endpoint: name, - uuid: uuid + endpoint: name } } MetricsService.emit_gauge(sent_to_info) From 90faa34c883f091d16ca42484275e9f0621043b8 Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Tue, 14 May 2024 12:32:29 -0400 Subject: [PATCH 2/8] pushing with some linting issues for visibility. --- Gemfile | 3 + Gemfile.lock | 192 ++++++++++++++++++++++++++ config/initializers/open_telemetry.rb | 28 ++++ 3 files changed, 223 insertions(+) create mode 100644 config/initializers/open_telemetry.rb diff --git a/Gemfile b/Gemfile index 73271918d97..2f89ebd8f7c 100644 --- a/Gemfile +++ b/Gemfile @@ -38,6 +38,9 @@ gem "moment_timezone-rails" gem "multiverse" gem "newrelic_rpm" gem "nokogiri", ">= 1.11.0.rc4" +gem "opentelemetry-exporter-otlp" +gem "opentelemetry-instrumentation-all" +gem "opentelemetry-sdk" gem "paper_trail", "~> 12.0" # Used to speed up reporting gem "parallel" diff --git a/Gemfile.lock b/Gemfile.lock index 2f603a7956a..118116022c4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -324,6 +324,9 @@ GEM rchardet (~> 1.8) globalid (1.1.0) activesupport (>= 5.0) + google-protobuf (3.25.3) + googleapis-common-protos-types (1.14.0) + google-protobuf (~> 3.18) govdelivery-tms (2.8.4) activesupport faraday @@ -464,6 +467,191 @@ GEM faraday (>= 0.9) sawyer (~> 0.8.0, >= 0.5.3) open4 (1.3.4) + opentelemetry-api (1.1.0) + opentelemetry-common (0.19.7) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-otlp (0.24.2) + google-protobuf (~> 3.19) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.19.6) + opentelemetry-sdk (~> 1.2) + opentelemetry-semantic_conventions + opentelemetry-instrumentation-action_pack (0.5.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rack (~> 0.21) + opentelemetry-instrumentation-action_view (0.4.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-active_support (~> 0.1) + opentelemetry-instrumentation-base (~> 0.20) + opentelemetry-instrumentation-active_job (0.4.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-active_model_serializers (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-active_record (0.5.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + ruby2_keywords + opentelemetry-instrumentation-active_support (0.3.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-all (0.33.0) + opentelemetry-instrumentation-active_model_serializers (~> 0.19.0) + opentelemetry-instrumentation-aws_sdk (~> 0.3.0) + opentelemetry-instrumentation-bunny (~> 0.19.0) + opentelemetry-instrumentation-concurrent_ruby (~> 0.20.0) + opentelemetry-instrumentation-dalli (~> 0.22.0) + opentelemetry-instrumentation-delayed_job (~> 0.19.0) + opentelemetry-instrumentation-ethon (~> 0.20.0) + opentelemetry-instrumentation-excon (~> 0.20.0) + opentelemetry-instrumentation-faraday (~> 0.22.0) + opentelemetry-instrumentation-graphql (~> 0.24.0) + opentelemetry-instrumentation-http (~> 0.21.0) + opentelemetry-instrumentation-http_client (~> 0.21.0) + opentelemetry-instrumentation-koala (~> 0.19.0) + opentelemetry-instrumentation-lmdb (~> 0.21.0) + opentelemetry-instrumentation-mongo (~> 0.21.0) + opentelemetry-instrumentation-mysql2 (~> 0.22.0) + opentelemetry-instrumentation-net_http (~> 0.21.0) + opentelemetry-instrumentation-pg (~> 0.23.0) + opentelemetry-instrumentation-que (~> 0.5.0) + opentelemetry-instrumentation-racecar (~> 0.1.0) + opentelemetry-instrumentation-rack (~> 0.22.0) + opentelemetry-instrumentation-rails (~> 0.25.0) + opentelemetry-instrumentation-rake (~> 0.1.0) + opentelemetry-instrumentation-rdkafka (~> 0.2.0) + opentelemetry-instrumentation-redis (~> 0.24.0) + opentelemetry-instrumentation-resque (~> 0.3.0) + opentelemetry-instrumentation-restclient (~> 0.21.0) + opentelemetry-instrumentation-ruby_kafka (~> 0.19.0) + opentelemetry-instrumentation-sidekiq (~> 0.22.0) + opentelemetry-instrumentation-sinatra (~> 0.21.0) + opentelemetry-instrumentation-trilogy (~> 0.52.0) + opentelemetry-instrumentation-aws_sdk (0.3.2) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-base (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-registry (~> 0.1) + opentelemetry-instrumentation-bunny (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-concurrent_ruby (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-dalli (0.22.2) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-delayed_job (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-ethon (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-excon (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-faraday (0.22.0) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-graphql (0.24.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-http (0.21.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-http_client (0.21.0) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-koala (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-lmdb (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-mongo (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-mysql2 (0.22.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-net_http (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-pg (0.23.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-que (0.5.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-racecar (0.1.2) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rack (0.22.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rails (0.25.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-action_pack (~> 0.5.0) + opentelemetry-instrumentation-action_view (~> 0.4.0) + opentelemetry-instrumentation-active_job (~> 0.4.0) + opentelemetry-instrumentation-active_record (~> 0.5.0) + opentelemetry-instrumentation-active_support (~> 0.3.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rake (0.1.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rdkafka (0.2.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-redis (0.24.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-resque (0.3.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-restclient (0.21.0) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-ruby_kafka (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-sidekiq (0.22.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-sinatra (0.21.5) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rack (~> 0.21) + opentelemetry-instrumentation-trilogy (0.52.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-semantic_conventions (>= 1.8.0) + opentelemetry-registry (0.2.0) + opentelemetry-api (~> 1.1) + opentelemetry-sdk (1.2.1) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.19.3) + opentelemetry-registry (~> 0.2) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.10.0) + opentelemetry-api (~> 1.0) paper_trail (12.3.0) activerecord (>= 5.2) request_store (~> 1.1) @@ -621,6 +809,7 @@ GEM ruby-plsql (0.8.0) ruby-prof (1.4.1) ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) ruby_dep (1.5.0) ruby_parser (3.20.3) sexp_processor (~> 4.16) @@ -802,6 +991,9 @@ DEPENDENCIES multiverse newrelic_rpm nokogiri (>= 1.11.0.rc4) + opentelemetry-exporter-otlp + opentelemetry-instrumentation-all + opentelemetry-sdk paper_trail (~> 12.0) parallel paranoia (~> 2.2) diff --git a/config/initializers/open_telemetry.rb b/config/initializers/open_telemetry.rb new file mode 100644 index 00000000000..0e401e21053 --- /dev/null +++ b/config/initializers/open_telemetry.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +DT_API_URL = ENV["DT_API_URL"] +DT_API_TOKEN = ENV["DT_API_TOKEN"] +def init_opentelemetry # rubocop:disable Metrics/MethodLength + OpenTelemetry::SDK.configure do |c| + c.service_name = "caseflow" + c.service_version = "1.0.1" + # automatic instrumentation + c.use_all + for name in ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties", "/var/lib/dynatrace/enrichment/dt_metadata.properties", "/var/lib/dynatrace/enrichment/dt_host_metadata.properties"] do + begin + c.resource = OpenTelemetry::SDK::Resources::Resource.create(Hash[*File.read(name.start_with?("/var") ? name : File.read(name)).split(/[=\n]+/)]) + rescue + end + end + c.add_span_processor( + OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: DT_API_URL + "/v1/traces", + headers: { + "Authorization": "Api-Token " + DT_API_TOKEN + } + ) + ) + ) + end +end From edc7dd3c44e35d46014c7b29c7b2ed5ae4a8ae9e Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Tue, 14 May 2024 13:44:57 -0400 Subject: [PATCH 3/8] Change config implementation. --- config/initializers/open_telemetry.rb | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/config/initializers/open_telemetry.rb b/config/initializers/open_telemetry.rb index 0e401e21053..bed1e4c4b27 100644 --- a/config/initializers/open_telemetry.rb +++ b/config/initializers/open_telemetry.rb @@ -2,27 +2,27 @@ DT_API_URL = ENV["DT_API_URL"] DT_API_TOKEN = ENV["DT_API_TOKEN"] -def init_opentelemetry # rubocop:disable Metrics/MethodLength - OpenTelemetry::SDK.configure do |c| - c.service_name = "caseflow" - c.service_version = "1.0.1" - # automatic instrumentation - c.use_all - for name in ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties", "/var/lib/dynatrace/enrichment/dt_metadata.properties", "/var/lib/dynatrace/enrichment/dt_host_metadata.properties"] do - begin - c.resource = OpenTelemetry::SDK::Resources::Resource.create(Hash[*File.read(name.start_with?("/var") ? name : File.read(name)).split(/[=\n]+/)]) - rescue - end + +OpenTelemetry::SDK.configure do |config| + config.service_name = "caseflow" + config.service_version = "1.0.1" + # automatic instrumentation + config.use_all + for name in ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties", "/var/lib/dynatrace/enrichment/dt_metadata.properties", "/var/lib/dynatrace/enrichment/dt_host_metadata.properties"] do + begin + config.resource = OpenTelemetry::SDK::Resources::Resource.create(Hash[*File.read(name.start_with?("/var") ? name : File.read(name)).split(/[=\n]+/)]) + rescue end - c.add_span_processor( - OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( - OpenTelemetry::Exporter::OTLP::Exporter.new( - endpoint: DT_API_URL + "/v1/traces", - headers: { - "Authorization": "Api-Token " + DT_API_TOKEN - } - ) + end + config.add_span_processor( + OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: DT_API_URL + "/v1/traces", + headers: { + "Authorization": "Api-Token " + DT_API_TOKEN + } ) ) - end + ) end + From 78f11e2a94b0d75dc61b2c6f55b859f2b0090474 Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Tue, 14 May 2024 14:30:21 -0400 Subject: [PATCH 4/8] Remove UUID from spec --- spec/services/metrics_service_spec.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/services/metrics_service_spec.rb b/spec/services/metrics_service_spec.rb index 6360a5273b8..178a302a672 100644 --- a/spec/services/metrics_service_spec.rb +++ b/spec/services/metrics_service_spec.rb @@ -38,8 +38,7 @@ app_name: "other", attrs: { service: service, - endpoint: name, - uuid: anything + endpoint: name } ) expect(MetricsService).to receive(:increment_counter).with( From 2cdf4873607aedcfc225a2d81c9ddccc01df35d1 Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Tue, 14 May 2024 16:20:27 -0400 Subject: [PATCH 5/8] remove uuid from update_appellant job --- spec/jobs/update_appellant_representation_job_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/jobs/update_appellant_representation_job_spec.rb b/spec/jobs/update_appellant_representation_job_spec.rb index f045696399a..35219a6cb01 100644 --- a/spec/jobs/update_appellant_representation_job_spec.rb +++ b/spec/jobs/update_appellant_representation_job_spec.rb @@ -43,7 +43,7 @@ ) expect(MetricsService).to receive(:emit_gauge).with( app_name: "queue_job", - attrs: { endpoint: "AppellantNotification.appeal_mapper", service: "queue_job", uuid: anything }, + attrs: { endpoint: "AppellantNotification.appeal_mapper", service: "queue_job" }, metric_group: "service", metric_name: "request_latency", metric_value: anything From 5b755662d02b63dfd7e3c41158f52cb606f5e4a4 Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Wed, 15 May 2024 14:36:40 -0400 Subject: [PATCH 6/8] diable rubocop for open_telemetry init --- config/initializers/open_telemetry.rb | 1 + spec/services/metrics_service_spec.rb | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/initializers/open_telemetry.rb b/config/initializers/open_telemetry.rb index bed1e4c4b27..d517bdfc48c 100644 --- a/config/initializers/open_telemetry.rb +++ b/config/initializers/open_telemetry.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +# rubocop:disable all DT_API_URL = ENV["DT_API_URL"] DT_API_TOKEN = ENV["DT_API_TOKEN"] diff --git a/spec/services/metrics_service_spec.rb b/spec/services/metrics_service_spec.rb index 178a302a672..9b349e84ebe 100644 --- a/spec/services/metrics_service_spec.rb +++ b/spec/services/metrics_service_spec.rb @@ -71,8 +71,7 @@ app_name: "other", attrs: { service: service, - endpoint: name, - uuid: anything + endpoint: name } }, start: anything, From 12305999400c7c3748b7906bda03292689f8565e Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Wed, 15 May 2024 16:20:16 -0400 Subject: [PATCH 7/8] OTEL fixes --- config/initializers/open_telemetry.rb | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/config/initializers/open_telemetry.rb b/config/initializers/open_telemetry.rb index d517bdfc48c..a32d9c73bac 100644 --- a/config/initializers/open_telemetry.rb +++ b/config/initializers/open_telemetry.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true -# rubocop:disable all + +# rubocop:disable Layout/LineLength DT_API_URL = ENV["DT_API_URL"] DT_API_TOKEN = ENV["DT_API_TOKEN"] @@ -9,11 +10,8 @@ config.service_version = "1.0.1" # automatic instrumentation config.use_all - for name in ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties", "/var/lib/dynatrace/enrichment/dt_metadata.properties", "/var/lib/dynatrace/enrichment/dt_host_metadata.properties"] do - begin - config.resource = OpenTelemetry::SDK::Resources::Resource.create(Hash[*File.read(name.start_with?("/var") ? name : File.read(name)).split(/[=\n]+/)]) - rescue - end + ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties", "/var/lib/dynatrace/enrichment/dt_metadata.properties", "/var/lib/dynatrace/enrichment/dt_host_metadata.properties"].each do |name| + config.resource = OpenTelemetry::SDK::Resources::Resource.create(Hash[*File.read(name.start_with?("/var") ? name : File.read(name)).split(/[=\n]+/)]) end config.add_span_processor( OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( @@ -26,4 +24,4 @@ ) ) end - + # rubocop:enable Layout/LineLength From a6b86531fa55b9ca35b6eb8ab6e450529df56053 Mon Sep 17 00:00:00 2001 From: Noelle Adkin <98478937+NoelleAd@users.noreply.github.com> Date: Thu, 16 May 2024 10:54:43 -0400 Subject: [PATCH 8/8] rescue, errors --- config/initializers/open_telemetry.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/config/initializers/open_telemetry.rb b/config/initializers/open_telemetry.rb index a32d9c73bac..f310ea399f4 100644 --- a/config/initializers/open_telemetry.rb +++ b/config/initializers/open_telemetry.rb @@ -11,7 +11,12 @@ # automatic instrumentation config.use_all ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties", "/var/lib/dynatrace/enrichment/dt_metadata.properties", "/var/lib/dynatrace/enrichment/dt_host_metadata.properties"].each do |name| - config.resource = OpenTelemetry::SDK::Resources::Resource.create(Hash[*File.read(name.start_with?("/var") ? name : File.read(name)).split(/[=\n]+/)]) + begin + config.resource = OpenTelemetry::SDK::Resources::Resource.create(Hash[*File.read(name.start_with?("/var") ? name : File.read(name)).split(/[=\n]+/)]) + rescue StandardError => error + Rails.logger.error(error.full_message) + raise error.full_message + end end config.add_span_processor( OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(