fix: start trace exporter only if required

AztecProtocol · Aug 22, 2024 · 1fe374f · 1fe374f
1 parent 0b50480
commit 1fe374f
Show file tree

Hide file tree

Showing 8 changed files with 37 additions and 21 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -62,7 +62,8 @@ services:
       P2P_ENABLED: true
       PEER_ID_PRIVATE_KEY:
       AZTEC_PORT: 8999
-      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-http://otel-collector:4318/v1/metrics}
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4318/v1/traces}
     secrets:
       - ethereum-host
       - p2p-boot-node
@@ -77,13 +78,14 @@ services:
         # if the stack is started with --profile metrics --profile node, give the collector a chance to start before the node
         i=0
         max=3
-        while ! curl --head --silent $$OTEL_EXPORTER_OTLP_ENDPOINT > /dev/null; do
+        while ! curl --head --silent $$OTEL_EXPORTER_OTLP_METRICS_ENDPOINT > /dev/null; do
           echo "OpenTelemetry collector not up. Retrying after 1s";
           sleep 1;
           i=$$((i+1));
           if [ $$i -eq $$max ]; then
-            echo "OpenTelemetry collector at $$OTEL_EXPORTER_OTLP_ENDPOINT not up after $${max}s. Running without metrics";
-            unset OTEL_EXPORTER_OTLP_ENDPOINT;
+            echo "OpenTelemetry collector at $$OTEL_EXPORTER_METRICS_ENDPOINT not up after $${max}s. Running without metrics";
+            unset OTEL_EXPORTER_METRICS_ENDPOINT;
+            unset OTEL_EXPORTER_TRACES_ENDPOINT;
             break
           fi;
         done;

diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf
@@ -337,8 +337,8 @@ resource "aws_ecs_task_definition" "aztec-node" {
           value = tostring(var.PROVING_ENABLED)
         },
         {
-          name  = "OTEL_EXPORTER_OTLP_ENDPOINT"
-          value = "http://aztec-otel.local:4318"
+          name  = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"
+          value = "http://aztec-otel.local:4318/v1/metrics"
         },
         {
           name  = "OTEL_SERVICE_NAME"

diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf
@@ -202,7 +202,7 @@ resource "aws_ecs_task_definition" "aztec-prover-node" {
         { name = "PROVER_NODE_MAX_PENDING_JOBS", value = tostring(var.PROVER_NODE_MAX_PENDING_JOBS) },
 
         // Metrics
-        { name = "OTEL_EXPORTER_OTLP_ENDPOINT", value = "http://aztec-otel.local:4318" },
+        { name = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", value = "http://aztec-otel.local:4318/v1/metrics" },
         { name = "OTEL_SERVICE_NAME", value = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" },
 
         // L1 addresses

diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf
@@ -274,8 +274,8 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" {
         "value": "${var.PROVING_ENABLED}"
       },
       {
-        "name": "OTEL_EXPORTER_OTLP_ENDPOINT",
-        "value": "http://aztec-otel.local:4318"
+        "name": "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
+        "value": "http://aztec-otel.local:4318/v1/metrics"
       },
       {
         "name": "OTEL_SERVICE_NAME",

diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts
@@ -37,8 +37,9 @@ export type EnvVar =
   | 'P2P_QUERY_FOR_IP'
   | 'P2P_TX_POOL_KEEP_PROVEN_FOR'
   | 'TELEMETRY'
-  | 'OTEL_EXPORTER_OTLP_ENDPOINT'
   | 'OTEL_SERVICE_NAME'
+  | 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT'
+  | 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'
   | 'NETWORK_NAME'
   | 'NETWORK'
   | 'API_KEY'

diff --git a/yarn-project/telemetry-client/src/config.ts b/yarn-project/telemetry-client/src/config.ts
@@ -1,15 +1,21 @@
 import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
 
 export interface TelemetryClientConfig {
-  collectorBaseUrl?: URL;
+  metricsCollectorUrl?: URL;
+  tracesCollectorUrl?: URL;
   serviceName: string;
   networkName: string;
 }
 
 export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientConfig> = {
-  collectorBaseUrl: {
-    env: 'OTEL_EXPORTER_OTLP_ENDPOINT',
-    description: 'The URL of the telemetry collector',
+  metricsCollectorUrl: {
+    env: 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
+    description: 'The URL of the telemetry collector for metrics',
+    parseEnv: (val: string) => new URL(val),
+  },
+  tracesCollectorUrl: {
+    env: 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT',
+    description: 'The URL of the telemetry collector for traces',
     parseEnv: (val: string) => new URL(val),
   },
   serviceName: {

diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts
@@ -77,7 +77,11 @@ export class OpenTelemetryClient implements TelemetryClient {
     await Promise.all([this.meterProvider.shutdown()]);
   }
 
-  public static createAndStart(collectorBaseUrl: URL, log: DebugLogger): OpenTelemetryClient {
+  public static createAndStart(
+    metricsCollector: URL,
+    tracesCollector: URL | undefined,
+    log: DebugLogger,
+  ): OpenTelemetryClient {
     const resource = detectResourcesSync({
       detectors: [
         osDetectorSync,
@@ -93,17 +97,20 @@ export class OpenTelemetryClient implements TelemetryClient {
     const tracerProvider = new NodeTracerProvider({
       resource,
     });
-    tracerProvider.addSpanProcessor(
-      new BatchSpanProcessor(new OTLPTraceExporter({ url: new URL('/v1/traces', collectorBaseUrl).href })),
-    );
+
+    // optionally push traces to an OTEL collector instance
+    if (tracesCollector) {
+      tracerProvider.addSpanProcessor(new BatchSpanProcessor(new OTLPTraceExporter({ url: tracesCollector.href })));
+    }
+
     tracerProvider.register();
 
     const meterProvider = new MeterProvider({
       resource,
       readers: [
         new PeriodicExportingMetricReader({
           exporter: new OTLPMetricExporter({
-            url: new URL('/v1/metrics', collectorBaseUrl).href,
+            url: metricsCollector.href,
           }),
         }),
       ],

diff --git a/yarn-project/telemetry-client/src/start.ts b/yarn-project/telemetry-client/src/start.ts
@@ -9,9 +9,9 @@ export * from './config.js';
 
 export function createAndStartTelemetryClient(config: TelemetryClientConfig): TelemetryClient {
   const log = createDebugLogger('aztec:telemetry-client');
-  if (config.collectorBaseUrl) {
+  if (config.metricsCollectorUrl) {
     log.info('Using OpenTelemetry client');
-    return OpenTelemetryClient.createAndStart(config.collectorBaseUrl, log);
+    return OpenTelemetryClient.createAndStart(config.metricsCollectorUrl, config.tracesCollectorUrl, log);
   } else {
     log.info('Using NoopTelemetryClient');
     return new NoopTelemetryClient();