fix(server): otel not working due to port conflicts after combining c…

…ontainers (#10078) fix: otel not working due to port conflicts after combining containers Fixes #9759
immich-app · Jun 10, 2024 · 2f2aecf · 2f2aecf
1 parent 20efd82
commit 2f2aecf
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 27 deletions.
diff --git a/docker/prometheus.yml b/docker/prometheus.yml
@@ -3,10 +3,10 @@ global:
   evaluation_interval: 15s
 
 scrape_configs:
-  - job_name: immich_server
+  - job_name: immich_api
     static_configs:
       - targets: ['immich-server:8081']
-  
+
   - job_name: immich_microservices
     static_configs:
-      - targets: ['immich-microservices:8081']
+      - targets: ['immich-server:8082']
diff --git a/server/src/config.ts b/server/src/config.ts
@@ -374,7 +374,8 @@ export const immichAppConfig: ConfigModuleOptions = {
     DB_SKIP_MIGRATIONS: Joi.boolean().optional().default(false),
 
     IMMICH_PORT: Joi.number().optional(),
-    IMMICH_METRICS_PORT: Joi.number().optional(),
+    IMMICH_API_METRICS_PORT: Joi.number().optional(),
+    IMMICH_MICROSERVICES_METRICS_PORT: Joi.number().optional(),
 
     IMMICH_METRICS: Joi.boolean().optional().default(false),
     IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),

diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.ts
@@ -17,7 +17,7 @@ import { StorageService } from 'src/services/storage.service';
 import { SystemConfigService } from 'src/services/system-config.service';
 import { UserService } from 'src/services/user.service';
 import { VersionService } from 'src/services/version.service';
-import { otelSDK } from 'src/utils/instrumentation';
+import { otelShutdown } from 'src/utils/instrumentation';
 
 @Injectable()
 export class MicroservicesService {
@@ -102,6 +102,6 @@ export class MicroservicesService {
   async teardown() {
     await this.libraryService.teardown();
     await this.metadataService.teardown();
-    await otelSDK.shutdown();
+    await otelShutdown();
   }
 }
diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts
@@ -33,23 +33,36 @@ const aggregation = new metrics.ExplicitBucketHistogramAggregation(
   true,
 );
 
-const metricsPort = Number.parseInt(process.env.IMMICH_METRICS_PORT ?? '8081');
-
-export const otelSDK = new NodeSDK({
-  resource: new resources.Resource({
-    [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
-    [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
-  }),
-  metricReader: new PrometheusExporter({ port: metricsPort }),
-  contextManager: new AsyncLocalStorageContextManager(),
-  instrumentations: [
-    new HttpInstrumentation(),
-    new IORedisInstrumentation(),
-    new NestInstrumentation(),
-    new PgInstrumentation(),
-  ],
-  views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
-});
+let otelSingleton: NodeSDK | undefined;
+
+export const otelStart = (port: number) => {
+  if (otelSingleton) {
+    throw new Error('OpenTelemetry SDK already started');
+  }
+  otelSingleton = new NodeSDK({
+    resource: new resources.Resource({
+      [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
+      [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
+    }),
+    metricReader: new PrometheusExporter({ port }),
+    contextManager: new AsyncLocalStorageContextManager(),
+    instrumentations: [
+      new HttpInstrumentation(),
+      new IORedisInstrumentation(),
+      new NestInstrumentation(),
+      new PgInstrumentation(),
+    ],
+    views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
+  });
+  otelSingleton.start();
+};
+
+export const otelShutdown = async () => {
+  if (otelSingleton) {
+    await otelSingleton.shutdown();
+    otelSingleton = undefined;
+  }
+};
 
 export const otelConfig: OpenTelemetryModuleOptions = {
   metrics: {

diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts
@@ -9,14 +9,16 @@ import { envName, excludePaths, isDev, serverVersion, WEB_ROOT } from 'src/const
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
 import { ApiService } from 'src/services/api.service';
-import { otelSDK } from 'src/utils/instrumentation';
+import { otelStart } from 'src/utils/instrumentation';
 import { useSwagger } from 'src/utils/misc';
 
 const host = process.env.HOST;
 
 async function bootstrap() {
   process.title = 'immich-api';
-  otelSDK.start();
+  const otelPort = Number.parseInt(process.env.IMMICH_API_METRICS_PORT ?? '8081');
+
+  otelStart(otelPort);
 
   const port = Number(process.env.IMMICH_PORT) || 3001;
   const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });

diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts
@@ -4,10 +4,12 @@ import { MicroservicesModule } from 'src/app.module';
 import { envName, serverVersion } from 'src/constants';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
-import { otelSDK } from 'src/utils/instrumentation';
+import { otelStart } from 'src/utils/instrumentation';
 
 export async function bootstrap() {
-  otelSDK.start();
+  const otelPort = Number.parseInt(process.env.IMMICH_MICROSERVICES_METRICS_PORT ?? '8082');
+
+  otelStart(otelPort);
 
   const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });
   const logger = await app.resolve(ILoggerRepository);