From a0b8cb6a5e162f89914798ed811b8d494a2e924e Mon Sep 17 00:00:00 2001 From: Jamie King Date: Fri, 22 Sep 2023 11:17:27 -0700 Subject: [PATCH] feat(logging): enable sending logs to OpenTelemetry (#1097) --- __tests__/integration/helpers/moduleMap.js | 2 +- __tests__/server/config/env/runTime.spec.js | 108 ++++-- __tests__/server/index.spec.js | 2 +- .../server/plugins/reactHtml/index.spec.jsx | 2 +- __tests__/server/shutdown.spec.js | 2 +- __tests__/server/ssrServer-requests.spec.js | 2 +- __tests__/server/ssrServer.spec.js | 2 +- .../server/utils/createCircuitBreaker.spec.js | 2 +- __tests__/server/utils/devCdnFactory.spec.js | 2 +- __tests__/server/utils/heapdump.spec.js | 2 +- .../server/utils/logging/config/otel.spec.js | 141 ++++++++ .../utils/logging/config/production.spec.js | 2 +- __tests__/server/utils/logging/logger.spec.js | 112 +++++-- __tests__/server/utils/onModuleLoad.spec.jsx | 2 +- __tests__/server/utils/pollModuleMap.spec.js | 2 +- .../server/utils/redirectAllowList.spec.js | 2 +- docs/api/server/Environment-Variables.md | 92 +++++ package-lock.json | 316 ++++++++++++++++++ package.json | 3 + src/server/config/env/runTime.js | 13 + src/server/plugins/healthCheck.js | 2 +- src/server/utils/heapdump.js | 2 +- src/server/utils/logging/config/base.js | 2 +- .../utils/logging/config/development.js | 2 +- src/server/utils/logging/config/otel.js | 87 +++++ src/server/utils/logging/config/production.js | 42 +-- src/server/utils/logging/logger.js | 36 +- src/server/utils/logging/setup.js | 1 - src/server/utils/logging/utils.js | 36 ++ 29 files changed, 901 insertions(+), 120 deletions(-) create mode 100644 __tests__/server/utils/logging/config/otel.spec.js create mode 100644 src/server/utils/logging/config/otel.js diff --git a/__tests__/integration/helpers/moduleMap.js b/__tests__/integration/helpers/moduleMap.js index 3ccca3896..3b9ac2cd2 100644 --- a/__tests__/integration/helpers/moduleMap.js +++ b/__tests__/integration/helpers/moduleMap.js @@ -16,7 +16,7 @@ const fs = require('fs-extra'); const { resolve, join } = require('path'); -const util = require('util'); +const util = require('node:util'); const childProcess = require('child_process'); const promisifiedExec = util.promisify(childProcess.exec); diff --git a/__tests__/server/config/env/runTime.spec.js b/__tests__/server/config/env/runTime.spec.js index 818a1eb19..9abd43640 100644 --- a/__tests__/server/config/env/runTime.spec.js +++ b/__tests__/server/config/env/runTime.spec.js @@ -14,6 +14,57 @@ * permissions and limitations under the License. */ +expect.extend({ + toValidateURL(input) { + let passNegativeCase; + let passPositiveCase; + try { + input('//example.com/path'); + input('/path'); + passNegativeCase = false; + } catch (e) { + passNegativeCase = true; + } + try { + input('https://example.com/path'); + passPositiveCase = true; + } catch (e) { + passPositiveCase = false; + } + return { + pass: passNegativeCase && passPositiveCase, + message: () => `${this.utils.matcherHint('toValidateURL', undefined, '') + }\n\nExpected function to validate input is a fetchable URL in Node`, + }; + }, + toValidatePositiveInteger(input) { + let passNegativeCase; + let passPositiveCase; + try { + input('string that evaluates to NaN'); + input('0'); + input('-6'); + input('4.3'); + passNegativeCase = false; + } catch (e) { + passNegativeCase = true; + } + try { + input('1'); + input('3001'); + input(undefined); + passPositiveCase = true; + } catch (e) { + passPositiveCase = false; + } + return { + pass: passNegativeCase && passPositiveCase, + message: () => `${this.utils.matcherHint('toValidatePositiveInteger', undefined, '') + }\n\nExpected function to validate input is a positive integer`, + }; + }, +}); + jest.mock('ip', () => ({ address: () => 'localhost', })); @@ -86,25 +137,6 @@ describe('runTime', () => { jest.resetAllMocks(); }); - function nodeUrl(entry) { - return () => { - expect(() => entry.validate('https://example.com/path')).not.toThrow(); - expect(() => entry.validate('/path')).toThrow(); - }; - } - - function positiveInteger(entry) { - return () => { - expect(() => entry.validate('1')).not.toThrow(); - expect(() => entry.validate('3001')).not.toThrow(); - expect(() => entry.validate(undefined)).not.toThrow(); - expect(() => entry.validate('string that evaluates to NaN')).toThrow(); - expect(() => entry.validate('0')).toThrow(); - expect(() => entry.validate('-6')).toThrow(); - expect(() => entry.validate('4.3')).toThrow(); - }; - } - it('has a name on every entry', () => { const runTime = require('../../../../src/server/config/env/runTime').default; runTime.forEach((entry) => { @@ -281,8 +313,9 @@ describe('runTime', () => { expect(holocronModuleMapPath.defaultValue()).not.toBeDefined(); }); - // eslint-disable-next-line jest/expect-expect -- assertion is in nodeUrl - it('ensures node can reach the URL', nodeUrl(holocronModuleMapPath)); + it('ensures node can reach the URL', () => { + expect(holocronModuleMapPath.validate).toValidateURL(); + }); it('should use port numbers specified via HTTP_ONE_APP_DEV_CDN_PORT', () => { process.env.NODE_ENV = 'development'; @@ -300,8 +333,9 @@ describe('runTime', () => { expect(holocronServerMaxModulesRetry.defaultValue).toBe(undefined); }); - // eslint-disable-next-line jest/expect-expect -- assertion is in positiveInteger - it('validates the value as a positive integer', positiveInteger(holocronServerMaxModulesRetry)); + it('validates the value as a positive integer', () => { + expect(holocronServerMaxModulesRetry.validate).toValidatePositiveInteger(); + }); }); describe('HOLOCRON_SERVER_MAX_SIM_MODULES_FETCH', () => { @@ -313,11 +347,9 @@ describe('runTime', () => { expect(holocronServerMaxSimModulesFetch.defaultValue).toBe(undefined); }); - // eslint-disable-next-line jest/expect-expect -- assertion is in positiveInteger - it( - 'validates the value as a positive integer', - positiveInteger(holocronServerMaxSimModulesFetch) - ); + it('validates the value as a positive integer', () => { + expect(holocronServerMaxSimModulesFetch.validate).toValidatePositiveInteger(); + }); }); describe('ONE_CLIENT_REPORTING_URL', () => { @@ -497,4 +529,24 @@ describe('runTime', () => { expect(() => postRequestMaxPayload.validate('20kb')).not.toThrow(); }); }); + + describe('OTEL_EXPORTER_OTLP_LOGS_ENDPOINT', () => { + const otelLogCollectorUrl = getEnvVarConfig('OTEL_EXPORTER_OTLP_LOGS_ENDPOINT'); + + it('ensures node can reach the URL', () => { + expect(otelLogCollectorUrl.validate).toValidateURL(); + }); + + it('is not required', () => { + expect(() => otelLogCollectorUrl.validate()).not.toThrow(); + }); + }); + + describe('OTEL_SERVICE_NAME', () => { + const otelServiceName = getEnvVarConfig('OTEL_SERVICE_NAME'); + + it('should have a default value of "One App"', () => { + expect(otelServiceName.defaultValue).toBe('One App'); + }); + }); }); diff --git a/__tests__/server/index.spec.js b/__tests__/server/index.spec.js index bbfc256a7..060180c94 100644 --- a/__tests__/server/index.spec.js +++ b/__tests__/server/index.spec.js @@ -15,7 +15,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import fs from 'fs'; import path from 'path'; diff --git a/__tests__/server/plugins/reactHtml/index.spec.jsx b/__tests__/server/plugins/reactHtml/index.spec.jsx index 07158875c..b33e8e8b1 100644 --- a/__tests__/server/plugins/reactHtml/index.spec.jsx +++ b/__tests__/server/plugins/reactHtml/index.spec.jsx @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import Fastify from 'fastify'; import { fromJS } from 'immutable'; import { setRequiredExternalsRegistry } from 'holocron'; diff --git a/__tests__/server/shutdown.spec.js b/__tests__/server/shutdown.spec.js index 7c1f8bcfd..1733451b0 100644 --- a/__tests__/server/shutdown.spec.js +++ b/__tests__/server/shutdown.spec.js @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; describe('shutdown', () => { jest.spyOn(global, 'setTimeout').mockImplementation(() => {}); diff --git a/__tests__/server/ssrServer-requests.spec.js b/__tests__/server/ssrServer-requests.spec.js index e82a1833d..909a7de09 100644 --- a/__tests__/server/ssrServer-requests.spec.js +++ b/__tests__/server/ssrServer-requests.spec.js @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import ssrServer from '../../src/server/ssrServer'; const { NODE_ENV } = process.env; diff --git a/__tests__/server/ssrServer.spec.js b/__tests__/server/ssrServer.spec.js index f9ee2ddd1..4ced03a43 100644 --- a/__tests__/server/ssrServer.spec.js +++ b/__tests__/server/ssrServer.spec.js @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import path from 'path'; import compress from '@fastify/compress'; import Fastify from 'fastify'; diff --git a/__tests__/server/utils/createCircuitBreaker.spec.js b/__tests__/server/utils/createCircuitBreaker.spec.js index aed6d0b3c..bfe6aa60f 100644 --- a/__tests__/server/utils/createCircuitBreaker.spec.js +++ b/__tests__/server/utils/createCircuitBreaker.spec.js @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import CircuitBreaker from 'opossum'; import { getModule } from 'holocron'; import createCircuitBreaker, { diff --git a/__tests__/server/utils/devCdnFactory.spec.js b/__tests__/server/utils/devCdnFactory.spec.js index 72b351009..619708bee 100644 --- a/__tests__/server/utils/devCdnFactory.spec.js +++ b/__tests__/server/utils/devCdnFactory.spec.js @@ -13,7 +13,7 @@ */ /* eslint-disable no-console -- console used in tests */ -import util from 'util'; +import util from 'node:util'; import fetch from 'node-fetch'; import fs from 'fs'; import rimraf from 'rimraf'; diff --git a/__tests__/server/utils/heapdump.spec.js b/__tests__/server/utils/heapdump.spec.js index c57e75424..30928e0e4 100644 --- a/__tests__/server/utils/heapdump.spec.js +++ b/__tests__/server/utils/heapdump.spec.js @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); diff --git a/__tests__/server/utils/logging/config/otel.spec.js b/__tests__/server/utils/logging/config/otel.spec.js new file mode 100644 index 000000000..63a6bd85a --- /dev/null +++ b/__tests__/server/utils/logging/config/otel.spec.js @@ -0,0 +1,141 @@ +/* + * Copyright 2023 American Express Travel Related Services Company, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import pino from 'pino'; +import otelConfig, { + createOtelTransport, +} from '../../../../../src/server/utils/logging/config/otel'; +import { serializeError } from '../../../../../src/server/utils/logging/utils'; + +jest.mock('../../../../../src/server/utils/logging/utils', () => ({ + serializeError: jest.fn(), + formatLogEntry: jest.fn((input) => ({ ...input, formatted: true })), +})); + +jest.mock('../../../../../src/server/utils/readJsonFile', () => () => ({ buildVersion: 'X.X.X' })); + +jest.mock('node:os', () => ({ + hostname: () => 'mockHostName', +})); + +jest.spyOn(pino, 'transport'); + +describe('OpenTelemetry logging', () => { + beforeEach(() => jest.clearAllMocks()); + + describe('config', () => { + it('should serialize errors', () => { + expect(otelConfig.serializers.err).toBe(serializeError); + }); + + it('should set base data', () => { + expect(otelConfig.base).toMatchInlineSnapshot(` + { + "schemaVersion": "1.0.0", + } + `); + }); + + it('should set the error key', () => { + expect(otelConfig.errorKey).toBe('error'); + }); + + it('should coerce "log" level to "info" level', () => { + expect(otelConfig.formatters.level('log', 35)).toEqual({ level: 30 }); + }); + + it('should not coerce other levels', () => { + expect(otelConfig.formatters.level('trace', 10)).toEqual({ level: 10 }); + expect(otelConfig.formatters.level('debug', 20)).toEqual({ level: 20 }); + expect(otelConfig.formatters.level('info', 30)).toEqual({ level: 30 }); + expect(otelConfig.formatters.level('warn', 40)).toEqual({ level: 40 }); + expect(otelConfig.formatters.level('error', 50)).toEqual({ level: 50 }); + }); + + it('should flatten log objects', () => { + const logEntry = { + foo: { + bar: 'baz', + }, + fizz: { + buzz: { + fizzbuzz: true, + }, + hello: 'world', + }, + }; + + expect(otelConfig.formatters.log(logEntry)).toMatchInlineSnapshot(` + { + "fizz.buzz.fizzbuzz": true, + "fizz.hello": "world", + "foo.bar": "baz", + "formatted": true, + } + `); + }); + }); + + describe('transport', () => { + process.env.OTEL_SERVICE_NAME = 'Mock Service Name'; + process.env.OTEL_SERVICE_NAMESPACE = 'Mock Service Namespace'; + + it('should include custom resource attributes', () => { + process.env.OTEL_RESOURCE_ATTRIBUTES = 'service.custom.id=XXXXX;deployment.env=qa'; + const transport = createOtelTransport(); + expect(pino.transport).toHaveBeenCalledTimes(1); + expect(pino.transport.mock.calls[0][0]).toMatchInlineSnapshot(` + { + "options": { + "resourceAttributes": { + "deployment.env": "qa", + "service.custom.id": "XXXXX", + "service.instance.id": "mockHostName", + "service.name": "Mock Service Name", + "service.namespace": "Mock Service Namespace", + "service.version": "X.X.X", + }, + }, + "target": "pino-opentelemetry-transport", + } + `); + expect(pino.transport.mock.results[0].value).toBe(transport); + }); + + it('should not throw if there are not custom resource attributes', () => { + delete process.env.OTEL_RESOURCE_ATTRIBUTES; + let transport; + expect(() => { + transport = createOtelTransport(); + }).not.toThrow(); + expect(pino.transport).toHaveBeenCalledTimes(1); + expect(pino.transport.mock.calls[0][0]).toMatchInlineSnapshot(` + { + "options": { + "resourceAttributes": { + "service.instance.id": "mockHostName", + "service.name": "Mock Service Name", + "service.namespace": "Mock Service Namespace", + "service.version": "X.X.X", + }, + }, + "target": "pino-opentelemetry-transport", + } + `); + expect(pino.transport.mock.results[0].value).toBe(transport); + }); + }); +}); diff --git a/__tests__/server/utils/logging/config/production.spec.js b/__tests__/server/utils/logging/config/production.spec.js index 9bd7416d0..ca66cba90 100644 --- a/__tests__/server/utils/logging/config/production.spec.js +++ b/__tests__/server/utils/logging/config/production.spec.js @@ -25,7 +25,7 @@ jest.mock('yargs', () => ({ }, })); -jest.mock('os', () => ({ +jest.mock('node:os', () => ({ hostname: () => 'mockHostname', type: () => 'mockType', arch: () => 'mockArch', diff --git a/__tests__/server/utils/logging/logger.spec.js b/__tests__/server/utils/logging/logger.spec.js index f8c8afe75..9201eefa6 100644 --- a/__tests__/server/utils/logging/logger.spec.js +++ b/__tests__/server/utils/logging/logger.spec.js @@ -13,7 +13,18 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ +import { EventEmitter } from 'events'; import deepmerge from 'deepmerge'; +import pino from 'pino'; +import { argv } from 'yargs'; +import exportedLogger, { createLogger } from '../../../../src/server/utils/logging/logger'; +import baseConfig from '../../../../src/server/utils/logging/config/base'; +import productionConfig from '../../../../src/server/utils/logging/config/production'; +import developmentStream from '../../../../src/server/utils/logging/config/development'; +import otelConfig, { createOtelTransport } from '../../../../src/server/utils/logging/config/otel'; + +jest.spyOn(pino, 'pino'); +jest.spyOn(pino, 'multistream'); jest.mock('yargs', () => ({ argv: { @@ -21,51 +32,86 @@ jest.mock('yargs', () => ({ }, })); -describe('logger', () => { - let logger; - let pino; - let productionConfig; - let baseConfig; - let developmentStream; +jest.mock('../../../../src/server/utils/logging/config/otel', () => { + const original = jest.requireActual('../../../../src/server/utils/logging/config/otel'); + const otelTransport = original.createOtelTransport(); + return { + default: original.otelConfig, + createOtelTransport: () => otelTransport, + }; +}); - function load(nodeEnv) { - jest.resetModules(); +describe('logger', () => { + const originalNodeEnv = process.env.NODE_ENV; - if (typeof nodeEnv === 'string') { - process.env.NODE_ENV = nodeEnv; - } else { - delete process.env.NODE_ENV; - } + beforeEach(() => { + delete process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; + process.env.NODE_ENV = 'production'; + argv.logLevel = 'debug'; + delete argv.logFormat; + jest.clearAllMocks(); + }); - jest.mock('pino', () => jest.fn(() => 'pino')); - pino = require('pino'); - baseConfig = require('../../../../src/server/utils/logging/config/base').default; - productionConfig = require('../../../../src/server/utils/logging/config/production').default; - developmentStream = require('../../../../src/server/utils/logging/config/development').default; - logger = require('../../../../src/server/utils/logging/logger').default; - } + afterAll(() => { + process.env.NODE_ENV = originalNodeEnv; + }); - it('is pino logger', () => { - load(); - expect(logger).toBe('pino'); + it('exports a pino logger', () => { + expect(exportedLogger).toBeInstanceOf(EventEmitter); + expect(Object.getPrototypeOf(exportedLogger).constructor.name).toBe('Pino'); }); it('uses the production formatter by default', () => { - load(); - expect(pino).toHaveBeenCalledTimes(1); - expect(pino).toHaveBeenCalledWith(deepmerge(baseConfig, productionConfig), undefined); + delete process.env.NODE_ENV; + const logger = createLogger(); + expect(pino.pino).toHaveBeenCalledTimes(1); + expect(pino.pino.mock.results[0].value).toBe(logger); + expect(pino.pino).toHaveBeenCalledWith(deepmerge(baseConfig, productionConfig), undefined); }); it('uses a development formatter when NODE_ENV is development', () => { - load('development'); - expect(pino).toHaveBeenCalledTimes(1); - expect(pino).toHaveBeenCalledWith(baseConfig, developmentStream); + process.env.NODE_ENV = 'development'; + const logger = createLogger(); + expect(pino.pino).toHaveBeenCalledTimes(1); + expect(pino.pino.mock.results[0].value).toBe(logger); + expect(pino.pino).toHaveBeenCalledWith(baseConfig, developmentStream); }); it('uses the production formatter when the log-format flag is set to machine', () => { - jest.mock('yargs', () => ({ argv: { logFormat: 'machine', logLevel: 'info' } })); - load(); - expect(pino).toHaveBeenCalledTimes(1); - expect(pino).toHaveBeenCalledWith(deepmerge(baseConfig, productionConfig), undefined); + argv.logLevel = 'info'; + argv.logFormat = 'machine'; + const logger = createLogger(); + expect(pino.pino).toHaveBeenCalledTimes(1); + expect(pino.pino.mock.results[0].value).toBe(logger); + expect(pino.pino).toHaveBeenCalledWith(deepmerge(baseConfig, productionConfig), undefined); + }); + + it('uses the OpenTelemetry config when OTEL_EXPORTER_OTLP_LOGS_ENDPOINT is set', () => { + process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = 'http://0.0.0.0:4317/v1/logs'; + const logger = createLogger(); + expect(pino.pino).toHaveBeenCalledTimes(1); + expect(pino.pino.mock.results[0].value).toBe(logger); + expect(pino.pino).toHaveBeenCalledWith( + deepmerge(baseConfig, otelConfig), + createOtelTransport() + ); + expect(pino.multistream).not.toHaveBeenCalled(); + }); + + it('uses both OpenTlemetry config and development config when OTEL_EXPORTER_OTLP_LOGS_ENDPOINT is set in development', () => { + process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = 'http://0.0.0.0:4317/v1/logs'; + process.env.NODE_ENV = 'development'; + const logger = createLogger(); + expect(pino.pino).toHaveBeenCalledTimes(1); + expect(pino.pino.mock.results[0].value).toBe(logger); + expect(pino.multistream).toHaveBeenCalledTimes(1); + expect(pino.multistream).toHaveBeenCalledWith([ + { stream: developmentStream }, + createOtelTransport(), + ]); + expect(pino.pino).toHaveBeenCalledWith( + deepmerge(baseConfig, otelConfig), + pino.multistream.mock.results[0].value + ); }); }); diff --git a/__tests__/server/utils/onModuleLoad.spec.jsx b/__tests__/server/utils/onModuleLoad.spec.jsx index 9035dad7c..380834705 100644 --- a/__tests__/server/utils/onModuleLoad.spec.jsx +++ b/__tests__/server/utils/onModuleLoad.spec.jsx @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import React from 'react'; import { preprocessEnvVar } from '@americanexpress/env-config-utils'; import { META_DATA_KEY } from '@americanexpress/one-app-bundler'; diff --git a/__tests__/server/utils/pollModuleMap.spec.js b/__tests__/server/utils/pollModuleMap.spec.js index 2ef68aac3..a9057b673 100644 --- a/__tests__/server/utils/pollModuleMap.spec.js +++ b/__tests__/server/utils/pollModuleMap.spec.js @@ -13,7 +13,7 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; jest.useFakeTimers(); jest.spyOn(global, 'setTimeout'); diff --git a/__tests__/server/utils/redirectAllowList.spec.js b/__tests__/server/utils/redirectAllowList.spec.js index 50bc2a856..abd0ddf53 100644 --- a/__tests__/server/utils/redirectAllowList.spec.js +++ b/__tests__/server/utils/redirectAllowList.spec.js @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import { setRedirectAllowList, isRedirectUrlAllowed, diff --git a/docs/api/server/Environment-Variables.md b/docs/api/server/Environment-Variables.md index e7bcf7f23..afa38be77 100644 --- a/docs/api/server/Environment-Variables.md +++ b/docs/api/server/Environment-Variables.md @@ -45,6 +45,11 @@ One App can be configured via Environment Variables: * [`ONE_MAP_POLLING_MIN`](#one_map_polling_min) * [`ONE_REFERRER_POLICY_OVERRIDE`](#one_referrer_policy_override) * [`ONE_SERVICE_WORKER`](#one_service_worker) +* OpenTelemetry + * [`OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`](#otel_log_collector_url) + * [`OTEL_SERVICE_NAME`](#otel_service_name) + * [`OTEL_SERVICE_NAMESPACE`](#otel_service_namespace) + * [`OTEL_RESOURCE_ATTRIBUTES`](#ote;_resource_attributes)
Alphabetical Contents @@ -75,6 +80,10 @@ One App can be configured via Environment Variables: * [`ONE_MAX_POST_REQUEST_PAYLOAD`](#one_max_post_request_payload) * [`ONE_REFERRER_POLICY_OVERRIDE`](#one_referrer_policy_override) * [`ONE_SERVICE_WORKER`](#one_service_worker) + * [`OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`](#otel_log_collector_url) + * [`OTEL_RESOURCE_ATTRIBUTES`](#otel_resource_attributes) + * [`OTEL_SERVICE_NAME`](#otel_service_name) + * [`OTEL_SERVICE_NAMESPACE`](#otel_service_namespace)
> ⚠️ = Required @@ -721,6 +730,87 @@ ONE_SERVICE_WORKER=false * [CLI Commands Documentation](./Cli-Commands.md) * [Module Map Documentation](./Module-Map-Schema.md) +## `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` + +**Runs In** +* ✅ Production +* ✅ Development + +When set, One App will emit OpenTelemetry logs over GRPC to the configured endpoint. +See the [OpenTlemetry documentation](https://opentelemetry.io/docs/concepts/sdk-configuration/otlp-exporter-configuration/#otel_exporter_otlp_logs_endpoint) for more information. + + +**Shape** +```bash +OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=String +``` + +**Example** +```bash +OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://localhost:4318/v1/logs +``` + +## `OTEL_SERVICE_NAME` + +**Runs In** +* ✅ Production +* ✅ Development + +Service name for OpenTelemtry resource. +See [OTel Environment Variable Specification] for more details + +**Shape** +```bash +OTEL_SERVICE_NAME=String +``` + +**Example** +```bash +OTEL_SERVICE_NAME=MyApplication +``` + +**Default Value** +```bash +OTEL_SERVICE_NAME="One App" +``` + +## `OTEL_SERVICE_NAMESPACE` + +**Runs In** +* ✅ Production +* ✅ Development + +Service namespace for OpenTelemtry resource. + +**Shape** +```bash +OTEL_SERVICE_NAMESPACE=String +``` + +**Example** +```bash +OTEL_SERVICE_NAMESPACE=MyApplicationNamespace +``` + +## `OTEL_RESOURCE_ATTRIBUTES` + +**Runs In** +* ✅ Production +* ✅ Development + +Additional OpenTelemetry resource attributes in [W3C Baggage format](https://w3c.github.io/baggage). +See OTel Environment Variable Specification] & [OTel Resource SDK documentation] for more details. + +**Shape** +```bash +OTEL_RESOURCE_ATTRIBUTES=String +``` + +**Example** +```bash +OTEL_RESOURCE_ATTRIBUTES="foo=bar;baz=qux" +``` + [☝️ Return To Top](#environment-variables) [One App Dev Proxy]: https://github.com/americanexpress/one-app-dev-proxy @@ -733,3 +823,5 @@ ONE_SERVICE_WORKER=false [`HTTPS_TRUSTED_CA_PATH`]: #https_trusted_ca_path [`HTTPS_PRIVATE_KEY_PASS_FILE_PATH`]: #https_private_key_pass_file_path [`HTTPS_PORT`]: #https_port +[OTel Environment Variable Specification]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ +[OTel Resource SDK documentation]: https://opentelemetry.io/docs/specs/otel/resource/sdk/#specifying-resource-information-via-an-environment-variable \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 6147bd30b..ec558fbc5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "@fastify/rate-limit": "^7.6.0", "@fastify/sensible": "^5.1.1", "@fastify/static": "^6.10.2", + "@opentelemetry/semantic-conventions": "^1.15.2", "abort-controller": "^3.0.0", "accepts": "^1.3.8", "body-parser": "^1.20.0", @@ -38,6 +39,7 @@ "fastify": "^4.18.0", "fastify-metrics": "^10.3.0", "fastify-plugin": "^4.2.0", + "flat": "^5.0.2", "helmet": "^7.0.0", "holocron": "^1.8.2", "holocron-module-route": "^1.7.0", @@ -51,6 +53,7 @@ "opossum-prometheus": "^0.3.0", "pidusage": "^3.0.2", "pino": "^8.14.1", + "pino-opentelemetry-transport": "^0.1.0", "prom-client": "^14.2.0", "prop-types": "^15.8.1", "react": "^17.0.2", @@ -3551,6 +3554,36 @@ "node": ">=10.0.0" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.9.0.tgz", + "integrity": "sha512-H8+iZh+kCE6VR/Krj6W28Y/ZlxoZ1fOzsNt77nrdE3knkbSelW1Uus192xOFCxHyeszLj8i4APQkSIXjAoOxXg==", + "dependencies": { + "@grpc/proto-loader": "^0.7.0", + "@types/node": ">=12.12.47" + }, + "engines": { + "node": "^8.13.0 || >=10.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.8", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.8.tgz", + "integrity": "sha512-GU12e2c8dmdXb7XUlOgYWZ2o2i+z9/VeACkxTA/zzAe2IjclC5PnVL0lpgjhrqfpDYHzM8B1TF6pqWegMYAzlA==", + "dependencies": { + "@types/long": "^4.0.1", + "lodash.camelcase": "^4.3.0", + "long": "^4.0.0", + "protobufjs": "^7.2.4", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@hapi/hoek": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", @@ -4741,6 +4774,232 @@ "@octokit/openapi-types": "^12.11.0" } }, + "node_modules/@opentelemetry/api": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", + "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.41.2.tgz", + "integrity": "sha512-JEV2RAqijAFdWeT6HddYymfnkiRu2ASxoTBr4WsnGJhOjWZkEy6vp+Sx9ozr1NaIODOa2HUyckExIqQjn6qywQ==", + "dependencies": { + "@opentelemetry/api": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/core": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.15.2.tgz", + "integrity": "sha512-+gBv15ta96WqkHZaPpcDHiaz0utiiHZVfm2YOYSqFGrUaJpPkMoSuLBB58YFQGi6Rsb9EHos84X6X5+9JspmLw==", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.15.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.5.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.41.2.tgz", + "integrity": "sha512-FCZqnVjqxh3FJenvlnaAqFtGSeYFtOzsT35c+QM7w7CnXI+Z9ipTtkI/v2Eb+TxKQQTl8bxFhUGNLSrNy3JsBg==", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.15.2", + "@opentelemetry/otlp-grpc-exporter-base": "0.41.2", + "@opentelemetry/otlp-transformer": "0.41.2", + "@opentelemetry/sdk-logs": "0.41.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.41.2.tgz", + "integrity": "sha512-pfwa6d+Dax3itZcGWiA0AoXeVaCuZbbqUTsCtOysd2re8C2PWXNxDONUfBWsn+KgxAdi+ljwTjJGiaVLDaIEvQ==", + "dependencies": { + "@opentelemetry/core": "1.15.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.41.2.tgz", + "integrity": "sha512-OErK8dYjXG01XIMIpmOV2SzL9ctkZ0Nyhf2UumICOAKtgLvR5dG1JMlsNVp8Jn0RzpsKc6Urv7JpP69wzRXN+A==", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.15.2", + "@opentelemetry/otlp-exporter-base": "0.41.2", + "protobufjs": "^7.2.3" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.41.2.tgz", + "integrity": "sha512-jJbPwB0tNu2v+Xi0c/v/R3YBLJKLonw1p+v3RVjT2VfzeUyzSp/tBeVdY7RZtL6dzZpA9XSmp8UEfWIFQo33yA==", + "dependencies": { + "@opentelemetry/api-logs": "0.41.2", + "@opentelemetry/core": "1.15.2", + "@opentelemetry/resources": "1.15.2", + "@opentelemetry/sdk-logs": "0.41.2", + "@opentelemetry/sdk-metrics": "1.15.2", + "@opentelemetry/sdk-trace-base": "1.15.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.5.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.15.2.tgz", + "integrity": "sha512-xmMRLenT9CXmm5HMbzpZ1hWhaUowQf8UB4jMjFlAxx1QzQcsD3KFNAVX/CAWzFPtllTyTplrA4JrQ7sCH3qmYw==", + "dependencies": { + "@opentelemetry/core": "1.15.2", + "@opentelemetry/semantic-conventions": "1.15.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.5.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.41.2.tgz", + "integrity": "sha512-smqKIw0tTW15waj7BAPHFomii5c3aHnSE4LQYTszGoK5P9nZs8tEAIpu15UBxi3aG31ZfsLmm4EUQkjckdlFrw==", + "dependencies": { + "@opentelemetry/core": "1.15.2", + "@opentelemetry/resources": "1.15.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.5.0", + "@opentelemetry/api-logs": ">=0.39.1" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.15.2.tgz", + "integrity": "sha512-9aIlcX8GnhcsAHW/Wl8bzk4ZnWTpNlLtud+fxUfBtFATu6OZ6TrGrF4JkT9EVrnoxwtPIDtjHdEsSjOqisY/iA==", + "dependencies": { + "@opentelemetry/core": "1.15.2", + "@opentelemetry/resources": "1.15.2", + "lodash.merge": "^4.6.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.5.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.15.2.tgz", + "integrity": "sha512-BEaxGZbWtvnSPchV98qqqqa96AOcb41pjgvhfzDij10tkBhIu9m0Jd6tZ1tJB5ZHfHbTffqYVYE0AOGobec/EQ==", + "dependencies": { + "@opentelemetry/core": "1.15.2", + "@opentelemetry/resources": "1.15.2", + "@opentelemetry/semantic-conventions": "1.15.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.5.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.15.2.tgz", + "integrity": "sha512-CjbOKwk2s+3xPIMcd5UNYQzsf+v94RczbdNix9/kQh38WiQkM90sUOi3if8eyHFgiBjBjhwXrA7W3ydiSQP9mw==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, "node_modules/@rollup/plugin-babel": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@rollup/plugin-babel/-/plugin-babel-6.0.3.tgz", @@ -5567,6 +5826,11 @@ "@types/node": "*" } }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, "node_modules/@types/minimist": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", @@ -12977,6 +13241,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "bin": { + "flat": "cli.js" + } + }, "node_modules/flat-cache": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", @@ -18244,6 +18516,11 @@ "integrity": "sha512-WpG9CcFAOjz/FtNht+QJeGpvVl/cdR6P0z6OcXSkr8wFJOsV2GRj2j10JLfjuA4aYkcKCNIEqRGCyTife9R8/g==", "dev": true }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -20322,6 +20599,17 @@ "node": ">= 10.x" } }, + "node_modules/pino-opentelemetry-transport": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/pino-opentelemetry-transport/-/pino-opentelemetry-transport-0.1.0.tgz", + "integrity": "sha512-MCPVdKPSXIaS/kY7NzGNK3zObuEDLR+sB+uy4TTAhdQ7iYcPhLy+pGF2gh82slkEUtBBCtd5Sl+39wyo8KOHag==", + "dependencies": { + "@opentelemetry/api-logs": "^0.41.0", + "@opentelemetry/exporter-logs-otlp-grpc": "^0.41.0", + "@opentelemetry/sdk-logs": "^0.41.0", + "pino-abstract-transport": "^1.0.0" + } + }, "node_modules/pino-pretty": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.1.0.tgz", @@ -21245,6 +21533,34 @@ "react-is": "^16.13.1" } }, + "node_modules/protobufjs": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", + "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/protobufjs/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", diff --git a/package.json b/package.json index 75a2ee4c2..7fda901fd 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "@fastify/rate-limit": "^7.6.0", "@fastify/sensible": "^5.1.1", "@fastify/static": "^6.10.2", + "@opentelemetry/semantic-conventions": "^1.15.2", "abort-controller": "^3.0.0", "accepts": "^1.3.8", "body-parser": "^1.20.0", @@ -109,6 +110,7 @@ "fastify": "^4.18.0", "fastify-metrics": "^10.3.0", "fastify-plugin": "^4.2.0", + "flat": "^5.0.2", "helmet": "^7.0.0", "holocron": "^1.8.2", "holocron-module-route": "^1.7.0", @@ -122,6 +124,7 @@ "opossum-prometheus": "^0.3.0", "pidusage": "^3.0.2", "pino": "^8.14.1", + "pino-opentelemetry-transport": "^0.1.0", "prom-client": "^14.2.0", "prop-types": "^15.8.1", "react": "^17.0.2", diff --git a/src/server/config/env/runTime.js b/src/server/config/env/runTime.js index c98a8859a..3a0c1d358 100644 --- a/src/server/config/env/runTime.js +++ b/src/server/config/env/runTime.js @@ -237,6 +237,19 @@ const runTime = [ } }, }, + // OpenTelemetry Configuration + { + name: 'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT', + validate: (input) => { + if (!input) return; + // eslint-disable-next-line no-new -- intentionally using new for side effect of validation + new URL(input); + }, + }, + { + name: 'OTEL_SERVICE_NAME', + defaultValue: 'One App', + }, ]; runTime.forEach(preprocessEnvVar); export { ip }; diff --git a/src/server/plugins/healthCheck.js b/src/server/plugins/healthCheck.js index c09a7a91b..f403a163c 100644 --- a/src/server/plugins/healthCheck.js +++ b/src/server/plugins/healthCheck.js @@ -24,7 +24,7 @@ migrated to Fastify import fp from 'fastify-plugin'; import pidusage from 'pidusage'; -import { promisify } from 'util'; +import { promisify } from 'node:util'; import { getModule } from 'holocron'; import { getClientStateConfig } from '../utils/stateConfig'; import { getModuleMapHealth } from '../utils/pollModuleMap'; diff --git a/src/server/utils/heapdump.js b/src/server/utils/heapdump.js index 9c24ea3a5..6174b37fd 100644 --- a/src/server/utils/heapdump.js +++ b/src/server/utils/heapdump.js @@ -16,7 +16,7 @@ import fs from 'fs'; import v8 from 'v8'; import { finished } from 'stream'; -import { promisify } from 'util'; +import { promisify } from 'node:util'; // Use `promisify(finished)` instead of importing from `stream/promises` for Node 12 compatibility // TODO: switch to import from `stream/promises` in v6.0.0 release when Node 12 support is dropped diff --git a/src/server/utils/logging/config/base.js b/src/server/utils/logging/config/base.js index bdd361f3f..9802c25fd 100644 --- a/src/server/utils/logging/config/base.js +++ b/src/server/utils/logging/config/base.js @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -import util from 'util'; +import util from 'node:util'; import { argv } from 'yargs'; export default { diff --git a/src/server/utils/logging/config/development.js b/src/server/utils/logging/config/development.js index fd8c1e2be..05f530f97 100644 --- a/src/server/utils/logging/config/development.js +++ b/src/server/utils/logging/config/development.js @@ -23,7 +23,7 @@ import { } from '../utils'; export const pinoPrettyOptions = { - ignore: 'pid,hostname,time,type,request', + ignore: 'pid,hostname,time,type,request,schemaVersion', // TODO: Uncomment once pino bug is resolved // https://github.com/pinojs/pino/issues/1790 // messageKey: 'message', diff --git a/src/server/utils/logging/config/otel.js b/src/server/utils/logging/config/otel.js new file mode 100644 index 000000000..616740570 --- /dev/null +++ b/src/server/utils/logging/config/otel.js @@ -0,0 +1,87 @@ +/* + * Copyright 2023 American Express Travel Related Services Company, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import os from 'node:os'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +// import { registerInstrumentations } from '@opentelemetry/instrumentation'; +// import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +// import { FastifyInstrumentation } from '@opentelemetry/instrumentation-fastify'; +// import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import pino from 'pino'; +import flatten from 'flat'; +import { + serializeError, + formatLogEntry, +} from '../utils'; +import readJsonFile from '../../readJsonFile'; + +const { buildVersion: version } = readJsonFile('../../../.build-meta.json'); + +// TODO: enable Fastify instrumentation once https://github.com/open-telemetry/opentelemetry-js-contrib/issues/1275 +// is resolved. Depends on https://github.com/fastify/fastify/pull/4470 + +// if (process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT) { +// const provider = new NodeTracerProvider(); +// provider.register(); +// registerInstrumentations({ +// instrumentations: [ +// new HttpInstrumentation(), +// new FastifyInstrumentation(), +// ], +// }); +// } + +export function createOtelTransport() { + const customResourceAttributes = process.env.OTEL_RESOURCE_ATTRIBUTES ? process.env.OTEL_RESOURCE_ATTRIBUTES.split(';').reduce((acc, curr) => { + const [key, value] = curr.split('='); + return { + ...acc, + [key]: value, + }; + }, {}) : {}; + + return pino.transport({ + target: 'pino-opentelemetry-transport', + options: { + resourceAttributes: { + [SemanticResourceAttributes.SERVICE_NAME]: process.env.OTEL_SERVICE_NAME, + [SemanticResourceAttributes.SERVICE_NAMESPACE]: process.env.OTEL_SERVICE_NAMESPACE, + [SemanticResourceAttributes.SERVICE_INSTANCE_ID]: os.hostname(), + [SemanticResourceAttributes.SERVICE_VERSION]: version, + ...customResourceAttributes, + }, + }, + }); +} + +export default { + errorKey: 'error', + base: { + schemaVersion: '1.0.0', + }, + serializers: { + err: serializeError, + }, + formatters: { + level(label, number) { + return { level: number === 35 ? 30 : number }; + }, + log(entry) { + const formattedEntry = formatLogEntry(entry); + return flatten(formattedEntry); + }, + }, +}; diff --git a/src/server/utils/logging/config/production.js b/src/server/utils/logging/config/production.js index 60b9c9179..5deb08f78 100644 --- a/src/server/utils/logging/config/production.js +++ b/src/server/utils/logging/config/production.js @@ -14,7 +14,11 @@ * permissions and limitations under the License. */ -import os from 'os'; +import os from 'node:os'; +import { + serializeError, + formatLogEntry, +} from '../utils'; import readJsonFile from '../../readJsonFile'; const { buildVersion: version } = readJsonFile('../../../.build-meta.json'); @@ -36,13 +40,7 @@ export default { }, }, serializers: { - err(err) { - return { - name: err.name, - message: err.message || '', - stacktrace: err.stack || '', - }; - }, + err: serializeError, }, formatters: { level(label) { @@ -54,32 +52,6 @@ export default { }; return { level: nodeLevelToSchemaLevel[label] || label }; }, - log(entry) { - /* eslint-disable no-param-reassign */ - if (entry.error) { - if (entry.error.name === 'ClientReportedError') { - entry.device = { - agent: entry.error.userAgent, - }; - entry.request = { - address: { - uri: entry.error.uri, - }, - metaData: entry.error.metaData, - }; - } else if (entry.error.metaData) { - entry.metaData = entry.error.metaData; - } - } - - // TODO: this is required due to a pino bug in the hook, remove once resolved - // https://github.com/pinojs/pino/issues/1790 - if (entry.msg) { - entry.message = entry.msg; - delete entry.msg; - } - - return entry; - }, + log: formatLogEntry, }, }; diff --git a/src/server/utils/logging/logger.js b/src/server/utils/logging/logger.js index c356a44a9..f3d611aec 100644 --- a/src/server/utils/logging/logger.js +++ b/src/server/utils/logging/logger.js @@ -16,14 +16,38 @@ import deepmerge from 'deepmerge'; import { argv } from 'yargs'; -import pino from 'pino'; +import { pino, multistream } from 'pino'; import productionConfig from './config/production'; +import otelConfig, { + createOtelTransport, +} from './config/otel'; import baseConfig from './config/base'; -const useProductionConfig = !!(argv.logFormat === 'machine' || process.env.NODE_ENV !== 'development'); +export function createLogger() { + const useProductionConfig = !!(argv.logFormat === 'machine' || process.env.NODE_ENV !== 'development'); -// development-formatters should not be loaded in production -// eslint-disable-next-line global-require -const logger = pino(deepmerge(baseConfig, useProductionConfig ? productionConfig : {}), useProductionConfig ? undefined : require('./config/development').default); + let transport; -export default logger; + if (process.env.NODE_ENV === 'development' && process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT) { + // Temporary solution until https://github.com/Vunovati/pino-opentelemetry-transport/issues/20 is resolved + // eslint-disable-next-line global-require -- do not load development logger in production + transport = multistream([{ stream: require('./config/development').default }, createOtelTransport()]); + } else if (!useProductionConfig && !process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT) { + // eslint-disable-next-line global-require -- do not load development logger in production + transport = require('./config/development').default; + } else if (process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT) { + transport = createOtelTransport(); + } + + return pino( + deepmerge( + baseConfig, + process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT + ? otelConfig + : useProductionConfig && productionConfig + ), + transport + ); +} + +export default createLogger(); diff --git a/src/server/utils/logging/setup.js b/src/server/utils/logging/setup.js index 6681e34ac..aebfe64c8 100644 --- a/src/server/utils/logging/setup.js +++ b/src/server/utils/logging/setup.js @@ -15,7 +15,6 @@ */ import { monkeypatches } from '@americanexpress/lumberjack'; - import logger from './logger'; import { startTimer, measureTime } from './timing'; diff --git a/src/server/utils/logging/utils.js b/src/server/utils/logging/utils.js index ae28375d9..3c1eb6814 100644 --- a/src/server/utils/logging/utils.js +++ b/src/server/utils/logging/utils.js @@ -68,9 +68,45 @@ function printDurationTime(obj) { return chalk.black.bgRed(duration); } +const serializeError = (err) => ({ + name: err.name, + message: err.message || '', + stacktrace: err.stack || '', +}); + +function formatLogEntry(entry) { + /* eslint-disable no-param-reassign */ + if (entry.error) { + if (entry.error.name === 'ClientReportedError') { + entry.device = { + agent: entry.error.userAgent, + }; + entry.request = { + address: { + uri: entry.error.uri, + }, + metaData: entry.error.metaData, + }; + } else if (entry.error.metaData) { + entry.metaData = entry.error.metaData; + } + } + + // TODO: this is required due to a pino bug in the hook, remove once resolved + // https://github.com/pinojs/pino/issues/1790 + if (entry.msg) { + entry.message = entry.msg; + delete entry.msg; + } + + return entry; +} + export { coloredLevels, printStatusCode, printStatusMessage, printDurationTime, + serializeError, + formatLogEntry, };