From 7c09011f4ba8405769b7a03c3751ff4e03ed9cd1 Mon Sep 17 00:00:00 2001 From: Chris Richards Date: Tue, 22 Feb 2022 11:43:12 +0000 Subject: [PATCH 1/7] feat: instrumentation lambda in the AWS SDK --- .../package.json | 9 +- .../src/services/ServicesExtensions.ts | 2 + .../src/services/lambda.ts | 119 ++++++ .../test/lambda.test.ts | 390 ++++++++++++++++++ 4 files changed, 517 insertions(+), 3 deletions(-) create mode 100644 plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts create mode 100644 plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json b/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json index 67c650cc0b..9d98c40117 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json @@ -49,29 +49,32 @@ "dependencies": { "@opentelemetry/core": "^1.0.0", "@opentelemetry/instrumentation": "^0.27.0", + "@opentelemetry/propagation-utils": "^0.26.0", "@opentelemetry/semantic-conventions": "^1.0.0", - "@opentelemetry/propagation-utils": "^0.26.0" + "aws-lambda": "^1.0.7" }, "devDependencies": { "@aws-sdk/client-dynamodb": "3.37.0", + "@aws-sdk/client-lambda": "3.37.0", "@aws-sdk/client-s3": "3.37.0", "@aws-sdk/client-sqs": "3.37.0", "@aws-sdk/types": "3.37.0", "@opentelemetry/api": "1.0.1", + "@opentelemetry/contrib-test-utils": "^0.29.0", "@opentelemetry/sdk-trace-base": "1.0.1", + "@types/aws-lambda": "^8.10.92", "@types/mocha": "8.2.3", "@types/node": "16.11.21", "@types/sinon": "10.0.6", "aws-sdk": "2.1008.0", "eslint": "8.7.0", "expect": "27.4.2", + "gts": "3.1.0", "mocha": "7.2.0", "nock": "13.2.1", "nyc": "15.1.0", "rimraf": "3.0.2", "sinon": "13.0.1", - "gts": "3.1.0", - "@opentelemetry/contrib-test-utils": "^0.29.0", "test-all-versions": "5.0.1", "ts-mocha": "8.0.0", "typescript": "4.3.4" diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts index 6796783013..52ab59d886 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts @@ -23,6 +23,7 @@ import { } from '../types'; import { DynamodbServiceExtension } from './dynamodb'; import { SnsServiceExtension } from './sns'; +import { LambdaServiceExtension } from './lambda'; export class ServicesExtensions implements ServiceExtension { services: Map = new Map(); @@ -31,6 +32,7 @@ export class ServicesExtensions implements ServiceExtension { this.services.set('SQS', new SqsServiceExtension()); this.services.set('SNS', new SnsServiceExtension()); this.services.set('DynamoDB', new DynamodbServiceExtension()); + this.services.set('Lambda', new LambdaServiceExtension()); } requestPreSpanHook(request: NormalizedRequest): RequestMetadata { diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts new file mode 100644 index 0000000000..1de228b0cf --- /dev/null +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts @@ -0,0 +1,119 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Span, SpanKind, Tracer } from '@opentelemetry/api'; +import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; +import { + AwsSdkInstrumentationConfig, + NormalizedRequest, + NormalizedResponse, +} from '../types'; +import { RequestMetadata, ServiceExtension } from './ServiceExtension'; +import { TextMapSetter, context, propagation } from '@opentelemetry/api'; +import type { ClientContext } from 'aws-lambda'; + +export class LambdaServiceExtension implements ServiceExtension { + requestPreSpanHook(request: NormalizedRequest): RequestMetadata { + const functionName = this.extractFunctionName(request.commandInput); + + let spanAttributes = {}; + let spanName: string | undefined; + + switch (request.commandName) { + case 'Invoke': + spanAttributes = { + [SemanticAttributes.FAAS_INVOKED_NAME]: functionName, + [SemanticAttributes.FAAS_INVOKED_PROVIDER]: 'aws', + }; + if (request.region) { + spanAttributes = { + ...spanAttributes, + [SemanticAttributes.FAAS_INVOKED_REGION]: request.region, + }; + } + spanName = `${functionName} invoke`; + break; + } + return { + isIncoming: false, + spanAttributes, + spanKind: SpanKind.CLIENT, + spanName, + }; + } + + requestPostSpanHook = (request: NormalizedRequest) => { + switch (request.commandName) { + case 'Invoke': + { + request.commandInput = injectPropagationContext(request.commandInput); + } + break; + } + }; + + responseHook( + response: NormalizedResponse, + span: Span, + tracer: Tracer, + config: AwsSdkInstrumentationConfig + ) { + const operation = response.request.commandName; + + if (operation === 'Invoke') { + if (response.data && '$metadata' in response.data) { + span.setAttribute( + SemanticAttributes.FAAS_EXECUTION, + response.data['$metadata'].requestId + ); + } + } + } + + extractFunctionName = (commandInput: Record): string => { + return commandInput?.FunctionName; + }; +} + +class ContextSetter implements TextMapSetter> { + set(carrier: Record, key: string, value: string): void { + const parsedClientContext: ClientContext = JSON.parse( + carrier.ClientContext !== undefined + ? Buffer.from(carrier.ClientContext, 'base64').toString('utf8') + : '{"Custom":{}}' + ); + const updatedPayload = { + ...parsedClientContext, + Custom: { + ...parsedClientContext.Custom, + [key]: value, + }, + }; + const encodedPayload = Buffer.from(JSON.stringify(updatedPayload)).toString( + 'base64' + ); + if (encodedPayload.length <= 3583) { + carrier.ClientContext = encodedPayload; + } + } +} +const contextSetter = new ContextSetter(); + +const injectPropagationContext = ( + invocationRequest: Record +): Record => { + propagation.inject(context.active(), invocationRequest, contextSetter); + return invocationRequest; +}; diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts new file mode 100644 index 0000000000..b8ac382801 --- /dev/null +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts @@ -0,0 +1,390 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AwsInstrumentation } from '../src'; +import { + getTestSpans, + registerInstrumentationTesting, +} from '@opentelemetry/contrib-test-utils'; +registerInstrumentationTesting(new AwsInstrumentation()); + +import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; +import { SpanKind } from '@opentelemetry/api'; + +import { Lambda, InvocationType } from '@aws-sdk/client-lambda'; +import { ClientRequest } from 'http'; +import * as nock from 'nock'; +import * as expect from 'expect'; + +process.env.AWS_ACCESS_KEY_ID = 'testing'; +process.env.AWS_SECRET_ACCESS_KEY = 'testing'; +const region = 'us-east-2'; + +describe('Lambda', () => { + describe('Invoke', () => { + describe('Request span attributes', () => { + const getInvokedSpan = async (params: any) => { + const lambdaClient = new Lambda({ region }); + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(200, 'null'); + + await lambdaClient.invoke(params); + expect(getTestSpans().length).toBe(1); + const [span] = getTestSpans(); + return span; + }; + + it("should set the span name to the ' invoke'", async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect(span.name).toEqual(`${params.FunctionName} invoke`); + }); + + it('should set the span kind to CLIENT', async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect(span.kind).toEqual(SpanKind.CLIENT); + }); + + it('should set the FAAS invoked provider as AWS', async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect( + span.attributes[SemanticAttributes.FAAS_INVOKED_PROVIDER] + ).toEqual('aws'); + }); + + it('should add the function name as a semantic attribute', async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect(span.attributes[SemanticAttributes.FAAS_INVOKED_NAME]).toEqual( + 'ot-test-function-name' + ); + }); + }); + + describe('Context propagation', () => { + it('should propagate client context onto the ClientContext in the invoke payload', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + await lambdaClient.invoke(params); + + // Context propagation + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + expect(requestHeaders['x-amz-client-context']).toBeDefined(); + const clientContext = JSON.parse( + Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString() + ) as Record; + expect(clientContext.Custom).toHaveProperty('traceparent'); + }); + + it('should skip context propagation in the event it would push the ClientContext over 3583 bytes', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const existingClientContext = Buffer.from( + JSON.stringify({ + Custom: { + text: [...Array(2600)] + .map(x => String.fromCharCode(48 + Math.random() * 74)) + .join(''), + }, + }) + ).toString('base64'); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: existingClientContext, + }; + + await lambdaClient.invoke(params); + + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + expect(requestHeaders['x-amz-client-context']).toStrictEqual( + existingClientContext + ); + }); + + it('should maintain any existing custom fields in the client context', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: Buffer.from( + JSON.stringify({ + Custom: { + existing: 'data', + }, + }) + ).toString('base64'), + }; + + await lambdaClient.invoke(params); + + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + const clientContext = JSON.parse( + Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString() + ) as Record; + expect(clientContext.Custom).toHaveProperty('existing', 'data'); + }); + + it('should maintain any existing top-level fields in the client context', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const clientContext = { + env: { + locale: 'en-US', + make: 'Nokia', + model: 'N95', + platform: 'Symbian', + platformVersion: '9.2', + }, + Custom: { + existing: 'data', + }, + }; + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: Buffer.from(JSON.stringify(clientContext)).toString( + 'base64' + ), + }; + + await lambdaClient.invoke(params); + + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + const updatedClientContext = JSON.parse( + Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString() + ) as Record; + expect(updatedClientContext.env).toStrictEqual(clientContext.env); + }); + + // It probably should be valid JSON, and I'm not sure what the lambda internals make of it if + // it isn't base64 encoded JSON, however there's absolutely nothing stopping an invoker passing + // absolute garbage in + it('should abandon context propagation if the existing client context is not valid JSON', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const clientContextContent = [...Array(16)] + .map(x => String.fromCharCode(48 + Math.random() * 74)) + .join(''); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: Buffer.from(clientContextContent).toString('base64'), + }; + + await lambdaClient.invoke(params); + + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + const clientContext = Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString(); + expect(clientContext).toStrictEqual(clientContextContent); + + // We still want span attributes though! + expect(getTestSpans().length).toBe(1); + const [span] = getTestSpans(); + + expect(span.kind).toEqual(SpanKind.CLIENT); + expect(span.attributes[SemanticAttributes.FAAS_INVOKED_NAME]).toEqual( + 'ot-test-function-name' + ); + expect( + span.attributes[SemanticAttributes.FAAS_INVOKED_PROVIDER] + ).toEqual('aws'); + }); + }); + + it('should add the request ID from the response onto the span', async () => { + const lambdaClient = new Lambda({ region }); + + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply((uri, requestBody, callback) => { + callback(null, [ + 200, + 'null', + { + 'x-amz-executed-version': '$LATEST', + 'x-amzn-requestid': '95882c2b-3fd2-485d-ada3-9fcb1ca65459', + }, + ]); + }); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + await lambdaClient.invoke(params); + expect(getTestSpans().length).toBe(1); + const [span] = getTestSpans(); + + expect(span.attributes[SemanticAttributes.FAAS_EXECUTION]).toEqual( + '95882c2b-3fd2-485d-ada3-9fcb1ca65459' + ); + }); + }); +}); From 5281b38283e6263dd2c33b28d58f7a2dee1e0afb Mon Sep 17 00:00:00 2001 From: Chris Richards Date: Tue, 22 Feb 2022 11:59:35 +0000 Subject: [PATCH 2/7] doc: document aws lambda sdk attributes --- .../opentelemetry-instrumentation-aws-sdk/README.md | 1 + .../doc/lambda.md | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md b/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md index b84fb36ee0..53b16b8d8e 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md @@ -98,6 +98,7 @@ Specific service logic currently implemented for: - [SQS](./docs/sqs.md) - [SNS](./docs/sns.md) +- [Lambda](./docs/lambda.md) - DynamoDb ## Potential Side Effects diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md new file mode 100644 index 0000000000..437d933435 --- /dev/null +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md @@ -0,0 +1,12 @@ +# Lambda + +Lambda is Amazon's function-as-a-service (FaaS) platform. Thus, it should follow the [OpenTelemetry specification for FaaS systems](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md). + +## Specific trace semantics + +The following methods are automatically enhanced: + +### invoke + +- [Outgoing Attributes](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md#outgoing-invocations) are added by this instrumentation according to the spec. +- OpenTelemetry trace context is injected into the `ClientContext` parameter, allowing functions to extract this using the `Custom` property within the function. From cb505ee621b7d22bbf3aef3de3f71df0e1c9a2be Mon Sep 17 00:00:00 2001 From: Chris Richards Date: Thu, 24 Feb 2022 13:13:31 +0000 Subject: [PATCH 3/7] fix: changes requested to pr #916 --- .../package.json | 4 +- .../src/aws-sdk.ts | 5 +- .../src/services/lambda.ts | 80 +++++++++++-------- .../src/types.ts | 1 + 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json b/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json index ce17674500..d0b2c0299a 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json @@ -49,8 +49,7 @@ "@opentelemetry/core": "^1.0.0", "@opentelemetry/instrumentation": "^0.27.0", "@opentelemetry/propagation-utils": "^0.26.0", - "@opentelemetry/semantic-conventions": "^1.0.0", - "aws-lambda": "^1.0.7" + "@opentelemetry/semantic-conventions": "^1.0.0" }, "devDependencies": { "@aws-sdk/client-dynamodb": "3.37.0", @@ -61,7 +60,6 @@ "@opentelemetry/api": "1.0.1", "@opentelemetry/contrib-test-utils": "^0.29.0", "@opentelemetry/sdk-trace-base": "1.0.1", - "@types/aws-lambda": "^8.10.92", "@types/mocha": "8.2.3", "@types/node": "16.11.21", "@types/sinon": "10.0.6", diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts index 5d9eddaf8a..d32cfad4b7 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts @@ -311,9 +311,11 @@ export class AwsInstrumentation extends InstrumentationBase { } delete v2Request[REQUEST_SPAN_KEY]; + const requestId = response.requestId; const normalizedResponse: NormalizedResponse = { data: response.data, request: normalizedRequest, + requestId: requestId, }; self._callUserResponseHook(span, normalizedResponse); @@ -328,7 +330,7 @@ export class AwsInstrumentation extends InstrumentationBase { ); } - span.setAttribute(AttributeNames.AWS_REQUEST_ID, response.requestId); + span.setAttribute(AttributeNames.AWS_REQUEST_ID, requestId); const httpStatusCode = response.httpResponse?.statusCode; if (httpStatusCode) { @@ -503,6 +505,7 @@ export class AwsInstrumentation extends InstrumentationBase { const normalizedResponse: NormalizedResponse = { data: response.output, request: normalizedRequest, + requestId: requestId, }; self.servicesExtensions.responseHook( normalizedResponse, diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts index 1de228b0cf..56d18ccd85 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Span, SpanKind, Tracer } from '@opentelemetry/api'; +import { + Span, + SpanKind, + Tracer, + diag, + SpanAttributes, +} from '@opentelemetry/api'; import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; import { AwsSdkInstrumentationConfig, @@ -22,13 +28,13 @@ import { } from '../types'; import { RequestMetadata, ServiceExtension } from './ServiceExtension'; import { TextMapSetter, context, propagation } from '@opentelemetry/api'; -import type { ClientContext } from 'aws-lambda'; +import { InvocationRequest } from '@aws-sdk/client-lambda'; export class LambdaServiceExtension implements ServiceExtension { requestPreSpanHook(request: NormalizedRequest): RequestMetadata { const functionName = this.extractFunctionName(request.commandInput); - let spanAttributes = {}; + let spanAttributes: SpanAttributes = {}; let spanName: string | undefined; switch (request.commandName) { @@ -38,10 +44,8 @@ export class LambdaServiceExtension implements ServiceExtension { [SemanticAttributes.FAAS_INVOKED_PROVIDER]: 'aws', }; if (request.region) { - spanAttributes = { - ...spanAttributes, - [SemanticAttributes.FAAS_INVOKED_REGION]: request.region, - }; + spanAttributes[SemanticAttributes.FAAS_INVOKED_REGION] = + request.region; } spanName = `${functionName} invoke`; break; @@ -58,7 +62,9 @@ export class LambdaServiceExtension implements ServiceExtension { switch (request.commandName) { case 'Invoke': { - request.commandInput = injectPropagationContext(request.commandInput); + request.commandInput = injectPropagationContext( + request.commandInput as InvocationRequest + ); } break; } @@ -73,12 +79,7 @@ export class LambdaServiceExtension implements ServiceExtension { const operation = response.request.commandName; if (operation === 'Invoke') { - if (response.data && '$metadata' in response.data) { - span.setAttribute( - SemanticAttributes.FAAS_EXECUTION, - response.data['$metadata'].requestId - ); - } + span.setAttribute(SemanticAttributes.FAAS_EXECUTION, response.requestId); } } @@ -86,34 +87,47 @@ export class LambdaServiceExtension implements ServiceExtension { return commandInput?.FunctionName; }; } - class ContextSetter implements TextMapSetter> { set(carrier: Record, key: string, value: string): void { - const parsedClientContext: ClientContext = JSON.parse( - carrier.ClientContext !== undefined - ? Buffer.from(carrier.ClientContext, 'base64').toString('utf8') - : '{"Custom":{}}' - ); - const updatedPayload = { - ...parsedClientContext, - Custom: { - ...parsedClientContext.Custom, - [key]: value, - }, - }; - const encodedPayload = Buffer.from(JSON.stringify(updatedPayload)).toString( - 'base64' - ); - if (encodedPayload.length <= 3583) { + try { + const parsedClientContext = carrier.ClientContext + ? JSON.parse( + Buffer.from(carrier.ClientContext, 'base64').toString('utf8') + ) + : {}; + const updatedPayload = { + ...parsedClientContext, + Custom: { + ...parsedClientContext.Custom, + [key]: value, + }, + }; + const encodedPayload = Buffer.from( + JSON.stringify(updatedPayload) + ).toString('base64'); + + // The length of client context is capped at 3583 bytes of base64 encoded data + // (https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html#API_Invoke_RequestSyntax) + if (encodedPayload.length > 3583) { + diag.warn( + 'lambda instrumentation: cannot set context propagation on lambda invoke parameters due to ClientContext length limitations.' + ); + return; + } carrier.ClientContext = encodedPayload; + } catch (e) { + diag.debug( + 'lambda instrumentation: failed to set context propagation on ClientContext', + e + ); } } } const contextSetter = new ContextSetter(); const injectPropagationContext = ( - invocationRequest: Record -): Record => { + invocationRequest: InvocationRequest +): InvocationRequest => { propagation.inject(context.active(), invocationRequest, contextSetter); return invocationRequest; }; diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts index 340be8deeb..0941018347 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts @@ -31,6 +31,7 @@ export interface NormalizedRequest { export interface NormalizedResponse { data: any; request: NormalizedRequest; + requestId: string; } export interface AwsSdkRequestHookInformation { From 6fb0c9e1e22c467b57c9c588ca6f350fa80e3794 Mon Sep 17 00:00:00 2001 From: chrisrichardsevergreen <95699748+chrisrichardsevergreen@users.noreply.github.com> Date: Tue, 8 Mar 2022 09:37:52 +0000 Subject: [PATCH 4/7] Update plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md Co-authored-by: Nathaniel Ruiz Nowell --- .../node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md index 437d933435..03e71b1a66 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md @@ -8,5 +8,5 @@ The following methods are automatically enhanced: ### invoke -- [Outgoing Attributes](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md#outgoing-invocations) are added by this instrumentation according to the spec. +- Attributes are added by this instrumentation according to the [spec for Outgoing Invocations of a FaaS from a client](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md#outgoing-invocations) . - OpenTelemetry trace context is injected into the `ClientContext` parameter, allowing functions to extract this using the `Custom` property within the function. From cd4a1745d210b4b3822ac4bc5624fb38d9a235f8 Mon Sep 17 00:00:00 2001 From: chrisrichardsevergreen <95699748+chrisrichardsevergreen@users.noreply.github.com> Date: Tue, 8 Mar 2022 09:37:59 +0000 Subject: [PATCH 5/7] Update plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md Co-authored-by: Nathaniel Ruiz Nowell --- .../node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md index 03e71b1a66..9e0dc46697 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md @@ -1,6 +1,6 @@ # Lambda -Lambda is Amazon's function-as-a-service (FaaS) platform. Thus, it should follow the [OpenTelemetry specification for FaaS systems](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md). +Lambda is Amazon's function-as-a-service (FaaS) platform. This instrumentation follows the [OpenTelemetry specification for FaaS systems](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md). ## Specific trace semantics From 2743961bbce82ea7a9eeed367327165fd1c8018e Mon Sep 17 00:00:00 2001 From: chrisrichardsevergreen <95699748+chrisrichardsevergreen@users.noreply.github.com> Date: Tue, 8 Mar 2022 09:44:05 +0000 Subject: [PATCH 6/7] Update plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md Co-authored-by: Nathaniel Ruiz Nowell --- .../node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md index 9e0dc46697..dbccf3365f 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md @@ -6,7 +6,7 @@ Lambda is Amazon's function-as-a-service (FaaS) platform. This instrumentation f The following methods are automatically enhanced: -### invoke +### Invoke - Attributes are added by this instrumentation according to the [spec for Outgoing Invocations of a FaaS from a client](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md#outgoing-invocations) . - OpenTelemetry trace context is injected into the `ClientContext` parameter, allowing functions to extract this using the `Custom` property within the function. From 8d545ea687156102025c6a515bfd399048d30456 Mon Sep 17 00:00:00 2001 From: Chris Richards Date: Tue, 8 Mar 2022 10:02:29 +0000 Subject: [PATCH 7/7] fix(aws-sdk): update context propagation as per comments on pr 916 --- .../src/services/lambda.ts | 108 ++++++++++-------- .../test/lambda.test.ts | 5 +- 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts index 56d18ccd85..35a9d4a8c5 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts @@ -27,8 +27,11 @@ import { NormalizedResponse, } from '../types'; import { RequestMetadata, ServiceExtension } from './ServiceExtension'; -import { TextMapSetter, context, propagation } from '@opentelemetry/api'; -import { InvocationRequest } from '@aws-sdk/client-lambda'; +import { context, propagation } from '@opentelemetry/api'; + +class LambdaCommands { + public static readonly Invoke: string = 'Invoke'; +} export class LambdaServiceExtension implements ServiceExtension { requestPreSpanHook(request: NormalizedRequest): RequestMetadata { @@ -47,7 +50,7 @@ export class LambdaServiceExtension implements ServiceExtension { spanAttributes[SemanticAttributes.FAAS_INVOKED_REGION] = request.region; } - spanName = `${functionName} invoke`; + spanName = `${functionName} ${LambdaCommands.Invoke}`; break; } return { @@ -60,11 +63,13 @@ export class LambdaServiceExtension implements ServiceExtension { requestPostSpanHook = (request: NormalizedRequest) => { switch (request.commandName) { - case 'Invoke': + case LambdaCommands.Invoke: { - request.commandInput = injectPropagationContext( - request.commandInput as InvocationRequest - ); + if (request.commandInput) { + request.commandInput.ClientContext = injectLambdaPropagationContext( + request.commandInput.ClientContext + ); + } } break; } @@ -76,10 +81,15 @@ export class LambdaServiceExtension implements ServiceExtension { tracer: Tracer, config: AwsSdkInstrumentationConfig ) { - const operation = response.request.commandName; - - if (operation === 'Invoke') { - span.setAttribute(SemanticAttributes.FAAS_EXECUTION, response.requestId); + switch (response.request.commandName) { + case LambdaCommands.Invoke: + { + span.setAttribute( + SemanticAttributes.FAAS_EXECUTION, + response.requestId + ); + } + break; } } @@ -87,47 +97,45 @@ export class LambdaServiceExtension implements ServiceExtension { return commandInput?.FunctionName; }; } -class ContextSetter implements TextMapSetter> { - set(carrier: Record, key: string, value: string): void { - try { - const parsedClientContext = carrier.ClientContext - ? JSON.parse( - Buffer.from(carrier.ClientContext, 'base64').toString('utf8') - ) - : {}; - const updatedPayload = { - ...parsedClientContext, - Custom: { - ...parsedClientContext.Custom, - [key]: value, - }, - }; - const encodedPayload = Buffer.from( - JSON.stringify(updatedPayload) - ).toString('base64'); - // The length of client context is capped at 3583 bytes of base64 encoded data - // (https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html#API_Invoke_RequestSyntax) - if (encodedPayload.length > 3583) { - diag.warn( - 'lambda instrumentation: cannot set context propagation on lambda invoke parameters due to ClientContext length limitations.' - ); - return; - } - carrier.ClientContext = encodedPayload; - } catch (e) { - diag.debug( - 'lambda instrumentation: failed to set context propagation on ClientContext', - e +const injectLambdaPropagationContext = ( + clientContext: string | undefined +): string | undefined => { + try { + const propagatedContext = {}; + propagation.inject(context.active(), propagatedContext); + + const parsedClientContext = clientContext + ? JSON.parse(Buffer.from(clientContext, 'base64').toString('utf8')) + : {}; + + const updatedClientContext = { + ...parsedClientContext, + Custom: { + ...parsedClientContext.Custom, + ...propagatedContext, + }, + }; + + const encodedClientContext = Buffer.from( + JSON.stringify(updatedClientContext) + ).toString('base64'); + + // The length of client context is capped at 3583 bytes of base64 encoded data + // (https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html#API_Invoke_RequestSyntax) + if (encodedClientContext.length > 3583) { + diag.warn( + 'lambda instrumentation: cannot set context propagation on lambda invoke parameters due to ClientContext length limitations.' ); + return clientContext; } - } -} -const contextSetter = new ContextSetter(); -const injectPropagationContext = ( - invocationRequest: InvocationRequest -): InvocationRequest => { - propagation.inject(context.active(), invocationRequest, contextSetter); - return invocationRequest; + return encodedClientContext; + } catch (e) { + diag.debug( + 'lambda instrumentation: failed to set context propagation on ClientContext', + e + ); + return clientContext; + } }; diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts index b8ac382801..372ebc4e67 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts @@ -60,7 +60,7 @@ describe('Lambda', () => { }; const span = await getInvokedSpan(params); - expect(span.name).toEqual(`${params.FunctionName} invoke`); + expect(span.name).toEqual(`${params.FunctionName} Invoke`); }); it('should set the span kind to CLIENT', async () => { @@ -241,6 +241,7 @@ describe('Lambda', () => { ).toString() ) as Record; expect(clientContext.Custom).toHaveProperty('existing', 'data'); + expect(clientContext.Custom).toHaveProperty('traceparent'); }); it('should maintain any existing top-level fields in the client context', async () => { @@ -294,6 +295,7 @@ describe('Lambda', () => { ).toString() ) as Record; expect(updatedClientContext.env).toStrictEqual(clientContext.env); + expect(updatedClientContext.Custom).toHaveProperty('traceparent'); }); // It probably should be valid JSON, and I'm not sure what the lambda internals make of it if @@ -331,6 +333,7 @@ describe('Lambda', () => { await lambdaClient.invoke(params); + // Keep whatever was there before expect(request).toBeDefined(); const requestHeaders = request!.headers; const clientContext = Buffer.from(