From b8029a05a59cbc492aa57914cc14930cb4871563 Mon Sep 17 00:00:00 2001 From: Florian CHAZAL Date: Mon, 3 Jan 2022 09:18:16 +0100 Subject: [PATCH 1/2] feat(metrics): rename method purgeStoredMetrics to publishStoredMetrics --- docs/core/metrics.md | 4 ++-- .../examples/decorator/manual-flushing.ts | 2 +- packages/metrics/examples/manual-flushing.ts | 2 +- packages/metrics/src/Metrics.ts | 17 +++++++++-------- packages/metrics/src/MetricsInterface.ts | 2 +- packages/metrics/src/middleware/middy.ts | 2 +- .../e2e/standardFunctions.test.MyFunction.ts | 2 +- packages/metrics/tests/unit/Metrics.test.ts | 4 ++-- 8 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/core/metrics.md b/docs/core/metrics.md index 9a9f906311..0092aa87f2 100644 --- a/docs/core/metrics.md +++ b/docs/core/metrics.md @@ -341,7 +341,7 @@ export class MyFunction { #### Manually -You can manually flush the metrics with `purgeStoredMetrics` as follows: +You can manually flush the metrics with `publishStoredMetrics` as follows: !!! warning Metrics, dimensions and namespace validation still applies. @@ -354,7 +354,7 @@ const metrics = new Metrics(); const lambdaHandler: Handler = async () => { metrics.addMetric('test-metric', MetricUnits.Count, 10); const metricsObject = metrics.serializeMetrics(); - metrics.purgeStoredMetrics(); + metrics.publishStoredMetrics(); console.log(JSON.stringify(metricsObject)); }; ``` diff --git a/packages/metrics/examples/decorator/manual-flushing.ts b/packages/metrics/examples/decorator/manual-flushing.ts index a0a9c8f26f..7e262956fa 100644 --- a/packages/metrics/examples/decorator/manual-flushing.ts +++ b/packages/metrics/examples/decorator/manual-flushing.ts @@ -15,7 +15,7 @@ const metrics = new Metrics(); const lambdaHandler: Handler = async () => { metrics.addMetric('test-metric', MetricUnits.Count, 10); - metrics.purgeStoredMetrics(); + metrics.publishStoredMetrics(); //Metrics will be published and cleared return { diff --git a/packages/metrics/examples/manual-flushing.ts b/packages/metrics/examples/manual-flushing.ts index 5c978930b4..657502b646 100644 --- a/packages/metrics/examples/manual-flushing.ts +++ b/packages/metrics/examples/manual-flushing.ts @@ -14,7 +14,7 @@ const metrics = new Metrics(); const lambdaHandler = async (): Promise => { metrics.addMetric('test-metric', MetricUnits.Count, 10); - metrics.purgeStoredMetrics(); + metrics.publishStoredMetrics(); }; const handlerWithMiddleware = middy(lambdaHandler) diff --git a/packages/metrics/src/Metrics.ts b/packages/metrics/src/Metrics.ts index 6a51589d37..8d286ff4fa 100644 --- a/packages/metrics/src/Metrics.ts +++ b/packages/metrics/src/Metrics.ts @@ -74,7 +74,7 @@ const DEFAULT_NAMESPACE = 'default_namespace'; * export const handler = async (_event: any, _context: any) => { * metrics.captureColdStart(); * metrics.addMetric('test-metric', MetricUnits.Count, 10); - * metrics.purgeStoredMetrics(); + * metrics.publishStoredMetrics(); * }; * ``` */ @@ -146,7 +146,7 @@ class Metrics implements MetricsInterface { */ public addMetric(name: string, unit: MetricUnit, value: number): void { this.storeMetric(name, unit, value); - if (this.isSingleMetric) this.purgeStoredMetrics(); + if (this.isSingleMetric) this.publishStoredMetrics(); } /** @@ -245,7 +245,7 @@ class Metrics implements MetricsInterface { return result; } finally { - this.purgeStoredMetrics(); + this.publishStoredMetrics(); } }; }; @@ -253,6 +253,7 @@ class Metrics implements MetricsInterface { /** * Synchronous function to actually publish your metrics. (Not needed if using logMetrics decorator). + * It will create a new EMF blob and log it to standard output to be then ingested by Cloudwatch logs and processed automatically for metrics creation. * * @example * @@ -263,11 +264,11 @@ class Metrics implements MetricsInterface { * * export const handler = async (_event: any, _context: any) => { * metrics.addMetric('test-metric', MetricUnits.Count, 10); - * metrics.purgeStoredMetrics(); + * metrics.publishStoredMetrics(); * }; * ``` */ - public purgeStoredMetrics(): void { + public publishStoredMetrics(): void { const target = this.serializeMetrics(); console.log(JSON.stringify(target)); this.storedMetrics = {}; @@ -286,7 +287,7 @@ class Metrics implements MetricsInterface { * * export const handler = async (event: any, context: Context) => { * metrics.raiseOnEmptyMetrics(); - * metrics.purgeStoredMetrics(); // will throw since no metrics added. + * metrics.publishStoredMetrics(); // will throw since no metrics added. * } * ``` */ @@ -357,7 +358,7 @@ class Metrics implements MetricsInterface { /** * CloudWatch EMF uses the same dimensions across all your metrics. Use singleMetric if you have a metric that should have different dimensions. * - * You don't need to call purgeStoredMetrics() after calling addMetric for a singleMetrics, they will be flushed directly. + * You don't need to call publishStoredMetrics() after calling addMetric for a singleMetrics, they will be flushed directly. * * @example * @@ -428,7 +429,7 @@ class Metrics implements MetricsInterface { private storeMetric(name: string, unit: MetricUnit, value: number): void { if (Object.keys(this.storedMetrics).length >= MAX_METRICS_SIZE) { - this.purgeStoredMetrics(); + this.publishStoredMetrics(); } this.storedMetrics[name] = { unit, diff --git a/packages/metrics/src/MetricsInterface.ts b/packages/metrics/src/MetricsInterface.ts index 00e26ba8aa..cda2fd577e 100644 --- a/packages/metrics/src/MetricsInterface.ts +++ b/packages/metrics/src/MetricsInterface.ts @@ -11,7 +11,7 @@ interface MetricsInterface { clearMetrics(): void clearDefaultDimensions(): void logMetrics(options?: MetricsOptions): HandlerMethodDecorator - purgeStoredMetrics(): void + publishStoredMetrics(): void serializeMetrics(): EmfOutput setDefaultDimensions(dimensions: Dimensions | undefined): void singleMetric(): Metrics diff --git a/packages/metrics/src/middleware/middy.ts b/packages/metrics/src/middleware/middy.ts index 6c9a077d3b..1c6948fd10 100644 --- a/packages/metrics/src/middleware/middy.ts +++ b/packages/metrics/src/middleware/middy.ts @@ -24,7 +24,7 @@ const logMetrics = (target: Metrics | Metrics[], options: ExtraOptions = {}): mi const logMetricsAfterOrError = async (): Promise => { metricsInstances.forEach((metrics: Metrics) => { - metrics.purgeStoredMetrics(); + metrics.publishStoredMetrics(); }); }; diff --git a/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts b/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts index a848e71416..c56fb4b785 100644 --- a/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts +++ b/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts @@ -32,6 +32,6 @@ export const handler = async (_event: unknown, _context: Context): Promise ); metricWithItsOwnDimensions.addMetric(singleMetricName, singleMetricUnit, singleMetricValue as number); - metrics.purgeStoredMetrics(); + metrics.publishStoredMetrics(); metrics.raiseOnEmptyMetrics(); }; diff --git a/packages/metrics/tests/unit/Metrics.test.ts b/packages/metrics/tests/unit/Metrics.test.ts index 2e348f39f5..46e4286bd5 100644 --- a/packages/metrics/tests/unit/Metrics.test.ts +++ b/packages/metrics/tests/unit/Metrics.test.ts @@ -378,7 +378,7 @@ describe('Class: Metrics', () => { const handler = async (_event: DummyEvent, _context: Context): Promise => { metrics.raiseOnEmptyMetrics(); // Logic goes here - metrics.purgeStoredMetrics(); + metrics.publishStoredMetrics(); }; try { @@ -474,7 +474,7 @@ describe('Class: Metrics', () => { _callback: Callback, ): void | Promise { metrics.addMetric('test_name_1', MetricUnits.Count, 1); - metrics.purgeStoredMetrics(); + metrics.publishStoredMetrics(); } } From 6ee93f1e483849c3db37c9b6cdc4bb86b73d14f2 Mon Sep 17 00:00:00 2001 From: Florian CHAZAL Date: Mon, 3 Jan 2022 10:43:08 +0100 Subject: [PATCH 2/2] add e2e tests for standard function --- .../e2e/standardFunctions.test.MyFunction.ts | 8 +- .../tests/e2e/standardFunctions.test.ts | 165 ++++++++++++------ 2 files changed, 116 insertions(+), 57 deletions(-) diff --git a/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts b/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts index c56fb4b785..cd95c83f27 100644 --- a/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts +++ b/packages/metrics/tests/e2e/standardFunctions.test.MyFunction.ts @@ -5,13 +5,13 @@ const namespace = process.env.EXPECTED_NAMESPACE ?? 'CDKExample'; const serviceName = process.env.EXPECTED_SERVICE_NAME ?? 'MyFunctionWithStandardHandler'; const metricName = process.env.EXPECTED_METRIC_NAME ?? 'MyMetric'; const metricUnit = (process.env.EXPECTED_METRIC_UNIT as MetricUnits) ?? MetricUnits.Count; -const metricValue = process.env.EXPECTED_METRIC_VALUE ?? 1; +const metricValue = process.env.EXPECTED_METRIC_VALUE ?? '1'; const defaultDimensions = process.env.EXPECTED_DEFAULT_DIMENSIONS ?? '{"MyDimension":"MyValue"}'; const extraDimension = process.env.EXPECTED_EXTRA_DIMENSION ?? '{"MyExtraDimension":"MyExtraValue"}'; const singleMetricDimension = process.env.EXPECTED_SINGLE_METRIC_DIMENSION ?? '{"MySingleMetricDim":"MySingleValue"}'; const singleMetricName = process.env.EXPECTED_SINGLE_METRIC_NAME ?? 'MySingleMetric'; const singleMetricUnit = (process.env.EXPECTED_SINGLE_METRIC_UNIT as MetricUnits) ?? MetricUnits.Percent; -const singleMetricValue = process.env.EXPECTED_SINGLE_METRIC_VALUE ?? 2; +const singleMetricValue = process.env.EXPECTED_SINGLE_METRIC_VALUE ?? '2'; const metrics = new Metrics({ namespace: namespace, service: serviceName }); @@ -19,7 +19,7 @@ export const handler = async (_event: unknown, _context: Context): Promise metrics.captureColdStartMetric(); metrics.raiseOnEmptyMetrics(); metrics.setDefaultDimensions(JSON.parse(defaultDimensions)); - metrics.addMetric(metricName, metricUnit, metricValue as number); + metrics.addMetric(metricName, metricUnit, parseInt(metricValue)); metrics.addDimension( Object.entries(JSON.parse(extraDimension))[0][0], Object.entries(JSON.parse(extraDimension))[0][1] as string, @@ -30,7 +30,7 @@ export const handler = async (_event: unknown, _context: Context): Promise Object.entries(JSON.parse(singleMetricDimension))[0][0], Object.entries(JSON.parse(singleMetricDimension))[0][1] as string, ); - metricWithItsOwnDimensions.addMetric(singleMetricName, singleMetricUnit, singleMetricValue as number); + metricWithItsOwnDimensions.addMetric(singleMetricName, singleMetricUnit, parseInt(singleMetricValue)); metrics.publishStoredMetrics(); metrics.raiseOnEmptyMetrics(); diff --git a/packages/metrics/tests/e2e/standardFunctions.test.ts b/packages/metrics/tests/e2e/standardFunctions.test.ts index 76b738501d..dd4688392a 100644 --- a/packages/metrics/tests/e2e/standardFunctions.test.ts +++ b/packages/metrics/tests/e2e/standardFunctions.test.ts @@ -2,9 +2,9 @@ // SPDX-License-Identifier: MIT-0 /** - * Test metrics decorator + * Test metrics standard functions * - * @group e2e/metrics/decorator + * @group e2e/metrics/standardFunctions */ import { randomUUID } from 'crypto'; @@ -20,44 +20,44 @@ const cloudwatchClient = new AWS.CloudWatch(); const lambdaClient = new AWS.Lambda(); const integTestApp = new App(); -const stack = new Stack(integTestApp, 'ExampleIntegTest'); +const stack = new Stack(integTestApp, 'MetricsE2EStandardFunctionsStack'); -describe('coldstart', () => { - it('can be deploy succcessfully', async () => { - // GIVEN - const startTime = new Date(); - const expectedNamespace = randomUUID(); // to easily find metrics back at assert phase - const expectedServiceName = 'MyFunctionWithStandardHandler'; - const expectedMetricName = 'MyMetric'; - const expectedMetricUnit = MetricUnits.Count; - const expectedMetricValue = '1'; - const expectedDefaultDimensions = { MyDimension: 'MyValue' }; - const expectedExtraDimension = { MyExtraDimension: 'MyExtraValue' }; - const expectedSingleMetricDimension = { MySingleMetricDim: 'MySingleValue' }; - const expectedSingleMetricName = 'MySingleMetric'; - const expectedSingleMetricUnit = MetricUnits.Percent; - const expectedSingleMetricValue = '2'; - const functionName = 'MyFunctionWithStandardHandler'; - new lambda.NodejsFunction(stack, 'MyFunction', { - functionName: functionName, - tracing: Tracing.ACTIVE, - environment: { - EXPECTED_NAMESPACE: expectedNamespace, - EXPECTED_SERVICE_NAME: expectedServiceName, - EXPECTED_METRIC_NAME: expectedMetricName, - EXPECTED_METRIC_UNIT: expectedMetricUnit, - EXPECTED_METRIC_VALUE: expectedMetricValue, - EXPECTED_DEFAULT_DIMENSIONS: JSON.stringify(expectedDefaultDimensions), - EXPECTED_EXTRA_DIMENSION: JSON.stringify(expectedExtraDimension), - EXPECTED_SINGLE_METRIC_DIMENSION: JSON.stringify(expectedSingleMetricDimension), - EXPECTED_SINGLE_METRIC_NAME: expectedSingleMetricName, - EXPECTED_SINGLE_METRIC_UNIT: expectedSingleMetricUnit, - EXPECTED_SINGLE_METRIC_VALUE: expectedSingleMetricValue, - }, - }); +// GIVEN +const startTime = new Date(); +const expectedNamespace = randomUUID(); // to easily find metrics back at assert phase +const expectedServiceName = 'MyFunctionWithStandardHandler'; +const expectedMetricName = 'MyMetric'; +const expectedMetricUnit = MetricUnits.Count; +const expectedMetricValue = '1'; +const expectedDefaultDimensions = { MyDimension: 'MyValue' }; +const expectedExtraDimension = { MyExtraDimension: 'MyExtraValue' }; +const expectedSingleMetricDimension = { MySingleMetricDim: 'MySingleValue' }; +const expectedSingleMetricName = 'MySingleMetric'; +const expectedSingleMetricUnit = MetricUnits.Percent; +const expectedSingleMetricValue = '2'; +const functionName = 'MyFunctionWithStandardHandler'; +new lambda.NodejsFunction(stack, 'MyFunction', { + functionName: functionName, + tracing: Tracing.ACTIVE, + environment: { + EXPECTED_NAMESPACE: expectedNamespace, + EXPECTED_SERVICE_NAME: expectedServiceName, + EXPECTED_METRIC_NAME: expectedMetricName, + EXPECTED_METRIC_UNIT: expectedMetricUnit, + EXPECTED_METRIC_VALUE: expectedMetricValue, + EXPECTED_DEFAULT_DIMENSIONS: JSON.stringify(expectedDefaultDimensions), + EXPECTED_EXTRA_DIMENSION: JSON.stringify(expectedExtraDimension), + EXPECTED_SINGLE_METRIC_DIMENSION: JSON.stringify(expectedSingleMetricDimension), + EXPECTED_SINGLE_METRIC_NAME: expectedSingleMetricName, + EXPECTED_SINGLE_METRIC_UNIT: expectedSingleMetricUnit, + EXPECTED_SINGLE_METRIC_VALUE: expectedSingleMetricValue, + }, +}); - const stackArtifact = integTestApp.synth().getStackByName(stack.stackName); +const stackArtifact = integTestApp.synth().getStackByName(stack.stackName); +describe('happy cases', () => { + beforeAll(async () => { const sdkProvider = await SdkProvider.withAwsCliCompatibleDefaults({ profile: process.env.AWS_PROFILE, }); @@ -68,7 +68,11 @@ describe('coldstart', () => { await cloudFormation.deployStack({ stack: stackArtifact, }); - // and invoked + }, 200000); + + it('capture ColdStart Metric', async () => { + // WHEN + // invoked await lambdaClient .invoke({ FunctionName: functionName, @@ -108,27 +112,82 @@ describe('coldstart', () => { MetricName: 'ColdStart', Statistics: ['Sum'], }, - undefined, + undefined ) .promise(); // Despite lambda has been called twice, coldstart metric sum should only be 1 const singleDataPoint = coldStartMetricStat.Datapoints ? coldStartMetricStat.Datapoints[0] : {}; expect(singleDataPoint.Sum).toBe(1); - }, 9000000); -}); + }, 15000); -afterAll(async () => { - if (!process.env.DISABLE_TEARDOWN) { - const stackArtifact = integTestApp.synth().getStackByName(stack.stackName); + it('produce added Metric with the default and extra one dimensions', async () => { + // GIVEN + const invocationCount = 2; - const sdkProvider = await SdkProvider.withAwsCliCompatibleDefaults({ - profile: process.env.AWS_PROFILE, - }); - const cloudFormation = new CloudFormationDeployments({ sdkProvider }); + // WHEN + // invoked + for (let i = 0; i < invocationCount; i++) { + await lambdaClient + .invoke({ + FunctionName: functionName, + }) + .promise(); + } - await cloudFormation.destroyStack({ - stack: stackArtifact, - }); - } -}, 9000000); + // THEN + // sleep to allow metrics to be collected + await new Promise((resolve) => setTimeout(resolve, 10000)); + + // Check metric dimensions + const metrics = await cloudwatchClient + .listMetrics({ + Namespace: expectedNamespace, + MetricName: expectedMetricName, + }) + .promise(); + expect(metrics.Metrics?.length).toBe(1); + const metric = metrics.Metrics?.[0]; + const expectedDimensions = [ + { Name: 'service', Value: expectedServiceName }, + { Name: Object.keys(expectedDefaultDimensions)[0], Value: expectedDefaultDimensions.MyDimension }, + { Name: Object.keys(expectedExtraDimension)[0], Value: expectedExtraDimension.MyExtraDimension }, + ]; + expect(metric?.Dimensions).toStrictEqual(expectedDimensions); + + // Check coldstart metric value + const metricStat = await cloudwatchClient + .getMetricStatistics( + { + Namespace: expectedNamespace, + StartTime: new Date(startTime.getTime() - 60 * 1000), // minus 1 minute, + Dimensions: expectedDimensions, + EndTime: new Date(new Date().getTime() + 60 * 1000), + Period: 60, + MetricName: expectedMetricName, + Statistics: ['Sum'], + }, + undefined + ) + .promise(); + + // Since lambda has been called twice in this test and potentially more in others, metric sum should be at least of expectedMetricValue * invocationCount + const singleDataPoint = metricStat.Datapoints ? metricStat.Datapoints[0] : {}; + expect(singleDataPoint.Sum).toBeGreaterThanOrEqual(parseInt(expectedMetricValue) * invocationCount); + }, 15000); + + afterAll(async () => { + if (!process.env.DISABLE_TEARDOWN) { + const stackArtifact = integTestApp.synth().getStackByName(stack.stackName); + + const sdkProvider = await SdkProvider.withAwsCliCompatibleDefaults({ + profile: process.env.AWS_PROFILE, + }); + const cloudFormation = new CloudFormationDeployments({ sdkProvider }); + + await cloudFormation.destroyStack({ + stack: stackArtifact, + }); + } + }, 200000); +});