Send metrics from Lambda nodeJS to Amazon Managed Prometheus using the Adot layer

[pmadc]

Hi everyone,

I am trying to send metrics from a Node.js lambda function using the ADOT layer. The function is using the layer arn:aws:lambda:eu-west-1:901920570463:layer:aws-otel-nodejs-amd64-ver-1-18-1:4 and has the permission to write to AMP Allow: aps:RemoteWrite.

Here is the Lambda code, based on the documentation :

const process = require('process');
const opentelemetry = require("@opentelemetry/sdk-node");
const opentelemetryapi = require('@opentelemetry/api');
const { Resource } = require("@opentelemetry/resources");
const { SemanticResourceAttributes } = require("@opentelemetry/semantic-conventions");
const { PeriodicExportingMetricReader, MeterProvider } = require("@opentelemetry/sdk-metrics");
const { OTLPMetricExporter } = require("@opentelemetry/exporter-metrics-otlp-grpc");

const _resource = Resource.default().merge(new Resource({
        [SemanticResourceAttributes.SERVICE_NAME]: "test-counter",
}));

const _metricReader = new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter(),
    exportIntervalMillis: 10
});

let notFirstinvocation = false;

const sdk = new opentelemetry.NodeSDK({
        metricReader: _metricReader,
        resource: _resource,
});
    
// gracefully shut down the SDK on process exit
process.on('SIGTERM', () => {
sdk.shutdown()
    .then(() => console.log('Metrics terminated'))
    .catch((error) => console.log('Error terminating metrics', error))
    .finally(() => process.exit(0));
});

const common_attributes = { signal: 'metric',  language: 'javascript', metricType: 'random' };

const mymeter = opentelemetryapi.metrics.getMeter('test-counter');
const counter = mymeter.createCounter('test-counter');

const handler = async (event, context) => {
    console.log('handler start');
    console.log(notFirstinvocation);
    if (notFirstinvocation == false) {
        // initialize the sdk on the first invocation
        await sdk.start();
        notFirstinvocation = true;
    }
    // increment the counter
    counter.add(1);
    console.log('added 1 to counter');
    return;  
};

// Export using module.exports
module.exports = { handler };

The counter is not exported to AMP and I have the following logs :

Function Logs:
{"level":"info","ts":1734026189.7502887,"msg":"Launching OpenTelemetry Lambda extension","version":"v0.40.0"}
{"level":"info","ts":1734026189.7631402,"logger":"telemetryAPI.Listener","msg":"Listening for requests","address":"sandbox.localdomain:53612"}
{"level":"info","ts":1734026189.7669532,"logger":"telemetryAPI.Client","msg":"Subscribing","baseURL":"http://127.0.0.1:9001/2022-07-01/telemetry"}
TELEMETRY	Name: collector	State: Subscribed	Types: [Platform]
{"level":"info","ts":1734026189.7685447,"logger":"telemetryAPI.Client","msg":"Subscription success","response":"\"OK\""}
{"level":"info","ts":1734026189.7686353,"logger":"NewCollector","msg":"Using config URI from environment","uri":"/var/task/collector.yaml"}
{"level":"info","ts":1734026189.7848382,"caller":"service@v0.102.1/service.go:113","msg":"Setting up own telemetry..."}
{"level":"info","ts":1734026189.7849793,"caller":"service@v0.102.1/telemetry.go:96","msg":"Serving metrics","address":":8888","level":"Normal"}
{"level":"info","ts":1734026189.7850993,"caller":"exporter@v0.102.1/exporter.go:275","msg":"Deprecated component. Will be removed in future releases.","kind":"exporter","data_type":"metrics","name":"logging"}
{"level":"warn","ts":1734026189.7858703,"caller":"common/factory.go:68","msg":"'loglevel' option is deprecated in favor of 'verbosity'. Set 'verbosity' to equivalent value to preserve behavior.","kind":"exporter","data_type":"metrics","name":"logging","loglevel":"debug","equivalent verbosity level":"Detailed"}
{"level":"info","ts":1734026189.7875144,"caller":"exporter@v0.102.1/exporter.go:275","msg":"Deprecated component. Will be removed in future releases.","kind":"exporter","data_type":"traces","name":"logging"}
{"level":"info","ts":1734026189.8045537,"caller":"service@v0.102.1/service.go:180","msg":"Starting aws-otel-lambda...","Version":"v0.40.0","NumCPU":2}
{"level":"info","ts":1734026189.8047967,"caller":"extensions/extensions.go:34","msg":"Starting extensions..."}
{"level":"info","ts":1734026189.804871,"caller":"extensions/extensions.go:37","msg":"Extension is starting...","kind":"extension","name":"sigv4auth"}
{"level":"info","ts":1734026189.8049858,"caller":"extensions/extensions.go:52","msg":"Extension started.","kind":"extension","name":"sigv4auth"}
{"level":"info","ts":1734026189.8100517,"caller":"otlpreceiver@v0.102.1/otlp.go:102","msg":"Starting GRPC server","kind":"receiver","name":"otlp","data_type":"traces","endpoint":"localhost:4317"}
{"level":"info","ts":1734026189.8106089,"caller":"otlpreceiver@v0.102.1/otlp.go:152","msg":"Starting HTTP server","kind":"receiver","name":"otlp","data_type":"traces","endpoint":"localhost:4318"}
{"level":"info","ts":1734026189.8109326,"caller":"service@v0.102.1/service.go:206","msg":"Everything is ready. Begin running and processing data."}
{"level":"warn","ts":1734026189.8111176,"caller":"localhostgate/featuregate.go:63","msg":"The default endpoints for all servers in components will change to use localhost instead of 0.0.0.0 in a future version. Use the feature gate to preview the new default.","feature gate ID":"component.UseLocalHostAsDefaultHost"}
Registering OpenTelemetry
Propagator "xray" requested through environment variable is unavailable.
2024-12-12T17:56:31.666Z	undefined	WARN	Failed extracting version /var/task
EXTENSION	Name: collector	State: Ready	Events: [INVOKE, SHUTDOWN]
START RequestId: 7ea48e1a-bf88-4086-8372-1adf6e6fa1d3 Version: $LATEST
2024-12-12T17:56:32.005Z	7ea48e1a-bf88-4086-8372-1adf6e6fa1d3	INFO	handler start
2024-12-12T17:56:32.184Z	7ea48e1a-bf88-4086-8372-1adf6e6fa1d3	INFO	added 1 to counter
END RequestId: 7ea48e1a-bf88-4086-8372-1adf6e6fa1d3
REPORT RequestId: 7ea48e1a-bf88-4086-8372-1adf6e6fa1d3	Duration: 3253.36 ms	Billed Duration: 3254 ms	Memory Size: 128 MB	Max Memory Used: 128 MB	Init Duration: 2233.34 ms

Request ID: 7ea48e1a-bf88-4086-8372-1adf6e6fa1d3

Any idea of what I m missing here ? There's no log about the counter even with loglevel set to Debug. Thank you.