From 693af666a05f0973c7d1d43a58ae368dcaf1d001 Mon Sep 17 00:00:00 2001 From: Michele Mancioppi Date: Thu, 4 Aug 2022 22:09:44 +0200 Subject: [PATCH 1/6] Implement aws.ecs.* and aws.logs.* resource attributes Add support for the aws.ecs.* and aws.logs.* resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available --- CHANGELOG.md | 2 + .../sdk/extension/aws/resource/ecs.py | 114 ++++++++++- .../metadatav4-response-container-ec2.json | 44 +++++ ...metadatav4-response-container-fargate.json | 50 +++++ .../ecs/metadatav4-response-task-ec2.json | 94 ++++++++++ .../ecs/metadatav4-response-task-fargate.json | 77 ++++++++ .../tests/resource/test_ecs.py | 177 +++++++++++++++++- 7 files changed, 556 insertions(+), 2 deletions(-) create mode 100644 sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-ec2.json create mode 100644 sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-fargate.json create mode 100644 sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-ec2.json create mode 100644 sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-fargate.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0c50b8e9..8cc14f92a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-exporter-richconsole` Fixing RichConsoleExpoter to allow multiple traces, fixing duplicate spans and include resources ([#1336](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1336)) - `opentelemetry-instrumentation-asgi` metrics record target attribute (FastAPI only) ([#1323](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1323)) +- Implement [`aws.ecs.*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md) and [`aws.logs.*`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud_provider/aws/logs/) resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available + ([#1212](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1212)) ## [1.13.0-0.34b0](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.13.0-0.34b0) - 2022-09-26 diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py index 3b0d6d79a1..a785f82a67 100644 --- a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py +++ b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os +import re import socket +from urllib.request import Request, urlopen from opentelemetry.sdk.resources import Resource, ResourceDetector from opentelemetry.semconv.resource import ( @@ -58,7 +61,7 @@ def detect(self) -> "Resource": "Failed to get container ID on ECS: %s.", exception ) - return Resource( + base_resource = Resource( { ResourceAttributes.CLOUD_PROVIDER: CloudProviderValues.AWS.value, ResourceAttributes.CLOUD_PLATFORM: CloudPlatformValues.AWS_ECS.value, @@ -66,6 +69,51 @@ def detect(self) -> "Resource": ResourceAttributes.CONTAINER_ID: container_id, } ) + + metadata_v4_endpoint = os.environ.get( + "ECS_CONTAINER_METADATA_URI_V4" + ) + + if not metadata_v4_endpoint: + return base_resource + + # Returns https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html#task-metadata-endpoint-v4-response + metadata_container = json.loads(_http_get(metadata_v4_endpoint)) + metadata_task = json.loads( + _http_get(f"{metadata_v4_endpoint}/task") + ) + + task_arn = metadata_task["TaskARN"] + base_arn = task_arn[0 : task_arn.rindex(":")] # noqa + cluster: str = metadata_task["Cluster"] + cluster_arn = ( + cluster + if cluster.startswith("arn:") + else f"{base_arn}:cluster/{cluster}" + ) + + logs_resource = _get_logs_resource(metadata_container) + + return base_resource.merge(logs_resource).merge( + Resource( + { + ResourceAttributes.AWS_ECS_CONTAINER_ARN: metadata_container[ + "ContainerARN" + ], + ResourceAttributes.AWS_ECS_CLUSTER_ARN: cluster_arn, + ResourceAttributes.AWS_ECS_LAUNCHTYPE: metadata_task[ + "LaunchType" + ].lower(), + ResourceAttributes.AWS_ECS_TASK_ARN: task_arn, + ResourceAttributes.AWS_ECS_TASK_FAMILY: metadata_task[ + "Family" + ], + ResourceAttributes.AWS_ECS_TASK_REVISION: metadata_task[ + "Revision" + ], + } + ) + ) # pylint: disable=broad-except except Exception as exception: if self.raise_on_error: @@ -73,3 +121,67 @@ def detect(self) -> "Resource": logger.warning("%s failed: %s", self.__class__.__name__, exception) return Resource.get_empty() + + +def _get_logs_resource(metadata_container): + if metadata_container.get("LogDriver") == "awslogs": + log_options = metadata_container.get("LogOptions") + if log_options: + logs_region = log_options.get("awslogs-region") + logs_group_name = log_options.get("awslogs-group") + logs_stream_name = log_options.get("awslogs-stream") + + container_arn = metadata_container["ContainerARN"] + + if not logs_region: + aws_region_match = re.match( + r'arn:aws:ecs:([^:]+):.*', container_arn + ) + if aws_region_match: + logs_region = aws_region_match.group(1) + + else: + logger.warning("Cannot parse AWS account out of ECS ARN") + + # We need to retrieve the account it from some other ARN to create the + # log-group and log-stream ARNs + aws_account = None + aws_account_match = re.match( + r'arn:aws:ecs:[^:]+:([^:]+):.*', container_arn + ) + if aws_account_match: + aws_account = aws_account_match.group(1) + + logs_group_arn = None + logs_stream_arn = None + if logs_region and aws_account: + if logs_group_name: + logs_group_arn = f"arn:aws:logs:{logs_region}:{aws_account}:log-group:{logs_group_name}" + + if logs_stream_name: + logs_stream_arn = f"arn:aws:logs:{logs_region}:{aws_account}:log-group:{logs_group_name}:log-stream:{logs_stream_name}" + + return Resource( + { + ResourceAttributes.AWS_LOG_GROUP_NAMES: [logs_group_name], + ResourceAttributes.AWS_LOG_GROUP_ARNS: [logs_group_arn], + ResourceAttributes.AWS_LOG_STREAM_NAMES: [ + logs_stream_name + ], + ResourceAttributes.AWS_LOG_STREAM_ARNS: [logs_stream_arn], + } + ) + else: + logger.warning( + "The metadata endpoint v4 has returned 'awslogs' as 'LogDriver', but there is no 'LogOptions' data" + ) + + return Resource.get_empty() + + +def _http_get(url): + with urlopen( + Request(url, method="GET"), + timeout=5, + ) as response: + return response.read().decode("utf-8") diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-ec2.json b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-ec2.json new file mode 100644 index 0000000000..b43c2b0d7d --- /dev/null +++ b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-ec2.json @@ -0,0 +1,44 @@ +{ + "DockerId": "ea32192c8553fbff06c9340478a2ff089b2bb5646fb718b4ee206641c9086d66", + "Name": "curl", + "DockerName": "ecs-curltest-24-curl-cca48e8dcadd97805600", + "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest", + "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553", + "Labels": { + "com.amazonaws.ecs.cluster": "default", + "com.amazonaws.ecs.container-name": "curl", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/8f03e41243824aea923aca126495f665", + "com.amazonaws.ecs.task-definition-family": "curltest", + "com.amazonaws.ecs.task-definition-version": "24" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 10, + "Memory": 128 + }, + "CreatedAt": "2020-10-02T00:15:07.620912337Z", + "StartedAt": "2020-10-02T00:15:08.062559351Z", + "Type": "NORMAL", + "LogDriver": "awslogs", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/metadata", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/curl/8f03e41243824aea923aca126495f665" + }, + "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/0206b271-b33f-47ab-86c6-a0ba208a70a9", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.2.100" + ], + "AttachmentIndex": 0, + "MACAddress": "0e:9e:32:c7:48:85", + "IPv4SubnetCIDRBlock": "10.0.2.0/24", + "PrivateDNSName": "ip-10-0-2-100.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "10.0.2.1/24" + } + ] +} diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-fargate.json b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-fargate.json new file mode 100644 index 0000000000..ccbe70bc44 --- /dev/null +++ b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-fargate.json @@ -0,0 +1,50 @@ +{ + "DockerId": "cd189a933e5849daa93386466019ab50-2495160603", + "Name": "curl", + "DockerName": "curl", + "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest", + "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "curl", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50", + "com.amazonaws.ecs.task-definition-family": "curltest", + "com.amazonaws.ecs.task-definition-version": "2" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 10, + "Memory": 128 + }, + "CreatedAt": "2020-10-08T20:09:11.44527186Z", + "StartedAt": "2020-10-08T20:09:11.44527186Z", + "Type": "NORMAL", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "192.0.2.3" + ], + "AttachmentIndex": 0, + "MACAddress": "0a:de:f6:10:51:e5", + "IPv4SubnetCIDRBlock": "192.0.2.0/24", + "DomainNameServers": [ + "192.0.2.2" + ], + "DomainNameSearchList": [ + "us-west-2.compute.internal" + ], + "PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "192.0.2.0/24" + } + ], + "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/containerlogs", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50" + }, + "LogDriver": "awslogs" +} \ No newline at end of file diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-ec2.json b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-ec2.json new file mode 100644 index 0000000000..101efe0214 --- /dev/null +++ b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-ec2.json @@ -0,0 +1,94 @@ +{ + "Cluster": "default", + "TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c", + "Family": "curltest", + "Revision": "26", + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "PullStartedAt": "2020-10-02T00:43:06.202617438Z", + "PullStoppedAt": "2020-10-02T00:43:06.31288465Z", + "AvailabilityZone": "us-west-2d", + "LaunchType": "EC2", + "Containers": [ + { + "DockerId": "598cba581fe3f939459eaba1e071d5c93bb2c49b7d1ba7db6bb19deeb70d8e38", + "Name": "~internal~ecs~pause", + "DockerName": "ecs-curltest-26-internalecspause-e292d586b6f9dade4a00", + "Image": "amazon/amazon-ecs-pause:0.1.0", + "ImageID": "", + "Labels": { + "com.amazonaws.ecs.cluster": "default", + "com.amazonaws.ecs.container-name": "~internal~ecs~pause", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c", + "com.amazonaws.ecs.task-definition-family": "curltest", + "com.amazonaws.ecs.task-definition-version": "26" + }, + "DesiredStatus": "RESOURCES_PROVISIONED", + "KnownStatus": "RESOURCES_PROVISIONED", + "Limits": { + "CPU": 0, + "Memory": 0 + }, + "CreatedAt": "2020-10-02T00:43:05.602352471Z", + "StartedAt": "2020-10-02T00:43:06.076707576Z", + "Type": "CNI_PAUSE", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.2.61" + ], + "AttachmentIndex": 0, + "MACAddress": "0e:10:e2:01:bd:91", + "IPv4SubnetCIDRBlock": "10.0.2.0/24", + "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "10.0.2.1/24" + } + ] + }, + { + "DockerId": "ee08638adaaf009d78c248913f629e38299471d45fe7dc944d1039077e3424ca", + "Name": "curl", + "DockerName": "ecs-curltest-26-curl-a0e7dba5aca6d8cb2e00", + "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest", + "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553", + "Labels": { + "com.amazonaws.ecs.cluster": "default", + "com.amazonaws.ecs.container-name": "curl", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c", + "com.amazonaws.ecs.task-definition-family": "curltest", + "com.amazonaws.ecs.task-definition-version": "26" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 10, + "Memory": 128 + }, + "CreatedAt": "2020-10-02T00:43:06.326590752Z", + "StartedAt": "2020-10-02T00:43:06.767535449Z", + "Type": "NORMAL", + "LogDriver": "awslogs", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/metadata", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/curl/158d1c8083dd49d6b527399fd6414f5c" + }, + "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/abb51bdd-11b4-467f-8f6c-adcfe1fe059d", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.2.61" + ], + "AttachmentIndex": 0, + "MACAddress": "0e:10:e2:01:bd:91", + "IPv4SubnetCIDRBlock": "10.0.2.0/24", + "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "10.0.2.1/24" + } + ] + } + ] +} diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-fargate.json b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-fargate.json new file mode 100644 index 0000000000..7979db708d --- /dev/null +++ b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-fargate.json @@ -0,0 +1,77 @@ +{ + "Cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3", + "Family": "curltest", + "Revision": "3", + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 0.25, + "Memory": 512 + }, + "PullStartedAt": "2020-10-08T20:47:16.053330955Z", + "PullStoppedAt": "2020-10-08T20:47:19.592684631Z", + "AvailabilityZone": "us-west-2a", + "Containers": [ + { + "DockerId": "e9028f8d5d8e4f258373e7b93ce9a3c3-2495160603", + "Name": "curl", + "DockerName": "curl", + "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest", + "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "curl", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3", + "com.amazonaws.ecs.task-definition-family": "curltest", + "com.amazonaws.ecs.task-definition-version": "3" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 10, + "Memory": 128 + }, + "CreatedAt": "2020-10-08T20:47:20.567813946Z", + "StartedAt": "2020-10-08T20:47:20.567813946Z", + "Type": "NORMAL", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "192.0.2.3" + ], + "IPv6Addresses": [ + "2001:dB8:10b:1a00:32bf:a372:d80f:e958" + ], + "AttachmentIndex": 0, + "MACAddress": "02:b7:20:19:72:39", + "IPv4SubnetCIDRBlock": "192.0.2.0/24", + "IPv6SubnetCIDRBlock": "2600:1f13:10b:1a00::/64", + "DomainNameServers": [ + "192.0.2.2" + ], + "DomainNameSearchList": [ + "us-west-2.compute.internal" + ], + "PrivateDNSName": "ip-172-31-30-173.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "192.0.2.0/24" + } + ], + "ClockDrift": { + "ClockErrorBound": 0.5458234999999999, + "ReferenceTimestamp": "2021-09-07T16:57:44Z", + "ClockSynchronizationStatus": "SYNCHRONIZED" + }, + "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/1bdcca8b-f905-4ee6-885c-4064cb70f6e6", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/containerlogs", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/curl/e9028f8d5d8e4f258373e7b93ce9a3c3" + }, + "LogDriver": "awslogs" + } + ], + "LaunchType": "FARGATE" +} \ No newline at end of file diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py index 04931e1b76..ac489622bd 100644 --- a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py +++ b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest from collections import OrderedDict from unittest.mock import mock_open, patch @@ -31,6 +32,46 @@ } +def _read_file(filename: str) -> str: + with open(os.path.join(os.path.dirname(__file__), "ecs", filename)) as f: + return f.read() + + +MetadataV4Uri = "mock-uri-4" + + +MetadataV4ContainerResponseEc2 = _read_file( + "metadatav4-response-container-ec2.json" +) + + +MetadataV4TaskResponseEc2 = _read_file("metadatav4-response-task-ec2.json") + + +MetadataV4ContainerResponseFargate = _read_file( + "metadatav4-response-container-fargate.json" +) + + +MetadataV4TaskResponseFargate = _read_file( + "metadatav4-response-task-fargate.json" +) + + +def _http_get_function_ec2(url: str, *args, **kwargs) -> str: + if url == MetadataV4Uri: + return MetadataV4ContainerResponseEc2 + if url == f"{MetadataV4Uri}/task": + return MetadataV4TaskResponseEc2 + + +def _http_get_function_fargate(url: str, *args, **kwargs) -> str: + if url == MetadataV4Uri: + return MetadataV4ContainerResponseFargate + if url == f"{MetadataV4Uri}/task": + return MetadataV4TaskResponseFargate + + class AwsEcsResourceDetectorTest(unittest.TestCase): @patch.dict( "os.environ", @@ -60,8 +101,142 @@ class AwsEcsResourceDetectorTest(unittest.TestCase): 1:cpuset:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked """, ) - def test_simple_create(self, mock_open_function, mock_socket_gethostname): + def test_simple_create_metadata_v3( + self, + mock_open_function, + mock_socket_gethostname, + ): actual = AwsEcsResourceDetector().detect() self.assertDictEqual( actual.attributes.copy(), OrderedDict(MockEcsResourceAttributes) ) + + @patch.dict( + "os.environ", + {"ECS_CONTAINER_METADATA_URI_V4": MetadataV4Uri}, + clear=True, + ) + @patch( + "socket.gethostname", + return_value=f"{MockEcsResourceAttributes[ResourceAttributes.CONTAINER_NAME]}", + ) + @patch( + "builtins.open", + new_callable=mock_open, + read_data=f"""14:name=systemd:/docker/{MockEcsResourceAttributes[ResourceAttributes.CONTAINER_ID]} +13:rdma:/ +12:pids:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +11:hugetlb:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +10:net_prio:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +9:perf_event:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +8:net_cls:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +7:freezer:/docker/ +6:devices:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +5:memory:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +4:blkio:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +3:cpuacct:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +2:cpu:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +1:cpuset:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +""", + ) + @patch( + "opentelemetry.sdk.extension.aws.resource.ecs._http_get", + ) + def test_simple_create_metadata_v4_launchtype_ec2( + self, + mock_http_get_function, + mock_open_function, + mock_socket_gethostname, + ): + mock_http_get_function.side_effect = _http_get_function_ec2 + actual = AwsEcsResourceDetector().detect() + self.maxDiff = None + self.assertDictEqual( + actual.attributes.copy(), + OrderedDict( + { + **MockEcsResourceAttributes, + ResourceAttributes.AWS_LOG_GROUP_NAMES: ("/ecs/metadata",), + ResourceAttributes.AWS_LOG_GROUP_ARNS: ( + "arn:aws:logs:us-west-2:111122223333:log-group:/ecs/metadata", + ), + ResourceAttributes.AWS_LOG_STREAM_NAMES: ( + "ecs/curl/8f03e41243824aea923aca126495f665", + ), + ResourceAttributes.AWS_LOG_STREAM_ARNS: ( + "arn:aws:logs:us-west-2:111122223333:log-group:/ecs/metadata:log-stream:ecs/curl/8f03e41243824aea923aca126495f665", + ), + ResourceAttributes.AWS_ECS_CONTAINER_ARN: "arn:aws:ecs:us-west-2:111122223333:container/0206b271-b33f-47ab-86c6-a0ba208a70a9", + ResourceAttributes.AWS_ECS_CLUSTER_ARN: "arn:aws:ecs:us-west-2:111122223333:cluster/default", + ResourceAttributes.AWS_ECS_LAUNCHTYPE: "ec2", + ResourceAttributes.AWS_ECS_TASK_ARN: "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c", + ResourceAttributes.AWS_ECS_TASK_FAMILY: "curltest", + ResourceAttributes.AWS_ECS_TASK_REVISION: "26", + } + ), + ) + + @patch.dict( + "os.environ", + {"ECS_CONTAINER_METADATA_URI_V4": MetadataV4Uri}, + clear=True, + ) + @patch( + "socket.gethostname", + return_value=f"{MockEcsResourceAttributes[ResourceAttributes.CONTAINER_NAME]}", + ) + @patch( + "builtins.open", + new_callable=mock_open, + read_data=f"""14:name=systemd:/docker/{MockEcsResourceAttributes[ResourceAttributes.CONTAINER_ID]} +13:rdma:/ +12:pids:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +11:hugetlb:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +10:net_prio:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +9:perf_event:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +8:net_cls:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +7:freezer:/docker/ +6:devices:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +5:memory:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +4:blkio:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +3:cpuacct:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +2:cpu:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +1:cpuset:/docker/bogusContainerIdThatShouldNotBeOneSetBecauseTheFirstOneWasPicked +""", + ) + @patch( + "opentelemetry.sdk.extension.aws.resource.ecs._http_get", + ) + def test_simple_create_metadata_v4_launchtype_fargate( + self, + mock_http_get_function, + mock_open_function, + mock_socket_gethostname, + ): + mock_http_get_function.side_effect = _http_get_function_fargate + actual = AwsEcsResourceDetector().detect() + self.maxDiff = None + self.assertDictEqual( + actual.attributes.copy(), + OrderedDict( + { + **MockEcsResourceAttributes, + ResourceAttributes.AWS_LOG_GROUP_NAMES: ("/ecs/containerlogs",), + ResourceAttributes.AWS_LOG_GROUP_ARNS: ( + "arn:aws:logs:us-west-2:111122223333:log-group:/ecs/containerlogs", + ), + ResourceAttributes.AWS_LOG_STREAM_NAMES: ( + "ecs/curl/cd189a933e5849daa93386466019ab50", + ), + ResourceAttributes.AWS_LOG_STREAM_ARNS: ( + "arn:aws:logs:us-west-2:111122223333:log-group:/ecs/containerlogs:log-stream:ecs/curl/cd189a933e5849daa93386466019ab50", + ), + ResourceAttributes.AWS_ECS_CONTAINER_ARN: "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1", + ResourceAttributes.AWS_ECS_CLUSTER_ARN: "arn:aws:ecs:us-west-2:111122223333:cluster/default", + ResourceAttributes.AWS_ECS_LAUNCHTYPE: "fargate", + ResourceAttributes.AWS_ECS_TASK_ARN: "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3", + ResourceAttributes.AWS_ECS_TASK_FAMILY: "curltest", + ResourceAttributes.AWS_ECS_TASK_REVISION: "3", + } + ), + ) From f5be8cda12b038c748ad2a121d94035aae3d24b9 Mon Sep 17 00:00:00 2001 From: Michele Mancioppi Date: Wed, 28 Sep 2022 18:15:29 +0200 Subject: [PATCH 2/6] lint --- .../src/opentelemetry/sdk/extension/aws/resource/ecs.py | 4 ++-- .../tests/resource/test_ecs.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py index a785f82a67..6bd396e69d 100644 --- a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py +++ b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py @@ -135,7 +135,7 @@ def _get_logs_resource(metadata_container): if not logs_region: aws_region_match = re.match( - r'arn:aws:ecs:([^:]+):.*', container_arn + r"arn:aws:ecs:([^:]+):.*", container_arn ) if aws_region_match: logs_region = aws_region_match.group(1) @@ -147,7 +147,7 @@ def _get_logs_resource(metadata_container): # log-group and log-stream ARNs aws_account = None aws_account_match = re.match( - r'arn:aws:ecs:[^:]+:([^:]+):.*', container_arn + r"arn:aws:ecs:[^:]+:([^:]+):.*", container_arn ) if aws_account_match: aws_account = aws_account_match.group(1) diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py index ac489622bd..0ef77ad49e 100644 --- a/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py +++ b/sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/test_ecs.py @@ -221,7 +221,9 @@ def test_simple_create_metadata_v4_launchtype_fargate( OrderedDict( { **MockEcsResourceAttributes, - ResourceAttributes.AWS_LOG_GROUP_NAMES: ("/ecs/containerlogs",), + ResourceAttributes.AWS_LOG_GROUP_NAMES: ( + "/ecs/containerlogs", + ), ResourceAttributes.AWS_LOG_GROUP_ARNS: ( "arn:aws:logs:us-west-2:111122223333:log-group:/ecs/containerlogs", ), From fbe2c808fd477af689304d36caaa5235d343d81f Mon Sep 17 00:00:00 2001 From: Michele Mancioppi Date: Sat, 22 Oct 2022 18:55:47 +0200 Subject: [PATCH 3/6] lint some more --- .../src/opentelemetry/sdk/extension/aws/resource/ecs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py index 6bd396e69d..44df3e7738 100644 --- a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py +++ b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py @@ -171,10 +171,10 @@ def _get_logs_resource(metadata_container): ResourceAttributes.AWS_LOG_STREAM_ARNS: [logs_stream_arn], } ) - else: - logger.warning( - "The metadata endpoint v4 has returned 'awslogs' as 'LogDriver', but there is no 'LogOptions' data" - ) + + logger.warning( + "The metadata endpoint v4 has returned 'awslogs' as 'LogDriver', but there is no 'LogOptions' data" + ) return Resource.get_empty() From 909513c907d5612d73c2ca8e39baba70ab6f49d8 Mon Sep 17 00:00:00 2001 From: Michele Mancioppi Date: Wed, 11 Jan 2023 21:57:32 +0100 Subject: [PATCH 4/6] Update sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py Co-authored-by: Anthony Mirabella --- .../src/opentelemetry/sdk/extension/aws/resource/ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py index 44df3e7738..61129e2fbd 100644 --- a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py +++ b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py @@ -141,7 +141,7 @@ def _get_logs_resource(metadata_container): logs_region = aws_region_match.group(1) else: - logger.warning("Cannot parse AWS account out of ECS ARN") + logger.warning("Cannot parse AWS region out of ECS ARN") # We need to retrieve the account it from some other ARN to create the # log-group and log-stream ARNs From 309245fe485a419b90a79c3a045f6cb6f184e7ad Mon Sep 17 00:00:00 2001 From: Michele Mancioppi Date: Wed, 11 Jan 2023 21:57:41 +0100 Subject: [PATCH 5/6] Update sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py Co-authored-by: Anthony Mirabella --- .../src/opentelemetry/sdk/extension/aws/resource/ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py index 61129e2fbd..b780633b69 100644 --- a/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py +++ b/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py @@ -143,7 +143,7 @@ def _get_logs_resource(metadata_container): else: logger.warning("Cannot parse AWS region out of ECS ARN") - # We need to retrieve the account it from some other ARN to create the + # We need to retrieve the account ID from some other ARN to create the # log-group and log-stream ARNs aws_account = None aws_account_match = re.match( From 1ae794dec2a9589114d0f6dbe8b160932c868846 Mon Sep 17 00:00:00 2001 From: Michele Mancioppi Date: Wed, 11 Jan 2023 21:59:30 +0100 Subject: [PATCH 6/6] Update CHANGELOG.md --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dbb36be73..f779f3f109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased -## Fixed +### Added + +- `opentelemetry/sdk/extension/aws` Implement [`aws.ecs.*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md) and [`aws.logs.*`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud_provider/aws/logs/) resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available + ([#1212](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1212)) + +### Fixed - Fix aiopg instrumentation to work with aiopg < 2.0.0 ([#1473](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1473)) @@ -44,8 +49,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1434](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1434)) - `opentelemetry-instrumentation-confluent-kafka` Add support for the latest versions of the library. ([#1468](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1468)) -- `opentelemetry/sdk/extension/aws` Implement [`aws.ecs.*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md) and [`aws.logs.*`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud_provider/aws/logs/) resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available - ([#1212](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1212)) ### Fixed