From ef9b15e0c12cee086ab8a6e7439e64258b029a3b Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 18 Aug 2023 12:39:33 -0400 Subject: [PATCH] write logs to gae_app instead of gae_instance (#702) --- exporter/collector/config.go | 2 +- exporter/collector/logs.go | 2 +- exporter/collector/metrics_test.go | 2 +- exporter/collector/monitoredresource.go | 23 +++---- exporter/collector/monitoredresource_test.go | 65 +++++++++++++++++++- exporter/metric/metric.go | 2 +- exporter/trace/trace_proto.go | 2 +- internal/resourcemapping/go.mod | 10 ++- internal/resourcemapping/go.sum | 10 +++ internal/resourcemapping/resourcemapping.go | 58 +++++++++++++---- 10 files changed, 141 insertions(+), 35 deletions(-) diff --git a/exporter/collector/config.go b/exporter/collector/config.go index 6d4be824c..7c66bb49d 100644 --- a/exporter/collector/config.go +++ b/exporter/collector/config.go @@ -203,7 +203,7 @@ func DefaultConfig() Config { ServiceResourceLabels: true, CumulativeNormalization: true, GetMetricName: defaultGetMetricName, - MapMonitoredResource: defaultResourceToMonitoredResource, + MapMonitoredResource: defaultResourceToMonitoringMonitoredResource, }, } } diff --git a/exporter/collector/logs.go b/exporter/collector/logs.go index 5fdddc9a1..928c6b3c3 100644 --- a/exporter/collector/logs.go +++ b/exporter/collector/logs.go @@ -247,7 +247,7 @@ func (l logMapper) createEntries(ld plog.Logs) (map[string][]*logpb.LogEntry, er entries := make(map[string][]*logpb.LogEntry) for i := 0; i < ld.ResourceLogs().Len(); i++ { rl := ld.ResourceLogs().At(i) - mr := defaultResourceToMonitoredResource(rl.Resource()) + mr := defaultResourceToLoggingMonitoredResource(rl.Resource()) extraResourceLabels := resourceToLabels(rl.Resource(), l.cfg.LogConfig.ServiceResourceLabels, l.cfg.LogConfig.ResourceFilters, l.obs.log) projectID := l.cfg.ProjectID // override project ID with gcp.project.id, if present diff --git a/exporter/collector/metrics_test.go b/exporter/collector/metrics_test.go index 04a462d15..f70014b38 100644 --- a/exporter/collector/metrics_test.go +++ b/exporter/collector/metrics_test.go @@ -2397,7 +2397,7 @@ func TestPushMetricsOntoWAL(t *testing.T) { obs := selfObservability{zap.NewNop()} cfg := Config{ MetricConfig: MetricConfig{ - MapMonitoredResource: defaultResourceToMonitoredResource, + MapMonitoredResource: defaultResourceToMonitoringMonitoredResource, GetMetricName: defaultGetMetricName, WALConfig: &WALConfig{ Directory: tmpDir, diff --git a/exporter/collector/monitoredresource.go b/exporter/collector/monitoredresource.go index 6952f3239..b2f6e0a0d 100644 --- a/exporter/collector/monitoredresource.go +++ b/exporter/collector/monitoredresource.go @@ -39,20 +39,17 @@ func (attrs *attributes) GetString(key string) (string, bool) { } // defaultResourceToMonitoredResource pdata Resource to a GCM Monitored Resource. -func defaultResourceToMonitoredResource(resource pcommon.Resource) *monitoredrespb.MonitoredResource { - attrs := resource.Attributes() - gmr := resourcemapping.ResourceAttributesToMonitoredResource(&attributes{ - Attrs: attrs, +func defaultResourceToMonitoringMonitoredResource(resource pcommon.Resource) *monitoredrespb.MonitoredResource { + return resourcemapping.ResourceAttributesToMonitoringMonitoredResource(&attributes{ + Attrs: resource.Attributes(), + }) +} + +// defaultResourceToMonitoredResource pdata Resource to a GCM Monitored Resource. +func defaultResourceToLoggingMonitoredResource(resource pcommon.Resource) *monitoredrespb.MonitoredResource { + return resourcemapping.ResourceAttributesToLoggingMonitoredResource(&attributes{ + Attrs: resource.Attributes(), }) - newLabels := make(labels, len(gmr.Labels)) - for k, v := range gmr.Labels { - newLabels[k] = sanitizeUTF8(v) - } - mr := &monitoredrespb.MonitoredResource{ - Type: gmr.Type, - Labels: newLabels, - } - return mr } // resourceToLabels converts the Resource attributes into labels. diff --git a/exporter/collector/monitoredresource_test.go b/exporter/collector/monitoredresource_test.go index 140e447ae..def2be48c 100644 --- a/exporter/collector/monitoredresource_test.go +++ b/exporter/collector/monitoredresource_test.go @@ -22,7 +22,7 @@ import ( monitoredrespb "google.golang.org/genproto/googleapis/api/monitoredres" ) -func TestResourceMetricsToMonitoredResource(t *testing.T) { +func TestResourceMetricsToMonitoringMonitoredResource(t *testing.T) { tests := []struct { resourceLabels map[string]string expectMr *monitoredrespb.MonitoredResource @@ -497,6 +497,65 @@ func TestResourceMetricsToMonitoredResource(t *testing.T) { }, }, }, + { + name: "Gae Instance", + resourceLabels: map[string]string{ + "cloud.provider": "gcp", + "cloud.platform": "gcp_app_engine", + "cloud.availability_zone": "my-zone", + "faas.id": "myinstanceid", + "faas.name": "myhostname", + "faas.version": "v1", + }, + expectMr: &monitoredrespb.MonitoredResource{ + Type: "gae_instance", + Labels: map[string]string{ + "instance_id": "myinstanceid", + "location": "my-zone", + "module_id": "myhostname", + "version_id": "v1", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := pcommon.NewResource() + for k, v := range test.resourceLabels { + r.Attributes().PutStr(k, v) + } + mr := defaultResourceToMonitoringMonitoredResource(r) + assert.Equal(t, test.expectMr, mr) + }) + } +} + +func TestResourceMetricsToLoggingMonitoredResource(t *testing.T) { + tests := []struct { + resourceLabels map[string]string + expectMr *monitoredrespb.MonitoredResource + name string + }{ + { + name: "Gae App", + resourceLabels: map[string]string{ + "cloud.provider": "gcp", + "cloud.platform": "gcp_app_engine", + "cloud.availability_zone": "my-zone", + "faas.id": "myhostid", + "faas.name": "myhostname", + "faas.version": "v1", + }, + expectMr: &monitoredrespb.MonitoredResource{ + Type: "gae_app", + Labels: map[string]string{ + "location": "my-zone", + "module_id": "myhostname", + "version_id": "v1", + }, + }, + }, } for _, test := range tests { @@ -505,7 +564,7 @@ func TestResourceMetricsToMonitoredResource(t *testing.T) { for k, v := range test.resourceLabels { r.Attributes().PutStr(k, v) } - mr := defaultResourceToMonitoredResource(r) + mr := defaultResourceToLoggingMonitoredResource(r) assert.Equal(t, test.expectMr, mr) }) } @@ -749,7 +808,7 @@ func TestResourceMetricsToMonitoredResourceUTF8(t *testing.T) { for k, v := range resourceLabels { r.Attributes().PutStr(k, v) } - mr := defaultResourceToMonitoredResource(r) + mr := defaultResourceToMonitoringMonitoredResource(r) assert.Equal(t, expectMr, mr) extraLabels := resourceToLabels(r, mapper.cfg.MetricConfig.ServiceResourceLabels, mapper.cfg.MetricConfig.ResourceFilters, nil) assert.Equal(t, expectExtraLabels, extraLabels) diff --git a/exporter/metric/metric.go b/exporter/metric/metric.go index 5137b23fa..125a5d8c9 100644 --- a/exporter/metric/metric.go +++ b/exporter/metric/metric.go @@ -357,7 +357,7 @@ func (attrs *attributes) GetString(key string) (string, bool) { // // https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.monitoredResourceDescriptors func (me *metricExporter) resourceToMonitoredResourcepb(res *resource.Resource) *monitoredrespb.MonitoredResource { - gmr := resourcemapping.ResourceAttributesToMonitoredResource(&attributes{ + gmr := resourcemapping.ResourceAttributesToMonitoringMonitoredResource(&attributes{ attrs: attribute.NewSet(res.Attributes()...), }) newLabels := make(map[string]string, len(gmr.Labels)) diff --git a/exporter/trace/trace_proto.go b/exporter/trace/trace_proto.go index 6ff92e93b..27e95e5b7 100644 --- a/exporter/trace/trace_proto.go +++ b/exporter/trace/trace_proto.go @@ -118,7 +118,7 @@ func attributeWithLabelsFromResources(sd sdktrace.ReadOnlySpan) []attribute.KeyV } // Monitored resource attributes (`g.co/r/{resource_type}/{resource_label}`) come next. - gceResource := resourcemapping.ResourceAttributesToMonitoredResource(&attrs{ + gceResource := resourcemapping.ResourceAttributesToMonitoringMonitoredResource(&attrs{ Attrs: sd.Resource().Attributes(), }) for key, value := range gceResource.Labels { diff --git a/internal/resourcemapping/go.mod b/internal/resourcemapping/go.mod index 8e87183c1..bea463439 100644 --- a/internal/resourcemapping/go.mod +++ b/internal/resourcemapping/go.mod @@ -2,4 +2,12 @@ module github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resou go 1.21 -require go.opentelemetry.io/otel v1.16.0 +require ( + go.opentelemetry.io/otel v1.16.0 + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e +) + +require ( + google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/internal/resourcemapping/go.sum b/internal/resourcemapping/go.sum index a8319c445..625eb4d63 100644 --- a/internal/resourcemapping/go.sum +++ b/internal/resourcemapping/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -8,5 +10,13 @@ github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gt github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 h1:Au6te5hbKUV8pIYWHqOUZ1pva5qK/rwbIhoXEUB9Lu8= +google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/resourcemapping/resourcemapping.go b/internal/resourcemapping/resourcemapping.go index 253da276d..31909af21 100644 --- a/internal/resourcemapping/resourcemapping.go +++ b/internal/resourcemapping/resourcemapping.go @@ -15,7 +15,10 @@ package resourcemapping import ( + "strings" + semconv "go.opentelemetry.io/otel/semconv/v1.18.0" + monitoredrespb "google.golang.org/genproto/googleapis/api/monitoredres" ) const ( @@ -44,6 +47,7 @@ const ( taskID = "task_id" zone = "zone" gaeInstance = "gae_instance" + gaeApp = "gae_app" gaeModuleID = "module_id" gaeVersionID = "version_id" cloudRunRevision = "cloud_run_revision" @@ -113,6 +117,14 @@ var ( gaeVersionID: {otelKeys: []string{string(semconv.FaaSVersionKey)}}, instanceID: {otelKeys: []string{string(semconv.FaaSIDKey)}}, }, + gaeApp: { + location: {otelKeys: []string{ + string(semconv.CloudAvailabilityZoneKey), + string(semconv.CloudRegionKey), + }}, + gaeModuleID: {otelKeys: []string{string(semconv.FaaSNameKey)}}, + gaeVersionID: {otelKeys: []string{string(semconv.FaaSVersionKey)}}, + }, awsEc2Instance: { instanceID: {otelKeys: []string{string(semconv.HostIDKey)}}, region: { @@ -153,27 +165,43 @@ var ( } ) -type GceResource struct { - Labels map[string]string - Type string -} - // ReadOnlyAttributes is an interface to abstract between pulling attributes from PData library or OTEL SDK. type ReadOnlyAttributes interface { GetString(string) (string, bool) } -// ResourceAttributesToMonitoredResource converts from a set of OTEL resource attributes into a -// GCP monitored resource type and label set. +// ResourceAttributesToLoggingMonitoredResource converts from a set of OTEL resource attributes into a +// GCP monitored resource type and label set for Cloud Logging. // E.g. // This may output `gce_instance` type with appropriate labels. -func ResourceAttributesToMonitoredResource(attrs ReadOnlyAttributes) *GceResource { +func ResourceAttributesToLoggingMonitoredResource(attrs ReadOnlyAttributes) *monitoredrespb.MonitoredResource { + cloudPlatform, _ := attrs.GetString(string(semconv.CloudPlatformKey)) + switch cloudPlatform { + case semconv.CloudPlatformGCPAppEngine.Value.AsString(): + return createMonitoredResource(gaeApp, attrs) + default: + return commonResourceAttributesToMonitoredResource(cloudPlatform, attrs) + } +} + +// ResourceAttributesToMonitoringMonitoredResource converts from a set of OTEL resource attributes into a +// GCP monitored resource type and label set for Cloud Monitoring +// E.g. +// This may output `gce_instance` type with appropriate labels. +func ResourceAttributesToMonitoringMonitoredResource(attrs ReadOnlyAttributes) *monitoredrespb.MonitoredResource { cloudPlatform, _ := attrs.GetString(string(semconv.CloudPlatformKey)) switch cloudPlatform { - case semconv.CloudPlatformGCPComputeEngine.Value.AsString(): - return createMonitoredResource(gceInstance, attrs) case semconv.CloudPlatformGCPAppEngine.Value.AsString(): return createMonitoredResource(gaeInstance, attrs) + default: + return commonResourceAttributesToMonitoredResource(cloudPlatform, attrs) + } +} + +func commonResourceAttributesToMonitoredResource(cloudPlatform string, attrs ReadOnlyAttributes) *monitoredrespb.MonitoredResource { + switch cloudPlatform { + case semconv.CloudPlatformGCPComputeEngine.Value.AsString(): + return createMonitoredResource(gceInstance, attrs) case semconv.CloudPlatformAWSEC2.Value.AsString(): return createMonitoredResource(awsEc2Instance, attrs) // TODO(alex-basinov): replace this string literal with semconv.CloudPlatformGCPBareMetalSolution @@ -214,7 +242,7 @@ func ResourceAttributesToMonitoredResource(attrs ReadOnlyAttributes) *GceResourc func createMonitoredResource( monitoredResourceType string, resourceAttrs ReadOnlyAttributes, -) *GceResource { +) *monitoredrespb.MonitoredResource { mappings := monitoredResourceMappings[monitoredResourceType] mrLabels := make(map[string]string, len(mappings)) @@ -231,10 +259,14 @@ func createMonitoredResource( if !ok || mrValue == "" { mrValue = mappingConfig.fallbackLiteral } - mrLabels[mrKey] = mrValue + mrLabels[mrKey] = sanitizeUTF8(mrValue) } - return &GceResource{ + return &monitoredrespb.MonitoredResource{ Type: monitoredResourceType, Labels: mrLabels, } } + +func sanitizeUTF8(s string) string { + return strings.ToValidUTF8(s, "�") +}