[exporter/signalfx] Add exclude_properties config option for limiting…

… dimension updates (#18464)

Adding a feature - These changes add exclude_properties support to* the signalfx exporter to filter which properties/dimensions are used in dimension update content. This work is based on existing Smart Agent functionality: https://github.com/signalfx/signalfx-agent/blob/main/docs/config-schema.md#propertiestoexclude

.chloggen/signalfx-exporter-filter-properties.yaml

@@ -0,0 +1,11 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: signalfxexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `exclude_properties` config option to filter dimension update property content
+
+# One or more tracking issues related to the change
+issues: [18464]

exporter/signalfxexporter/README.md

@@ -85,6 +85,18 @@ The following configuration options can also be configured:
   processor is enabled in the pipeline with one of the cloud provider detectors
   or environment variable detector setting a unique value to `host.name` attribute
   within your k8s cluster. And keep `override=true` in resourcedetection config.
+- `exclude_properties`: A list of property filters to limit dimension update content.
+  Property filters can contain any number of the following fields, supporting (negated)
+  string literals, re2 `/regex/`, and [glob](https://github.com/gobwas/glob) syntax values:
+  `dimension_name`, `dimension_value`, `property_name`, and `property_value`. For any field
+  not expressly configured for each filter object, a default catch-all value of `/^.*$/` is used
+  to allow each specified field to require a match for the filter to take effect:
+  ```yaml
+  # will filter all 'k8s.workload.name' properties from 'k8s.pod.uid' dimension updates:
+  exclude_properties:
+    - dimension_name: k8s.pod.uid
+      property_name: k8s.workload.name
+  ```
 - `nonalphanumeric_dimension_chars`: (default = `"_-."`) A string of characters 
 that are allowed to be used as a dimension key in addition to alphanumeric 
 characters. Each nonalphanumeric dimension key character that isn't in this string 

exporter/signalfxexporter/config.go

@@ -126,6 +126,10 @@ type Config struct {
 	// See ./translation/default_metrics.go for a list of metrics that are dropped by default.
 	IncludeMetrics []dpfilters.MetricFilter `mapstructure:"include_metrics"`
 
+	// ExcludeProperties defines dpfilter.PropertyFilters to prevent inclusion of
+	// properties to include with dimension updates to the SignalFx backend.
+	ExcludeProperties []dpfilters.PropertyFilter `mapstructure:"exclude_properties"`
+
 	// Correlation configuration for syncing traces service and environment to metrics.
 	Correlation *correlation.Config `mapstructure:"correlation"`
 

exporter/signalfxexporter/config_test.go

@@ -163,6 +163,26 @@ func TestLoadConfig(t *testing.T) {
 					},
 				},
 				DeltaTranslationTTL: 3600,
+				ExcludeProperties: []dpfilters.PropertyFilter{
+					{
+						PropertyName: mustStringFilter(t, "globbed*"),
+					},
+					{
+						PropertyValue: mustStringFilter(t, "!globbed*value"),
+					},
+					{
+						DimensionName: mustStringFilter(t, "globbed*"),
+					},
+					{
+						DimensionValue: mustStringFilter(t, "!globbed*value"),
+					},
+					{
+						PropertyName:   mustStringFilter(t, "globbed*"),
+						PropertyValue:  mustStringFilter(t, "!globbed*value"),
+						DimensionName:  mustStringFilter(t, "globbed*"),
+						DimensionValue: mustStringFilter(t, "!globbed*value"),
+					},
+				},
 				Correlation: &correlation.Config{
 					HTTPClientSettings: confighttp.HTTPClientSettings{
 						Endpoint: "",
@@ -483,3 +503,9 @@ func TestUnmarshalExcludeMetrics(t *testing.T) {
 		})
 	}
 }
+
+func mustStringFilter(t *testing.T, filter string) *dpfilters.StringFilter {
+	sf, err := dpfilters.NewStringFilter([]string{filter})
+	require.NoError(t, err)
+	return sf
+}

exporter/signalfxexporter/exporter.go

@@ -161,6 +161,7 @@ func (se *signalfxExporter) start(ctx context.Context, host component.Host) (err
 			// to make configurable.
 			PropertiesMaxBuffered: 10000,
 			MetricsConverter:      *se.converter,
+			ExcludeProperties:     se.config.ExcludeProperties,
 		})
 	dimClient.Start()
 

exporter/signalfxexporter/exporter_test.go

@@ -752,8 +752,11 @@ func TestConsumeMetadata(t *testing.T) {
 		name                   string
 		fields                 fields
 		args                   args
+		excludeProperties      []dpfilters.PropertyFilter
 		expectedDimensionKey   string
 		expectedDimensionValue string
+		sendDelay              int
+		shouldNotSendUpdate    bool
 	}{
 		{
 			name: "Test property updates",
@@ -769,6 +772,26 @@ func TestConsumeMetadata(t *testing.T) {
 					"tagsToRemove": nil,
 				},
 			},
+			excludeProperties: []dpfilters.PropertyFilter{
+				{
+					DimensionName:  mustStringFilter(t, "/^.*$/"),
+					DimensionValue: mustStringFilter(t, "/^.*$/"),
+					PropertyName:   mustStringFilter(t, "/^property2$/"),
+					PropertyValue:  mustStringFilter(t, "some*value"),
+				},
+				{
+					DimensionName:  mustStringFilter(t, "/^.*$/"),
+					DimensionValue: mustStringFilter(t, "/^.*$/"),
+					PropertyName:   mustStringFilter(t, "property5"),
+					PropertyValue:  mustStringFilter(t, "/^.*$/"),
+				},
+				{
+					DimensionName:  mustStringFilter(t, "*"),
+					DimensionValue: mustStringFilter(t, "*"),
+					PropertyName:   mustStringFilter(t, "/^pro[op]erty6$/"),
+					PropertyValue:  mustStringFilter(t, "property*value"),
+				},
+			},
 			args: args{
 				[]*metadata.MetadataUpdate{
 					{
@@ -777,9 +800,12 @@ func TestConsumeMetadata(t *testing.T) {
 						MetadataDelta: metadata.MetadataDelta{
 							MetadataToAdd: map[string]string{
 								"prop.erty1": "val1",
+								"property5":  "added.value",
+								"property6":  "property6.value",
 							},
 							MetadataToRemove: map[string]string{
 								"property2": "val2",
+								"property5": "removed.value",
 							},
 							MetadataToUpdate: map[string]string{
 								"prop.erty3": "val33",
@@ -805,6 +831,15 @@ func TestConsumeMetadata(t *testing.T) {
 					},
 				},
 			},
+			excludeProperties: []dpfilters.PropertyFilter{
+				{
+					// confirms tags aren't affected by excludeProperties filters
+					DimensionName:  mustStringFilter(t, "/^.*$/"),
+					DimensionValue: mustStringFilter(t, "/^.*$/"),
+					PropertyName:   mustStringFilter(t, "/^.*$/"),
+					PropertyValue:  mustStringFilter(t, "/^.*$/"),
+				},
+			},
 			args: args{
 				[]*metadata.MetadataUpdate{
 					{
@@ -893,6 +928,7 @@ func TestConsumeMetadata(t *testing.T) {
 			},
 			expectedDimensionKey:   "key",
 			expectedDimensionValue: "id",
+			sendDelay:              1,
 		},
 		{
 			name: "Test updates on dimensions with nonalphanumeric characters (other than the default allow list)",
@@ -931,14 +967,52 @@ func TestConsumeMetadata(t *testing.T) {
 			expectedDimensionKey:   "k_e_y",
 			expectedDimensionValue: "id",
 		},
+		{
+			name:                "no dimension update for empty properties",
+			shouldNotSendUpdate: true,
+			excludeProperties: []dpfilters.PropertyFilter{
+				{
+					DimensionName:  mustStringFilter(t, "key"),
+					DimensionValue: mustStringFilter(t, "/^.*$/"),
+					PropertyName:   mustStringFilter(t, "/^prop\\.erty[13]$/"),
+					PropertyValue:  mustStringFilter(t, "/^.*$/"),
+				},
+				{
+					DimensionName:  mustStringFilter(t, "*"),
+					DimensionValue: mustStringFilter(t, "id"),
+					PropertyName:   mustStringFilter(t, "property*"),
+					PropertyValue:  mustStringFilter(t, "/^.*$/"),
+				},
+			},
+			args: args{
+				[]*metadata.MetadataUpdate{
+					{
+						ResourceIDKey: "key",
+						ResourceID:    "id",
+						MetadataDelta: metadata.MetadataDelta{
+							MetadataToAdd: map[string]string{
+								"prop.erty1": "val1",
+								"property2":  "val2",
+								"property5":  "added.value",
+								"property6":  "property6.value",
+							},
+							MetadataToUpdate: map[string]string{
+								"prop.erty3": "val33",
+								"property4":  "val",
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
-		// Use WaitGroup to ensure the mocked server has encountered
-		// a request from the exporter.
-		wg := sync.WaitGroup{}
-		wg.Add(1)
-
 		t.Run(tt.name, func(t *testing.T) {
+			// Use WaitGroup to ensure the mocked server has encountered
+			// a request from the exporter.
+			wg := sync.WaitGroup{}
+			wg.Add(1)
+
 			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				b, err := io.ReadAll(r.Body)
 				assert.NoError(t, err)
@@ -974,9 +1048,10 @@ func TestConsumeMetadata(t *testing.T) {
 					APIURL:                serverURL,
 					LogUpdates:            true,
 					Logger:                logger,
-					SendDelay:             1,
+					SendDelay:             tt.sendDelay,
 					PropertiesMaxBuffered: 10,
 					MetricsConverter:      *converter,
+					ExcludeProperties:     tt.excludeProperties,
 				})
 			dimClient.Start()
 
@@ -991,9 +1066,18 @@ func TestConsumeMetadata(t *testing.T) {
 			}
 
 			err = sme.ConsumeMetadata(tt.args.metadata)
+			c := make(chan struct{})
+			go func() {
+				defer close(c)
+				wg.Wait()
+			}()
 
-			// Wait for requests to be handled by the mocked server before assertion.
-			wg.Wait()
+			select {
+			case <-c:
+			// wait 5ms longer than send delay
+			case <-time.After(time.Duration(tt.sendDelay)*time.Second + 5*time.Millisecond):
+				require.True(t, tt.shouldNotSendUpdate, "timeout waiting for response")
+			}
 
 			require.NoError(t, err)
 		})

exporter/signalfxexporter/internal/dimensions/dimclient.go

@@ -33,6 +33,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/sanitize"
 )
 
@@ -65,6 +66,8 @@ type DimensionClient struct {
 	logUpdates       bool
 	logger           *zap.Logger
 	metricsConverter translation.MetricsConverter
+	// ExcludeProperties will filter DimensionUpdate content to not submit undesired metadata.
+	ExcludeProperties []dpfilters.PropertyFilter
 }
 
 type queuedDimension struct {
@@ -81,6 +84,7 @@ type DimensionClientOptions struct {
 	SendDelay             int
 	PropertiesMaxBuffered int
 	MetricsConverter      translation.MetricsConverter
+	ExcludeProperties     []dpfilters.PropertyFilter
 }
 
 // NewDimensionClient returns a new client
@@ -104,18 +108,19 @@ func NewDimensionClient(ctx context.Context, options DimensionClientOptions) *Di
 	sender := NewReqSender(ctx, client, 20, map[string]string{"client": "dimension"})
 
 	return &DimensionClient{
-		ctx:              ctx,
-		Token:            options.Token,
-		APIURL:           options.APIURL,
-		sendDelay:        time.Duration(options.SendDelay) * time.Second,
-		delayedSet:       make(map[DimensionKey]*DimensionUpdate),
-		delayedQueue:     make(chan *queuedDimension, options.PropertiesMaxBuffered),
-		requestSender:    sender,
-		client:           client,
-		now:              time.Now,
-		logger:           options.Logger,
-		logUpdates:       options.LogUpdates,
-		metricsConverter: options.MetricsConverter,
+		ctx:               ctx,
+		Token:             options.Token,
+		APIURL:            options.APIURL,
+		sendDelay:         time.Duration(options.SendDelay) * time.Second,
+		delayedSet:        make(map[DimensionKey]*DimensionUpdate),
+		delayedQueue:      make(chan *queuedDimension, options.PropertiesMaxBuffered),
+		requestSender:     sender,
+		client:            client,
+		now:               time.Now,
+		logger:            options.Logger,
+		logUpdates:        options.LogUpdates,
+		metricsConverter:  options.MetricsConverter,
+		ExcludeProperties: options.ExcludeProperties,
 	}
 }
 
@@ -127,6 +132,10 @@ func (dc *DimensionClient) Start() {
 // acceptDimension to be sent to the API.  This will return fairly quickly and
 // won't block. If the buffer is full, the dim update will be dropped.
 func (dc *DimensionClient) acceptDimension(dimUpdate *DimensionUpdate) error {
+	if dimUpdate = dc.filterDimensionUpdate(dimUpdate); dimUpdate == nil {
+		return nil
+	}
+
 	dc.Lock()
 	defer dc.Unlock()
 
@@ -325,3 +334,29 @@ func (dc *DimensionClient) makePatchRequest(dim *DimensionUpdate) (*http.Request
 
 	return req, nil
 }
+
+func (dc *DimensionClient) filterDimensionUpdate(update *DimensionUpdate) *DimensionUpdate {
+	for _, excludeRule := range dc.ExcludeProperties {
+		if excludeRule.DimensionName.Matches(update.Name) && excludeRule.DimensionValue.Matches(update.Value) {
+			for k, v := range update.Properties {
+				if excludeRule.PropertyName.Matches(k) {
+					vVal := ""
+					if v != nil {
+						vVal = *v
+					}
+					if excludeRule.PropertyValue.Matches(vVal) {
+						delete(update.Properties, k)
+					}
+				}
+			}
+		}
+	}
+
+	// Prevent needless dimension updates if all content has been filtered.
+	// Based on https://github.com/signalfx/signalfx-agent/blob/a10f69ec6b95d7426adaf639773628fa034628b8/pkg/core/propfilters/dimfilter.go#L95
+	if len(update.Properties) == 0 && len(update.Tags) == 0 {
+		return nil
+	}
+
+	return update
+}

exporter/signalfxexporter/internal/translation/dpfilters/propertyfilter.go

@@ -0,0 +1,44 @@
+// Copyright 2021, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dpfilters // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter/internal/translation/dpfilters"
+
+// PropertyFilter is a collection of *StringFilter items used in determining if a given property (name and value)
+// should be included with a dimension update request. The default values for all fields is equivalent to the regex
+// StringFilter `/^.*$/` to match with any potential value.
+//
+// Examples:
+// Don't send any dimension updates for `k8s.pod.uid` dimension:
+// - dimension_name: "k8s.pod.uid"
+// Don't send dimension updates for any dimension with a value of `some.value`:
+// - dimension_value: "some.value"
+// Don't send dimension updates including a `some.property` property for any dimension:
+// - property_name: "some.property"
+// Don't send dimension updates including a `some.property` property with a "some.value" value for any dimension
+//   - property_name: "some.property"
+//     property_value: "some.value"
+type PropertyFilter struct {
+	// PropertyName is the (inverted) literal, regex, or globbed property name/key to not include in dimension updates
+	PropertyName *StringFilter `mapstructure:"property_name"`
+	// PropertyValue is the (inverted) literal or globbed property value to not include in dimension updates
+	PropertyValue *StringFilter `mapstructure:"property_value"`
+	// DimensionName is the (inverted) literal, regex, or globbed dimension name/key to not target for dimension updates.
+	// If there are no sub-property filters for its enclosing entry, it will disable dimension updates
+	// for this dimension name in total.
+	DimensionName *StringFilter `mapstructure:"dimension_name"`
+	// PropertyValue is the (inverted) literal, regex, or globbed dimension value to not target with a dimension update
+	// If there are no sub-property filters for its enclosing entry, it will disable dimension updates
+	// for this dimension value in total.
+	DimensionValue *StringFilter `mapstructure:"dimension_value"`
+}