Additional masking functions to be applied to log fields

.changelog/87.txt

@@ -0,0 +1,11 @@
+```release-note:feature
+tflog: Added `MaskAllFieldValuesRegexes()`, `MaskAllFieldValuesStrings()`, `MaskLogRegexes()` and `MaskLogStrings()` functions, which extend further the log masking filtering, for the provider root logger
+```
+
+```release-note:feature
+tflog: Added `SubsystemMaskAllFieldValuesRegexes()`, `SubsystemMaskAllFieldValuesStrings()`, `SubsystemMaskLogRegexes()` and `SubsystemMaskLogStrings()` functions, which extend further the log masking filtering, for provider subsystem loggers
+```
+
+```release-note:feature
+tfsdklog: Same functions added to the `tflog` package
+```

internal/logging/filtering.go

@@ -48,6 +48,7 @@ func (lo LoggerOpts) ShouldOmit(msg *string, fieldMaps ...map[string]interface{}
 //
 // Note that the given input is changed-in-place by this method.
 func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{}) {
+	// Replace any log field value with the corresponding field key equal to the configured strings
 	if len(lo.MaskFieldValuesWithFieldKeys) > 0 {
 		for _, k := range lo.MaskFieldValuesWithFieldKeys {
 			for _, f := range fieldMaps {
@@ -60,16 +61,44 @@ func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{})
 		}
 	}
 
-	// Replace any part of the log message matching any of the configured regexp,
-	// with a masking replacement string
+	// Replace any part of any log field matching any of the configured regexp
+	if len(lo.MaskAllFieldValuesRegexes) > 0 {
+		for _, r := range lo.MaskAllFieldValuesRegexes {
+			for _, f := range fieldMaps {
+				for fk, fv := range f {
+					// Can apply the regexp replacement, only if the field value is indeed a string
+					fvStr, ok := fv.(string)
+					if ok {
+						f[fk] = r.ReplaceAllString(fvStr, logMaskingReplacementString)
+					}
+				}
+			}
+		}
+	}
+
+	// Replace any part of any log field matching any of the configured strings
+	if len(lo.MaskAllFieldValuesStrings) > 0 {
+		for _, s := range lo.MaskAllFieldValuesStrings {
+			for _, f := range fieldMaps {
+				for fk, fv := range f {
+					// Can apply the regexp replacement, only if the field value is indeed a string
+					fvStr, ok := fv.(string)
+					if ok {
+						f[fk] = strings.ReplaceAll(fvStr, s, logMaskingReplacementString)
+					}
+				}
+			}
+		}
+	}
+
+	// Replace any part of the log message matching any of the configured regexp
 	if len(lo.MaskMessageRegexes) > 0 {
 		for _, r := range lo.MaskMessageRegexes {
 			*msg = r.ReplaceAllString(*msg, logMaskingReplacementString)
 		}
 	}
 
-	// Replace any part of the log message equal to any of the configured strings,
-	// with a masking replacement string
+	// Replace any part of the log message equal to any of the configured strings
 	if len(lo.MaskMessageStrings) > 0 {
 		for _, s := range lo.MaskMessageStrings {
 			*msg = strings.ReplaceAll(*msg, s, logMaskingReplacementString)

internal/logging/filtering_test.go

@@ -215,6 +215,50 @@ func TestApplyMask(t *testing.T) {
 				},
 			},
 		},
+		"mask-log-and-fields-matching-regexp": {
+			lOpts: logging.LoggerOpts{
+				MaskMessageRegexes: []*regexp.Regexp{
+					regexp.MustCompile("incorrectly configured BAZ"),
+				},
+				MaskAllFieldValuesRegexes: []*regexp.Regexp{
+					regexp.MustCompile("v1|v2"),
+				},
+			},
+			msg: testLogMsg,
+			fieldMaps: []map[string]interface{}{
+				{
+					"k1": "v1 with some extra text",
+					"k2": "v2 with more extra text",
+				},
+			},
+			expectedMsg: "System FOO has caused error BAR because of ***",
+			expectedFieldMaps: []map[string]interface{}{
+				{
+					"k1": "*** with some extra text",
+					"k2": "*** with more extra text",
+				},
+			},
+		},
+		"mask-log-and-fields-matching-strings": {
+			lOpts: logging.LoggerOpts{
+				MaskMessageStrings:        []string{"FOO", "BAR", "BAZ"},
+				MaskAllFieldValuesStrings: []string{"v1", "v2"},
+			},
+			msg: testLogMsg,
+			fieldMaps: []map[string]interface{}{
+				{
+					"k1": "v1 with some extra text",
+					"k2": "v2 with more extra text",
+				},
+			},
+			expectedMsg: "System *** has caused error *** because of incorrectly configured ***",
+			expectedFieldMaps: []map[string]interface{}{
+				{
+					"k1": "*** with some extra text",
+					"k2": "*** with more extra text",
+				},
+			},
+		},
 		"mask-log-by-key-and-matching-regexp": {
 			lOpts: logging.LoggerOpts{
 				MaskMessageRegexes:           []*regexp.Regexp{regexp.MustCompile("incorrectly configured BAZ")},

internal/logging/options.go

@@ -56,9 +56,9 @@ type LoggerOpts struct {
 	//
 	//   OmitLogWithFieldKeys = `['foo', 'baz']`
 	//
-	//   log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }`   -> omitted
-	//   log2 = `{ msg = "...", fields = { 'bar', '...' }`                 -> printed
-	//   log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }`  -> omitted
+	//   log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }`  -> omitted
+	//   log2 = `{ msg = "...", fields = { 'bar': '...' }`                -> printed
+	//   log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }`  -> omitted
 	//
 	OmitLogWithFieldKeys []string
 
@@ -95,33 +95,62 @@ type LoggerOpts struct {
 	//
 	//   MaskFieldValuesWithFieldKeys = `['foo', 'baz']`
 	//
-	//   log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }`   -> masked value
-	//   log2 = `{ msg = "...", fields = { 'bar', '...' }`                 -> as-is value
-	//   log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }`  -> masked value
+	//   log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }`  -> masked value
+	//   log2 = `{ msg = "...", fields = { 'bar': '...' }`                -> as-is value
+	//   log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }`  -> masked value
 	//
 	MaskFieldValuesWithFieldKeys []string
 
+	// MaskAllFieldValuesRegexes indicates that the logger should replace, within
+	// all the log field values, the portion matching one of the given *regexp.Regexp.
+	//
+	// Note that the replacement will happen, only for field values that are of type string.
+	//
+	// Example:
+	//
+	//   MaskAllFieldValuesRegexes = `[regexp.MustCompile("(foo|bar)")]`
+	//
+	//   log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }`  -> masked value
+	//   log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }`  -> as-is value
+	//   log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }`                    -> masked value
+	//
+	MaskAllFieldValuesRegexes []*regexp.Regexp
+
+	// MaskAllFieldValuesStrings indicates that the logger should replace, within
+	// all the log field values, the portion equal to one of the given strings.
+	//
+	// Note that the replacement will happen, only for field values that are of type string.
+	//
+	// Example:
+	//
+	//   MaskAllFieldValuesStrings = `['foo', 'baz']`
+	//
+	//   log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': 'bar', 'k3': '***' }`  -> masked value
+	//   log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': '***' }`  -> as-is value
+	//   log2 = `{ msg = "...", fields = { 'k1': '*** bar ***' }`                    -> masked value
+	MaskAllFieldValuesStrings []string
+
 	// MaskMessageRegexes indicates that the logger should replace, within
 	// a log message, the portion matching one of the given *regexp.Regexp.
 	//
 	// Example:
 	//
 	//   MaskMessageRegexes = `[regexp.MustCompile("(foo|bar)")]`
 	//
-	//   log1 = `{ msg = "banana apple ***", fields = {...}`     -> masked portion
+	//   log1 = `{ msg = "banana apple ***", fields = { ... }`   -> masked portion
 	//   log2 = `{ msg = "pineapple mango", fields = {...}`      -> as-is
 	//   log3 = `{ msg = "pineapple mango ***", fields = {...}`  -> masked portion
 	//
 	MaskMessageRegexes []*regexp.Regexp
 
 	// MaskMessageStrings indicates that the logger should replace, within
-	// a log message, the portion matching one of the given strings.
+	// a log message, the portion equal to one of the given strings.
 	//
 	// Example:
 	//
 	//   MaskMessageStrings = `['foo', 'bar']`
 	//
-	//   log1 = `{ msg = "banana apple ***", fields = {...}`     -> masked portion
+	//   log1 = `{ msg = "banana apple ***", fields = { ... }`   -> masked portion
 	//   log2 = `{ msg = "pineapple mango", fields = {...}`      -> as-is
 	//   log3 = `{ msg = "pineapple mango ***", fields = {...}`  -> masked portion
 	//
@@ -266,6 +295,22 @@ func WithMaskFieldValuesWithFieldKeys(keys ...string) Option {
 	}
 }
 
+// WithMaskAllFieldValuesRegexes appends keys to the LoggerOpts.MaskAllFieldValuesRegexes field.
+func WithMaskAllFieldValuesRegexes(expressions ...*regexp.Regexp) Option {
+	return func(l LoggerOpts) LoggerOpts {
+		l.MaskAllFieldValuesRegexes = append(l.MaskAllFieldValuesRegexes, expressions...)
+		return l
+	}
+}
+
+// WithMaskAllFieldValuesStrings appends keys to the LoggerOpts.MaskAllFieldValuesStrings field.
+func WithMaskAllFieldValuesStrings(matchingStrings ...string) Option {
+	return func(l LoggerOpts) LoggerOpts {
+		l.MaskAllFieldValuesStrings = append(l.MaskAllFieldValuesStrings, matchingStrings...)
+		return l
+	}
+}
+
 // WithMaskMessageRegexes appends *regexp.Regexp to the LoggerOpts.MaskMessageRegexes field.
 func WithMaskMessageRegexes(expressions ...*regexp.Regexp) Option {
 	return func(l LoggerOpts) LoggerOpts {

tflog/provider.go

@@ -146,9 +146,9 @@ func Error(ctx context.Context, msg string, additionalFields ...map[string]inter
 //
 //   configuration = `['foo', 'baz']`
 //
-//   log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }`   -> omitted
-//   log2 = `{ msg = "...", fields = { 'bar', '...' }`                 -> printed
-//   log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }`  -> omitted
+//   log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }`  -> omitted
+//   log2 = `{ msg = "...", fields = { 'bar': '...' }`                -> printed
+//   log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }`  -> omitted
 //
 func OmitLogWithFieldKeys(ctx context.Context, keys ...string) context.Context {
 	lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
@@ -214,9 +214,9 @@ func OmitLogWithMessageStrings(ctx context.Context, matchingStrings ...string) c
 //
 //   configuration = `['foo', 'baz']`
 //
-//   log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }`   -> masked value
-//   log2 = `{ msg = "...", fields = { 'bar', '...' }`                 -> as-is value
-//   log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }`  -> masked value
+//   log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }`  -> masked value
+//   log2 = `{ msg = "...", fields = { 'bar': '...' }`                -> as-is value
+//   log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }`  -> masked value
 //
 func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.Context {
 	lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
@@ -226,9 +226,59 @@ func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.C
 	return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
 }
 
+// MaskAllFieldValuesRegexes returns a new context.Context that has a modified logger
+// that masks (replaces) with asterisks (`***`) all field value substrings,
+// matching one of the given *regexp.Regexp.
+//
+// Note that the replacement will happen, only for field values that are of type string.
+//
+// Each call to this function is additive:
+// the regexp to mask by are added to the existing configuration.
+//
+// Example:
+//
+//   configuration = `[regexp.MustCompile("(foo|bar)")]`
+//
+//   log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }`  -> masked value
+//   log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }`  -> as-is value
+//   log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }`                    -> masked value
+//
+func MaskAllFieldValuesRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context {
+	lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
+
+	lOpts = logging.WithMaskAllFieldValuesRegexes(expressions...)(lOpts)
+
+	return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
+}
+
+// MaskAllFieldValuesStrings returns a new context.Context that has a modified logger
+// that masks (replaces) with asterisks (`***`) all field value substrings,
+// equal to one of the given strings.
+//
+// Note that the replacement will happen, only for field values that are of type string.
+//
+// Each call to this function is additive:
+// the regexp to mask by are added to the existing configuration.
+//
+// Example:
+//
+//   configuration = `[regexp.MustCompile("(foo|bar)")]`
+//
+//   log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }`  -> masked value
+//   log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }`  -> as-is value
+//   log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }`                    -> masked value
+//
+func MaskAllFieldValuesStrings(ctx context.Context, matchingStrings ...string) context.Context {
+	lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
+
+	lOpts = logging.WithMaskAllFieldValuesStrings(matchingStrings...)(lOpts)
+
+	return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
+}
+
 // MaskMessageRegexes returns a new context.Context that has a modified logger
-// that masks (replaces) with asterisks (`***`) all message substrings matching one
-// of the given strings.
+// that masks (replaces) with asterisks (`***`) all message substrings,
+// matching one of the given *regexp.Regexp.
 //
 // Each call to this function is additive:
 // the regexp to mask by are added to the existing configuration.
@@ -237,7 +287,7 @@ func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.C
 //
 //   configuration = `[regexp.MustCompile("(foo|bar)")]`
 //
-//   log1 = `{ msg = "banana apple ***", fields = {...}`     -> masked portion
+//   log1 = `{ msg = "banana apple ***", fields = { ... }`   -> masked portion
 //   log2 = `{ msg = "pineapple mango", fields = {...}`      -> as-is
 //   log3 = `{ msg = "pineapple mango ***", fields = {...}`  -> masked portion
 //
@@ -250,8 +300,8 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont
 }
 
 // MaskMessageStrings returns a new context.Context that has a modified logger
-// that masks (replace) with asterisks (`***`) all message substrings equal to one
-// of the given strings.
+// that masks (replace) with asterisks (`***`) all message substrings,
+// equal to one of the given strings.
 //
 // Each call to this function is additive:
 // the string to mask by are added to the existing configuration.
@@ -260,9 +310,9 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont
 //
 //   configuration = `['foo', 'bar']`
 //
-//   log1 = `{ msg = "banana apple ***", fields = {...}`     -> masked portion
-//   log2 = `{ msg = "pineapple mango", fields = {...}`      -> as-is
-//   log3 = `{ msg = "pineapple mango ***", fields = {...}`  -> masked portion
+//   log1 = `{ msg = "banana apple ***", fields = { 'k1': 'foo, bar, baz' }`  -> masked portion
+//   log2 = `{ msg = "pineapple mango", fields = {...}`                       -> as-is
+//   log3 = `{ msg = "pineapple mango ***", fields = {...}`                   -> masked portion
 //
 func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.Context {
 	lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
@@ -271,3 +321,15 @@ func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.
 
 	return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
 }
+
+// MaskLogRegexes is a shortcut to invoke MaskMessageRegexes and MaskAllFieldValuesRegexes using the same input.
+// Refer to those functions for details.
+func MaskLogRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context {
+	return MaskMessageRegexes(MaskAllFieldValuesRegexes(ctx, expressions...), expressions...)
+}
+
+// MaskLogStrings is a shortcut to invoke MaskMessageStrings and MaskAllFieldValuesStrings using the same input.
+// Refer to those functions for details.
+func MaskLogStrings(ctx context.Context, matchingStrings ...string) context.Context {
+	return MaskMessageStrings(MaskAllFieldValuesStrings(ctx, matchingStrings...), matchingStrings...)
+}