Skip to content

Commit

Permalink
x-pack/filebeat/input/internal/private: add global path redaction
Browse files Browse the repository at this point in the history 
The original package was not designed to deal with values that could not
have sibling fields to mark privacy. This adds the capacity to redact
such types.
  • Loading branch information
@efd6
efd6 committed Sep 30, 2024
1 parent a628d62 commit 5cd84f6
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 18 deletions.
68 changes: 51 additions & 17 deletions x-pack/filebeat/input/internal/private/private.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,17 @@ var privateKey = reflect.ValueOf("private")
// used, falling back to the field name if not present. The tag parameter is
// ignored for map values.
//
// The global parameter indicates a set of dot-separated paths to redact. Paths
// originate at the root of val. If global is used, the resultin redaction is on
// the union of the fields redacted with tags and the fields redacted with the
// global paths.
//
// If a field has a `private:...` tag, its tag value will also be used to
// determine the list of private fields. If the private tag is empty,
// `private:""`, the fields with the tag will be marked as private. Otherwise
// the comma-separated list of names with be used. The list may refer to its
// own field.
func Redact[T any](val T, tag string) (redacted T, err error) {
func Redact[T any](val T, tag string, global []string) (redacted T, err error) {
defer func() {
switch r := recover().(type) {
case nil:
Expand All @@ -49,13 +54,13 @@ func Redact[T any](val T, tag string) (redacted T, err error) {
rv := reflect.ValueOf(val)
switch rv.Kind() {
case reflect.Map, reflect.Pointer, reflect.Struct:
return redact(rv, tag, 0, make(map[any]int)).Interface().(T), nil
return redact(rv, tag, slices.Clone(global), 0, make(map[any]int)).Interface().(T), nil
default:
return val, nil
}
}

func redact(v reflect.Value, tag string, depth int, seen map[any]int) reflect.Value {
func redact(v reflect.Value, tag string, global []string, depth int, seen map[any]int) reflect.Value {
switch v.Kind() {
case reflect.Pointer:
if v.IsNil() {
Expand All @@ -69,19 +74,19 @@ func redact(v reflect.Value, tag string, depth int, seen map[any]int) reflect.Va
seen[ident] = depth
defer delete(seen, ident)
}
return redact(v.Elem(), tag, depth+1, seen).Addr()
return redact(v.Elem(), tag, global, depth+1, seen).Addr()
case reflect.Interface:
if v.IsNil() {
return v
}
return redact(v.Elem(), tag, depth+1, seen)
return redact(v.Elem(), tag, global, depth+1, seen)
case reflect.Array:
if v.Len() == 0 {
return v
}
r := reflect.New(v.Type()).Elem()
for i := 0; i < v.Len(); i++ {
r.Index(i).Set(redact(v.Index(i), tag, depth+1, seen))
r.Index(i).Set(redact(v.Index(i), tag, global, depth+1, seen))
}
return r
case reflect.Slice:
Expand All @@ -104,7 +109,7 @@ func redact(v reflect.Value, tag string, depth int, seen map[any]int) reflect.Va
}
r := reflect.MakeSlice(v.Type(), v.Len(), v.Cap())
for i := 0; i < v.Len(); i++ {
r.Index(i).Set(redact(v.Index(i), tag, depth+1, seen))
r.Index(i).Set(redact(v.Index(i), tag, global, depth+1, seen))
}
return r
case reflect.Map:
Expand All @@ -119,34 +124,34 @@ func redact(v reflect.Value, tag string, depth int, seen map[any]int) reflect.Va
seen[ident] = depth
defer delete(seen, ident)
}
var private []string
private := nextStep(global)
if privateKey.CanConvert(v.Type().Key()) {
p := v.MapIndex(privateKey.Convert(v.Type().Key()))
if p.IsValid() && p.CanInterface() {
switch p := p.Interface().(type) {
case string:
private = []string{p}
private = append(private, p)
case []string:
private = p
private = append(private, p...)
case []any:
private = make([]string, len(p))
for i, s := range p {
private[i] = fmt.Sprint(s)
for _, s := range p {
private = append(private, fmt.Sprint(s))
}
}
}
}
r := reflect.MakeMap(v.Type())
it := v.MapRange()
for it.Next() {
if slices.Contains(private, it.Key().String()) {
name := it.Key().String()
if slices.Contains(private, name) {
continue
}
r.SetMapIndex(it.Key(), redact(it.Value(), tag, depth+1, seen))
r.SetMapIndex(it.Key(), redact(it.Value(), tag, nextPath(name, global), depth+1, seen))
}
return r
case reflect.Struct:
var private []string
private := nextStep(global)
rt := v.Type()
names := make([]string, rt.NumField())
for i := range names {
Expand Down Expand Up @@ -217,14 +222,43 @@ func redact(v reflect.Value, tag string, depth int, seen map[any]int) reflect.Va
continue
}
if r.Field(i).CanSet() {
r.Field(i).Set(redact(f, tag, depth+1, seen))
r.Field(i).Set(redact(f, tag, nextPath(names[i], global), depth+1, seen))
}
}
return r
}
return v
}

func nextStep(global []string) (private []string) {
if len(global) == 0 {
return nil
}
private = make([]string, 0, len(global))
for _, s := range global {
key, _, more := strings.Cut(s, ".")
if !more {
private = append(private, key)
}
}
return private
}

func nextPath(step string, global []string) []string {
if len(global) == 0 {
return nil
}
step += "."
next := make([]string, 0, len(global))
for _, s := range global {
if !strings.HasPrefix(s, step) {
continue
}
next = append(next, s[len(step):])
}
return next
}

type cycle struct {
typ reflect.Type
}
Expand Down
77 changes: 76 additions & 1 deletion x-pack/filebeat/input/internal/private/private_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package private
import (
"bytes"
"encoding/json"
"net/url"
"reflect"
"testing"

Expand All @@ -17,6 +18,7 @@ type redactTest struct {
name string
in any
tag string
global []string
want any
wantErr error
}
Expand Down Expand Up @@ -48,6 +50,79 @@ var redactTests = []redactTest{
"not_secret": "2",
}},
},
{
name: "map_string_inner_global",
in: map[string]any{
"inner": map[string]any{
"secret": "1",
"not_secret": "2",
}},
global: []string{"inner.secret"},
want: map[string]any{
"inner": map[string]any{
"not_secret": "2",
}},
},
{
name: "map_string_inner_next_inner_global",
in: map[string]any{
"inner": map[string]any{
"next_inner": map[string]any{
"secret": "1",
"not_secret": "2",
},
}},
global: []string{"inner.next_inner.secret"},
want: map[string]any{
"inner": map[string]any{
"next_inner": map[string]any{
"not_secret": "2",
},
}},
},
{
name: "map_string_inner_next_inner_params_global",
in: map[string]any{
"inner": map[string]any{
"next_inner": map[string]any{
"headers": url.Values{
"secret": []string{"1"},
"not_secret": []string{"2"},
},
"not_secret": "2",
},
}},
global: []string{"inner.next_inner.headers.secret"},
want: map[string]any{
"inner": map[string]any{
"next_inner": map[string]any{
"headers": url.Values{
"not_secret": []string{"2"},
},
"not_secret": "2",
},
}},
},
{
name: "map_string_inner_next_inner_params_global_internal",
in: map[string]any{
"inner": map[string]any{
"next_inner": map[string]any{
"headers": url.Values{
"secret": []string{"1"},
"not_secret": []string{"2"},
},
"not_secret": "2",
},
}},
global: []string{"inner.next_inner.headers"},
want: map[string]any{
"inner": map[string]any{
"next_inner": map[string]any{
"not_secret": "2",
},
}},
},
{
name: "map_slice",
in: map[string]any{
Expand Down Expand Up @@ -270,7 +345,7 @@ func TestRedact(t *testing.T) {
t.Fatalf("failed to get before state: %v", err)
}
}
got, err := Redact(test.in, test.tag)
got, err := Redact(test.in, test.tag, test.global)
if err != test.wantErr {

Check failure on line 349 in x-pack/filebeat/input/internal/private/private_test.go

View workflow job for this annotation

GitHub Actions / lint (darwin) 

comparing with != will fail on wrapped errors. Use errors.Is to check for a specific error (errorlint)
t.Fatalf("unexpected error from Redact: %v", err)
}
Expand Down

0 comments on commit 5cd84f6

Please sign in to comment.