diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5eed05448d48..f04bf64fae47 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -123,6 +123,7 @@ CHANGELOG* /x-pack/filebeat/input/httpjson/ @elastic/security-service-integrations /x-pack/filebeat/input/internal/httplog @elastic/security-service-integrations /x-pack/filebeat/input/internal/httpmon @elastic/security-service-integrations +/x-pack/filebeat/input/internal/private @elastic/security-service-integrations /x-pack/filebeat/input/lumberjack/ @elastic/security-service-integrations /x-pack/filebeat/input/netflow/ @elastic/sec-deployment-and-devices /x-pack/filebeat/input/o365audit/ @elastic/security-service-integrations diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index abe0164fe70d..879a4138e2e6 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -205,6 +205,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Add a configuration option for TCP/UDP network type. {issue}40407[40407] {pull}40623[40623] - Added debug logging to parquet reader in x-pack/libbeat/reader. {pull}40651[40651] - Added filebeat debug histograms for s3 object size and events per processed s3 object. {pull}40775[40775] +- Add field redaction package. {pull}40997[40997] ==== Deprecated diff --git a/x-pack/filebeat/input/internal/private/private.go b/x-pack/filebeat/input/internal/private/private.go new file mode 100644 index 000000000000..cc3046baf9cf --- /dev/null +++ b/x-pack/filebeat/input/internal/private/private.go @@ -0,0 +1,234 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package private implements field redaction in maps and structs. +package private + +import ( + "fmt" + "reflect" + "slices" + "strings" + "unsafe" +) + +const tooDeep = 100 + +var privateKey = reflect.ValueOf("private") + +// Redact returns a copy of val with any fields or map elements that have been +// marked as private removed. Fields can be marked as private by including a +// sibling string- or []string-valued field or element with the name of the +// private field. The names of fields are interpreted through the tag parameter +// if present. For example if tag is "json", the `json:""` name would be +// used, falling back to the field name if not present. The tag parameter is +// ignored for map values. +// +// If a field has a `private:...` tag, its tag value will also be used to +// determine the list of private fields. If the private tag is empty, +// `private:""`, the fields with the tag will be marked as private. Otherwise +// the comma-separated list of names with be used. The list may refer to its +// own field. +func Redact[T any](val T, tag string) (redacted T, err error) { + defer func() { + switch r := recover().(type) { + case nil: + return + case cycle: + // Make the returned type informative in all cases. + // If Redact[any](v) is called and we use the zero + // value, we would return a nil any, which is less + // informative. + redacted = reflect.New(reflect.TypeOf(val)).Elem().Interface().(T) + err = r + default: + panic(r) + } + }() + rv := reflect.ValueOf(val) + switch rv.Kind() { + case reflect.Map, reflect.Pointer, reflect.Struct: + return redact(rv, tag, 0, make(map[any]int)).Interface().(T), nil + default: + return val, nil + } +} + +func redact(v reflect.Value, tag string, depth int, seen map[any]int) reflect.Value { + switch v.Kind() { + case reflect.Pointer: + if v.IsNil() { + return v + } + if depth > tooDeep { + ident := v.Interface() + if last, ok := seen[ident]; ok && last < depth { + panic(cycle{v.Type()}) + } + seen[ident] = depth + defer delete(seen, ident) + } + return redact(v.Elem(), tag, depth+1, seen).Addr() + case reflect.Interface: + if v.IsNil() { + return v + } + return redact(v.Elem(), tag, depth+1, seen) + case reflect.Array: + if v.Len() == 0 { + return v + } + r := reflect.New(v.Type()).Elem() + for i := 0; i < v.Len(); i++ { + r.Index(i).Set(redact(v.Index(i), tag, depth+1, seen)) + } + return r + case reflect.Slice: + if v.Len() == 0 { + return v + } + if depth > tooDeep { + ident := struct { + data unsafe.Pointer + len int + }{ + v.UnsafePointer(), + v.Len(), + } + if last, ok := seen[ident]; ok && last < depth { + panic(cycle{v.Type()}) + } + seen[ident] = depth + defer delete(seen, ident) + } + r := reflect.MakeSlice(v.Type(), v.Len(), v.Cap()) + for i := 0; i < v.Len(); i++ { + r.Index(i).Set(redact(v.Index(i), tag, depth+1, seen)) + } + return r + case reflect.Map: + if v.IsNil() { + return v + } + if depth > tooDeep { + ident := v.UnsafePointer() + if last, ok := seen[ident]; ok && last < depth { + panic(cycle{v.Type()}) + } + seen[ident] = depth + defer delete(seen, ident) + } + var private []string + if privateKey.CanConvert(v.Type().Key()) { + p := v.MapIndex(privateKey.Convert(v.Type().Key())) + if p.IsValid() && p.CanInterface() { + switch p := p.Interface().(type) { + case string: + private = []string{p} + case []string: + private = p + case []any: + private = make([]string, len(p)) + for i, s := range p { + private[i] = fmt.Sprint(s) + } + } + } + } + r := reflect.MakeMap(v.Type()) + it := v.MapRange() + for it.Next() { + if slices.Contains(private, it.Key().String()) { + continue + } + r.SetMapIndex(it.Key(), redact(it.Value(), tag, depth+1, seen)) + } + return r + case reflect.Struct: + var private []string + rt := v.Type() + names := make([]string, rt.NumField()) + for i := range names { + f := rt.Field(i) + + // Look for `private:` tags. + p, ok := f.Tag.Lookup("private") + if ok { + if p != "" { + private = append(private, strings.Split(p, ",")...) + } else { + if tag == "" { + names[i] = f.Name + private = append(private, f.Name) + } else { + p = f.Tag.Get(tag) + if p != "" { + name, _, _ := strings.Cut(p, ",") + names[i] = name + private = append(private, name) + } + } + } + } + + // Look after Private fields if we are not using a tag. + if tag == "" { + names[i] = f.Name + if f.Name == "Private" { + switch p := v.Field(i).Interface().(type) { + case string: + private = append(private, p) + case []string: + private = append(private, p...) + } + } + continue + } + + // If we are using a tag, look for `tag:""` + // falling back to fields named Private if no tag is + // present. + p = f.Tag.Get(tag) + var name string + if p == "" { + name = f.Name + } else { + name, _, _ = strings.Cut(p, ",") + } + names[i] = name + if name == "private" { + switch p := v.Field(i).Interface().(type) { + case string: + private = append(private, p) + case []string: + private = append(private, p...) + } + } + } + + r := reflect.New(v.Type()).Elem() + for i := 0; i < v.NumField(); i++ { + f := v.Field(i) + if f.IsZero() || !rt.Field(i).IsExported() { + continue + } + if slices.Contains(private, names[i]) { + continue + } + if r.Field(i).CanSet() { + r.Field(i).Set(redact(f, tag, depth+1, seen)) + } + } + return r + } + return v +} + +type cycle struct { + typ reflect.Type +} + +func (e cycle) Error() string { + return fmt.Sprintf("cycle including %s", e.typ) +} diff --git a/x-pack/filebeat/input/internal/private/private_test.go b/x-pack/filebeat/input/internal/private/private_test.go new file mode 100644 index 000000000000..cc34ac290e8f --- /dev/null +++ b/x-pack/filebeat/input/internal/private/private_test.go @@ -0,0 +1,291 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package private + +import ( + "bytes" + "encoding/json" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" +) + +type redactTest struct { + name string + in any + tag string + want any + wantErr error +} + +var redactTests = []redactTest{ + { + name: "map_string", + in: map[string]any{ + "private": "secret", + "secret": "1", + "not_secret": "2", + }, + want: map[string]any{ + "private": "secret", + "not_secret": "2", + }, + }, + { + name: "map_string_inner", + in: map[string]any{ + "inner": map[string]any{ + "private": "secret", + "secret": "1", + "not_secret": "2", + }}, + want: map[string]any{ + "inner": map[string]any{ + "private": "secret", + "not_secret": "2", + }}, + }, + { + name: "map_slice", + in: map[string]any{ + "private": []string{"secret"}, + "secret": "1", + "not_secret": "2", + }, + want: map[string]any{ + "private": []string{"secret"}, + "not_secret": "2", + }, + }, + { + name: "map_cycle", + in: func() any { + m := map[string]any{ + "private": "secret", + "secret": "1", + "not_secret": "2", + } + m["loop"] = m + return m + }(), + want: map[string]any(nil), + wantErr: cycle{reflect.TypeOf(map[string]any(nil))}, + }, + func() redactTest { + type s struct { + Private string + Secret string + NotSecret string + } + return redactTest{ + name: "struct_string", + in: s{ + Private: "Secret", + Secret: "1", + NotSecret: "2", + }, + tag: "", + want: s{ + Private: "Secret", + NotSecret: "2", + }, + } + }(), + func() redactTest { + type s struct { + Private []string + Secret string + NotSecret string + } + return redactTest{ + name: "struct_slice", + in: s{ + Private: []string{"Secret"}, + Secret: "1", + NotSecret: "2", + }, + tag: "", + want: s{ + Private: []string{"Secret"}, + NotSecret: "2", + }, + } + }(), + func() redactTest { + type s struct { + Private string + Secret string + NotSecret string + Loop *s + } + v := s{ + Private: "Secret", + Secret: "1", + NotSecret: "2", + } + v.Loop = &v + return redactTest{ + name: "struct_loop", + in: v, + tag: "", + want: s{}, + wantErr: cycle{reflect.TypeOf(&s{})}, + } + }(), + func() redactTest { + type s struct { + Private string `json:"private"` + Secret string `json:"secret"` + NotSecret string `json:"not_secret"` + } + return redactTest{ + name: "struct_string_json", + in: s{ + Private: "secret", + Secret: "1", + NotSecret: "2", + }, + tag: "json", + want: s{ + Private: "secret", + NotSecret: "2", + }, + } + }(), + func() redactTest { + type s struct { + Private struct{} `private:"secret"` + Secret string `json:"secret"` + NotSecret string `json:"not_secret"` + } + return redactTest{ + name: "struct_string_on_tag_json", + in: s{ + Secret: "1", + NotSecret: "2", + }, + tag: "json", + want: s{ + NotSecret: "2", + }, + } + }(), + func() redactTest { + type s struct { + Private struct{} `private:"secret1,secret2"` + Secret1 string `json:"secret1"` + Secret2 string `json:"secret2"` + NotSecret string `json:"not_secret"` + } + return redactTest{ + name: "struct_string_list_on_tag_json", + in: s{ + Secret1: "1", + Secret2: "1", + NotSecret: "2", + }, + tag: "json", + want: s{ + NotSecret: "2", + }, + } + }(), + func() redactTest { + type s struct { + Private string `json:"private"` + Secret string + NotSecret string `json:"not_secret"` + } + return redactTest{ + name: "struct_string_json_missing_tag", + in: s{ + Private: "Secret", + Secret: "1", + NotSecret: "2", + }, + tag: "json", + want: s{ + Private: "Secret", + NotSecret: "2", + }, + } + }(), + func() redactTest { + type s struct { + Private []string `json:"private"` + Secret string `json:"secret"` + NotSecret string `json:"not_secret"` + } + return redactTest{ + name: "struct_slice_json", + in: s{ + Private: []string{"secret"}, + Secret: "1", + NotSecret: "2", + }, + tag: "json", + want: s{ + Private: []string{"secret"}, + NotSecret: "2", + }, + } + }(), + func() redactTest { + type s struct { + Private string `json:"private"` + Secret string `json:"secret"` + NotSecret string `json:"not_secret"` + Loop *s `json:"loop"` + } + v := s{ + Private: "secret", + Secret: "1", + NotSecret: "2", + } + v.Loop = &v + return redactTest{ + name: "struct_loop_json", + in: v, + tag: "json", + want: s{}, + wantErr: cycle{reflect.TypeOf(&s{})}, + } + }(), +} + +func TestRedact(t *testing.T) { + allow := cmp.AllowUnexported() + + for _, test := range redactTests { + t.Run(test.name, func(t *testing.T) { + var before []byte + _, isCycle := test.wantErr.(cycle) + if !isCycle { + var err error + before, err = json.Marshal(test.in) + if err != nil { + t.Fatalf("failed to get before state: %v", err) + } + } + got, err := Redact(test.in, test.tag) + if err != test.wantErr { + t.Fatalf("unexpected error from Redact: %v", err) + } + if !isCycle { + after, err := json.Marshal(test.in) + if err != nil { + t.Fatalf("failed to get after state: %v", err) + } + if !bytes.Equal(before, after) { + t.Errorf("unexpected change in input:\n---:\n+++:\n%s", cmp.Diff(before, after)) + } + } + if !cmp.Equal(test.want, got, allow) { + t.Errorf("unexpected paths:\n--- want:\n+++ got:\n%s", cmp.Diff(test.want, got, allow)) + } + }) + } +}