From d330f612cb3c8e550491f4531e9b31948791dedc Mon Sep 17 00:00:00 2001 From: whisky-is-life Date: Wed, 13 Sep 2023 19:29:57 +0900 Subject: [PATCH] feat(configx): add HTTP header log redaction config --- configx/stub/from-files/config.schema.json | 10 +++++ configx/stub/hydra/config.schema.json | 10 +++++ configx/stub/kratos/config.schema.json | 10 +++++ configx/stub/multi/config.schema.json | 10 +++++ logrusx/config.schema.json | 10 +++++ logrusx/helper.go | 14 ++++--- logrusx/logrus.go | 48 +++++++++++++++------- logrusx/logrus_test.go | 13 ++++++ 8 files changed, 104 insertions(+), 21 deletions(-) diff --git a/configx/stub/from-files/config.schema.json b/configx/stub/from-files/config.schema.json index 75847b2f..b7a9511e 100644 --- a/configx/stub/from-files/config.schema.json +++ b/configx/stub/from-files/config.schema.json @@ -850,6 +850,16 @@ "title": "Sensitive log value redaction text", "description": "Text to use, when redacting sensitive log value." }, + "redactable_http_headers": { + "type": "array", + "title": "Redactable HTTP headers", + "description": "Additional HTTP headers to redact in the logs.", + "items": { + "type": "string" + }, + "examples": [["X-Auth-Token", "X-Authenticated-User"]], + "uniqueItems": true + }, "format": { "type": "string", "enum": ["json", "text"] diff --git a/configx/stub/hydra/config.schema.json b/configx/stub/hydra/config.schema.json index e2ce4aff..80fc74b4 100644 --- a/configx/stub/hydra/config.schema.json +++ b/configx/stub/hydra/config.schema.json @@ -203,6 +203,16 @@ "title": "Sensitive log value redaction text", "description": "Text to use, when redacting sensitive log value." }, + "redactable_http_headers": { + "type": "array", + "title": "Redactable HTTP headers", + "description": "Additional HTTP headers to redact in the logs.", + "items": { + "type": "string" + }, + "examples": [["X-Auth-Token", "X-Authenticated-User"]], + "uniqueItems": true + }, "format": { "type": "string", "description": "Sets the log format.", diff --git a/configx/stub/kratos/config.schema.json b/configx/stub/kratos/config.schema.json index 75847b2f..b7a9511e 100644 --- a/configx/stub/kratos/config.schema.json +++ b/configx/stub/kratos/config.schema.json @@ -850,6 +850,16 @@ "title": "Sensitive log value redaction text", "description": "Text to use, when redacting sensitive log value." }, + "redactable_http_headers": { + "type": "array", + "title": "Redactable HTTP headers", + "description": "Additional HTTP headers to redact in the logs.", + "items": { + "type": "string" + }, + "examples": [["X-Auth-Token", "X-Authenticated-User"]], + "uniqueItems": true + }, "format": { "type": "string", "enum": ["json", "text"] diff --git a/configx/stub/multi/config.schema.json b/configx/stub/multi/config.schema.json index 75847b2f..b7a9511e 100644 --- a/configx/stub/multi/config.schema.json +++ b/configx/stub/multi/config.schema.json @@ -850,6 +850,16 @@ "title": "Sensitive log value redaction text", "description": "Text to use, when redacting sensitive log value." }, + "redactable_http_headers": { + "type": "array", + "title": "Redactable HTTP headers", + "description": "Additional HTTP headers to redact in the logs.", + "items": { + "type": "string" + }, + "examples": [["X-Auth-Token", "X-Authenticated-User"]], + "uniqueItems": true + }, "format": { "type": "string", "enum": ["json", "text"] diff --git a/logrusx/config.schema.json b/logrusx/config.schema.json index b5b963e8..78146693 100644 --- a/logrusx/config.schema.json +++ b/logrusx/config.schema.json @@ -29,6 +29,16 @@ "type": "string", "title": "Sensitive log value redaction text", "description": "Text to use, when redacting sensitive log value." + }, + "redactable_http_headers": { + "type": "array", + "title": "Redactable HTTP headers", + "description": "Additional HTTP headers to redact in the logs.", + "items": { + "type": "string" + }, + "examples": [["X-Auth-Token", "X-Authenticated-User"]], + "uniqueItems": true } }, "additionalProperties": false diff --git a/logrusx/helper.go b/logrusx/helper.go index 51bca86b..93f6f058 100644 --- a/logrusx/helper.go +++ b/logrusx/helper.go @@ -21,15 +21,17 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/ory/x/errorsx" + "github.com/ory/x/stringslice" ) type Logger struct { *logrus.Entry - leakSensitive bool - redactionText string - opts []Option - name string - version string + leakSensitive bool + redactionText string + redactableHTTPHeaders []string + opts []Option + name string + version string } var opts = otelhttptrace.WithPropagators(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) @@ -59,7 +61,7 @@ func (l *Logger) HTTPHeadersRedacted(h http.Header) map[string]interface{} { for key, value := range h { keyLower := strings.ToLower(key) - if keyLower == "authorization" || keyLower == "cookie" || keyLower == "set-cookie" { + if keyLower == "authorization" || keyLower == "cookie" || keyLower == "set-cookie" || stringslice.HasI(l.redactableHTTPHeaders, keyLower) { headers[keyLower] = l.maybeRedact(value) } else { headers[keyLower] = h.Get(key) diff --git a/logrusx/logrus.go b/logrusx/logrus.go index 0cbae82b..b650b876 100644 --- a/logrusx/logrus.go +++ b/logrusx/logrus.go @@ -20,22 +20,24 @@ import ( type ( options struct { - l *logrus.Logger - level *logrus.Level - formatter logrus.Formatter - format string - reportCaller bool - exitFunc func(int) - leakSensitive bool - redactionText string - hooks []logrus.Hook - c configurator + l *logrus.Logger + level *logrus.Level + formatter logrus.Formatter + format string + reportCaller bool + exitFunc func(int) + leakSensitive bool + redactionText string + redactableHTTPHeaders []string + hooks []logrus.Hook + c configurator } Option func(*options) nullConfigurator struct{} configurator interface { Bool(key string) bool String(key string) string + Strings(key string) []string } ) @@ -178,6 +180,12 @@ func RedactionText(text string) Option { } } +func RedactableHTTPHeaders(headers []string) Option { + return func(o *options) { + o.redactableHTTPHeaders = headers + } +} + func (c *nullConfigurator) Bool(_ string) bool { return false } @@ -186,6 +194,10 @@ func (c *nullConfigurator) String(_ string) string { return "" } +func (c *nullConfigurator) Strings(_ string) []string { + return []string{} +} + func newOptions(opts []Option) *options { o := new(options) o.c = new(nullConfigurator) @@ -199,11 +211,12 @@ func newOptions(opts []Option) *options { func New(name string, version string, opts ...Option) *Logger { o := newOptions(opts) return &Logger{ - opts: opts, - name: name, - version: version, - leakSensitive: o.leakSensitive || o.c.Bool("log.leak_sensitive_values"), - redactionText: stringsx.DefaultIfEmpty(o.redactionText, `Value is sensitive and has been redacted. To see the value set config key "log.leak_sensitive_values = true" or environment variable "LOG_LEAK_SENSITIVE_VALUES=true".`), + opts: opts, + name: name, + version: version, + leakSensitive: o.leakSensitive || o.c.Bool("log.leak_sensitive_values"), + redactionText: stringsx.DefaultIfEmpty(o.redactionText, `Value is sensitive and has been redacted. To see the value set config key "log.leak_sensitive_values = true" or environment variable "LOG_LEAK_SENSITIVE_VALUES=true".`), + redactableHTTPHeaders: o.redactableHTTPHeaders, Entry: newLogger(o.l, o).WithFields(logrus.Fields{ "audience": "application", "service_name": name, "service_version": version}), } @@ -216,6 +229,11 @@ func NewAudit(name string, version string, opts ...Option) *Logger { func (l *Logger) UseConfig(c configurator) { l.leakSensitive = l.leakSensitive || c.Bool("log.leak_sensitive_values") l.redactionText = stringsx.DefaultIfEmpty(c.String("log.redaction_text"), l.redactionText) + r := c.Strings("log.redactable_http_headers") + if len(r) == 0 { + r = l.redactableHTTPHeaders + } + l.redactableHTTPHeaders = r o := newOptions(append(l.opts, WithConfigurator(c))) setLevel(l.Entry.Logger, o) setFormatter(l.Entry.Logger, o) diff --git a/logrusx/logrus_test.go b/logrusx/logrus_test.go index 671315ff..e79ad122 100644 --- a/logrusx/logrus_test.go +++ b/logrusx/logrus_test.go @@ -135,6 +135,19 @@ func TestTextLogger(t *testing.T) { l.WithRequest(fakeRequest).Error("An error occurred.") }, }, + { + l: New("logrusx-app", "v0.0.0", ForceFormat("text"), ForceLevel(logrus.TraceLevel), RedactionText("redacted"), RedactableHTTPHeaders([]string{"X-Request-ID"})), + expect: []string{"logrus_test.go", "logrusx_test.TestTextLogger", + "audience=application", "service_name=logrusx-app", "service_version=v0.0.0", + "An error occurred.", "headers:map[", "accept:application/json", "accept-encoding:gzip", + "user-agent:Go-http-client/1.1", "host:127.0.0.1:63232", "method:GET", + "query:redacted", + }, + notExpect: []string{"testing.tRunner", "bar=foo", "x-request-id:id1234"}, + call: func(l *Logger) { + l.WithRequest(fakeRequest).Error("An error occurred.") + }, + }, { l: New("logrusx-server", "v0.0.1", ForceFormat("text"), LeakSensitive(), ForceLevel(logrus.DebugLevel)), expect: []string{