From b869cee9ff82cea70c0f4ba2d15152f67428c3f3 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 6 Feb 2020 18:55:53 +0100 Subject: [PATCH 1/2] Elasticsearch index must be lowercase (#16081) * Index names must be lowercase When indexing into Elasticsearch index names must always be lowercase. If the index or indices setting are configured to produce non-lowercase strings (e.g. by extracting part of the index name from the event contents), we need to normalize them to be lowercase. This change ensure that index names are always converted to lowercase. Static strings are converted to lowercase upfront, while dynamic strings will be post-processed. * update kafka/redis/LS output to guarantee lowercase index * add godoc (cherry picked from commit 7ddcb1e47d92fbd91d347c4600045972f461033d) --- CHANGELOG.next.asciidoc | 2 + libbeat/outputs/kafka/client.go | 2 +- libbeat/outputs/logstash/config.go | 3 +- libbeat/outputs/logstash/enc.go | 3 + .../logstash/logstash_integration_test.go | 3 +- libbeat/outputs/outil/select.go | 135 ++++++----- libbeat/outputs/outil/select_test.go | 222 +++++++++--------- libbeat/outputs/redis/client.go | 3 +- 8 files changed, 209 insertions(+), 164 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7f63b0348531..9ddaca96e997 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -71,6 +71,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516} - Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] +- Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} +- Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] *Auditbeat* diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 329b9f5df67d..8de05ef6077d 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -79,7 +79,7 @@ func newKafkaClient( hosts: hosts, topic: topic, key: key, - index: index, + index: strings.ToLower(index), codec: writer, config: *cfg, } diff --git a/libbeat/outputs/logstash/config.go b/libbeat/outputs/logstash/config.go index 598413f5814f..6d2a30d39ea6 100644 --- a/libbeat/outputs/logstash/config.go +++ b/libbeat/outputs/logstash/config.go @@ -18,6 +18,7 @@ package logstash import ( + "strings" "time" "github.com/elastic/beats/libbeat/beat" @@ -80,7 +81,7 @@ func readConfig(cfg *common.Config, info beat.Info) (*Config, error) { } if c.Index == "" { - c.Index = info.IndexPrefix + c.Index = strings.ToLower(info.IndexPrefix) } return &c, nil diff --git a/libbeat/outputs/logstash/enc.go b/libbeat/outputs/logstash/enc.go index b9fa409b6e9c..fb42626ec5cd 100644 --- a/libbeat/outputs/logstash/enc.go +++ b/libbeat/outputs/logstash/enc.go @@ -18,6 +18,8 @@ package logstash import ( + "strings" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/outputs/codec/json" ) @@ -27,6 +29,7 @@ func makeLogstashEventEncoder(info beat.Info, escapeHTML bool, index string) fun Pretty: false, EscapeHTML: escapeHTML, }) + index = strings.ToLower(index) return func(event interface{}) (d []byte, err error) { d, err = enc.Encode(index, event.(*beat.Event)) if err != nil { diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 5a4e97c3a1bc..e6800766b331 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -92,7 +92,8 @@ func esConnect(t *testing.T, index string) *esConnection { host := getElasticsearchHost() indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index)) - indexSel := outil.MakeSelector(outil.FmtSelectorExpr(indexFmt, "")) + indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "") + indexSel := outil.MakeSelector(indexFmtExpr) index, _ = indexSel.Select(&beat.Event{ Timestamp: ts, }) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index d6ff4931b64e..d06ee4e32091 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -19,6 +19,7 @@ package outil import ( "fmt" + "strings" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -26,10 +27,14 @@ import ( "github.com/elastic/beats/libbeat/conditions" ) +// Selector is used to produce a string based on the contents of a Beats event. +// A selector supports multiple rules that need to be configured. type Selector struct { sel SelectorExpr } +// Settings configures how BuildSelectorFromConfig creates a Selector from +// a given configuration object. type Settings struct { // single selector key and default option keyword Key string @@ -44,6 +49,8 @@ type Settings struct { FailEmpty bool } +// SelectorExpr represents an expression object that can be composed with other +// expressions in order to build a Selector. type SelectorExpr interface { sel(evt *beat.Event) (string, error) } @@ -76,6 +83,7 @@ type mapSelector struct { var nilSelector SelectorExpr = &emptySelector{} +// MakeSelector creates a selector from a set of selector expressions. func MakeSelector(es ...SelectorExpr) Selector { switch len(es) { case 0: @@ -95,10 +103,12 @@ func (s Selector) Select(evt *beat.Event) (string, error) { return s.sel.sel(evt) } +// IsEmpty checks if the selector is not configured and will always return an empty string. func (s Selector) IsEmpty() bool { return s.sel == nilSelector || s.sel == nil } +// IsConst checks if the selector will always return the same string. func (s Selector) IsConst() bool { if s.sel == nilSelector { return true @@ -108,6 +118,7 @@ func (s Selector) IsConst() bool { return ok } +// BuildSelectorFromConfig creates a selector from a configuration object. func BuildSelectorFromConfig( cfg *common.Config, settings Settings, @@ -156,17 +167,13 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - if fmtstr.IsConst() { - str, err := fmtstr.Run(nil) - if err != nil { - return Selector{}, err - } + fmtsel, err := FmtSelectorExpr(fmtstr, "") + if err != nil { + return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) + } - if str != "" { - sel = append(sel, ConstSelectorExpr(str)) - } - } else { - sel = append(sel, FmtSelectorExpr(fmtstr, "")) + if fmtsel != nilSelector { + sel = append(sel, fmtsel) } } @@ -183,22 +190,44 @@ func BuildSelectorFromConfig( return MakeSelector(sel...), nil } +// EmptySelectorExpr create a selector expression that returns an empty string. func EmptySelectorExpr() SelectorExpr { return nilSelector } +// ConstSelectorExpr creates a selector expression that always returns the configured string. func ConstSelectorExpr(s string) SelectorExpr { - return &constSelector{s} + if s == "" { + return EmptySelectorExpr() + } + return &constSelector{strings.ToLower(s)} } -func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) SelectorExpr { - return &fmtSelector{*fmt, fallback} +// FmtSelectorExpr creates a selector expression using a format string. If the +// event can not be applied the default fallback constant string will be returned. +func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorExpr, error) { + if fmt.IsConst() { + str, err := fmt.Run(nil) + if err != nil { + return nil, err + } + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil } +// ConcatSelectorExpr combines multiple expressions that are run one after the other. +// The first expression that returns a string wins. func ConcatSelectorExpr(s ...SelectorExpr) SelectorExpr { return &listSelector{s} } +// ConditionalSelectorExpr executes the given expression only if the event +// matches the given condition. func ConditionalSelectorExpr( s SelectorExpr, cond conditions.Condition, @@ -206,12 +235,39 @@ func ConditionalSelectorExpr( return &condSelector{s, cond} } +// LookupSelectorExpr replaces the produced string with an table entry. +// If there is no entry in the table the default fallback string will be reported. func LookupSelectorExpr( - s SelectorExpr, + evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, -) SelectorExpr { - return &mapSelector{s, fallback, table} +) (SelectorExpr, error) { + if evtfmt.IsConst() { + str, err := evtfmt.Run(nil) + if err != nil { + return nil, err + } + + str = table[strings.ToLower(str)] + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &mapSelector{ + from: &fmtSelector{f: *evtfmt}, + to: table, + otherwise: fallback, + }, nil +} + +func lowercaseTable(table map[string]string) map[string]string { + tmp := make(map[string]string, len(table)) + for k, v := range table { + tmp[strings.ToLower(k)] = strings.ToLower(v) + } + return tmp } func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { @@ -239,7 +295,7 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - otherwise = tmp + otherwise = strings.ToLower(tmp) } // 3. extract optional `mapping` @@ -276,45 +332,14 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // 5. build selector from available fields var sel SelectorExpr if len(mapping.Table) > 0 { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - str = mapping.Table[str] - if str == "" { - str = otherwise - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = &mapSelector{ - from: FmtSelectorExpr(evtfmt, ""), - to: mapping.Table, - otherwise: otherwise, - } - } + sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) } else { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = FmtSelectorExpr(evtfmt, otherwise) - } + sel, err = FmtSelectorExpr(evtfmt, otherwise) } + if err != nil { + return nil, err + } + if cond != nil && sel != nilSelector { sel = ConditionalSelectorExpr(sel, cond) } @@ -363,7 +388,7 @@ func (s *fmtSelector) sel(evt *beat.Event) (string, error) { if n == "" { return s.otherwise, nil } - return n, nil + return strings.ToLower(n), nil } func (s *mapSelector) sel(evt *beat.Event) (string, error) { diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go index d4093ed56606..f6e837966b2d 100644 --- a/libbeat/outputs/outil/select_test.go +++ b/libbeat/outputs/outil/select_test.go @@ -31,74 +31,92 @@ import ( type node map[string]interface{} func TestSelector(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string event common.MapStr expected string }{ - { - "constant key", + "constant key": { `key: value`, common.MapStr{}, "value", }, - { - "format string key", + "lowercase constant key": { + `key: VaLuE`, + common.MapStr{}, + "value", + }, + "format string key": { `key: '%{[key]}'`, common.MapStr{"key": "value"}, "value", }, - { - "key with empty keys", + "lowercase format string key": { + `key: '%{[key]}'`, + common.MapStr{"key": "VaLuE"}, + "value", + }, + "key with empty keys": { `{key: value, keys: }`, common.MapStr{}, "value", }, - { - "constant in multi key", + "lowercase key with empty keys": { + `{key: vAlUe, keys: }`, + common.MapStr{}, + "value", + }, + "constant in multi key": { `keys: [key: 'value']`, common.MapStr{}, "value", }, - { - "format string in multi key", + "format string in multi key": { `keys: [key: '%{[key]}']`, common.MapStr{"key": "value"}, "value", }, - { - "missing format string key with default in rule", + "missing format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{}, "value", }, - { - "empty format string key with default in rule", + "lowercase missing format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAlUe`, + common.MapStr{}, + "value", + }, + "empty format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{"key": ""}, "value", }, - { - "missing format string key with constant in next rule", + "lowercase empty format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAluE`, + common.MapStr{"key": ""}, + "value", + }, + "missing format string key with constant in next rule": { `keys: - key: '%{[key]}' - key: value`, common.MapStr{}, "value", }, - { - "missing format string key with constant in top-level rule", + "missing format string key with constant in top-level rule": { `{ key: value, keys: [key: '%{[key]}']}`, common.MapStr{}, "value", }, - { - "apply mapping", + "apply mapping": { `keys: - key: '%{[key]}' mappings: @@ -106,8 +124,15 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping with default on empty key", + "lowercase applied mapping": { + `keys: + - key: '%{[key]}' + mappings: + v: vAlUe`, + common.MapStr{"key": "v"}, + "value", + }, + "apply mapping with default on empty key": { `keys: - key: '%{[key]}' default: value @@ -116,8 +141,16 @@ func TestSelector(t *testing.T) { common.MapStr{"key": ""}, "value", }, - { - "apply mapping with default on empty lookup", + "lowercase apply mapping with default on empty key": { + `keys: + - key: '%{[key]}' + default: vAluE + mappings: + v: 'v'`, + common.MapStr{"key": ""}, + "value", + }, + "apply mapping with default on empty lookup": { `keys: - key: '%{[key]}' default: value @@ -126,8 +159,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping without match", + "apply mapping without match": { `keys: - key: '%{[key]}' mappings: @@ -136,8 +168,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "x"}, "value", }, - { - "mapping with constant key", + "mapping with constant key": { `keys: - key: k mappings: @@ -145,8 +176,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key", + "mapping with missing constant key": { `keys: - key: unknown mappings: {k: wrong} @@ -154,8 +184,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key, but default", + "mapping with missing constant key, but default": { `keys: - key: unknown default: value @@ -163,16 +192,14 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "matching condition", + "matching condition": { `keys: - key: value when.equals.test: test`, common.MapStr{"test": "test"}, "value", }, - { - "failing condition", + "failing condition": { `keys: - key: wrong when.equals.test: test @@ -182,113 +209,98 @@ func TestSelector(t *testing.T) { }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) - - yaml := strings.Replace(test.config, "\t", " ", -1) - cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") - if err != nil { - t.Errorf("YAML parse error: %v\n%v", err, yaml) - continue - } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + yaml := strings.Replace(test.config, "\t", " ", -1) + cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") + if err != nil { + t.Fatalf("YAML parse error: %v\n%v", err, yaml) + } - sel, err := BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, - }) - if err != nil { - t.Error(err) - continue - } + sel, err := BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) + if err != nil { + t.Fatal(err) + } - event := beat.Event{ - Timestamp: time.Now(), - Fields: test.event, - } - actual, err := sel.Select(&event) - if err != nil { - t.Error(err) - continue - } + event := beat.Event{ + Timestamp: time.Now(), + Fields: test.event, + } + actual, err := sel.Select(&event) + if err != nil { + t.Fatal(err) + } - assert.Equal(t, test.expected, actual) + assert.Equal(t, test.expected, actual) + }) } } func TestSelectorInitFail(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string }{ - { - "keys missing", + "keys missing": { `test: no key`, }, - { - "invalid keys type", + "invalid keys type": { `keys: 5`, }, - { - "invaid keys element type", + "invaid keys element type": { `keys: [5]`, }, - { - "invalid key type", + "invalid key type": { `key: {}`, }, - { - "missing key in list", + "missing key in list": { `keys: [default: value]`, }, - { - "invalid key type in list", + "invalid key type in list": { `keys: [key: {}]`, }, - { - "fail on invalid format string", + "fail on invalid format string": { `key: '%{[abc}'`, }, - { - "fail on invalid format string in list", + "fail on invalid format string in list": { `keys: [key: '%{[abc}']`, }, - { - "default value type mismatch", + "default value type mismatch": { `keys: [{key: ok, default: {}}]`, }, - { - "mappings type mismatch", + "mappings type mismatch": { `keys: - key: '%{[k]}' mappings: {v: {}}`, }, - { - "condition empty", + "condition empty": { `keys: - key: value when:`, }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) + for name, test := range tests { + t.Run(name, func(t *testing.T) { + cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") + if err != nil { + t.Fatal(err) + } - cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") - if err != nil { - t.Error(err) - continue - } + _, err = BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) - _, err = BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, + assert.Error(t, err) + t.Log(err) }) - assert.Error(t, err) - t.Log(err) } } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 7238ba367fd0..1764d9e60f99 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -21,6 +21,7 @@ import ( "errors" "regexp" "strconv" + "strings" "time" "github.com/garyburd/redigo/redis" @@ -77,7 +78,7 @@ func newClient( observer: observer, timeout: timeout, password: pass, - index: index, + index: strings.ToLower(index), db: db, dataType: dt, key: key, From e9572fac345524ad8260f33479374bcdd7a86d44 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 6 Feb 2020 23:07:28 +0100 Subject: [PATCH 2/2] fix changelog --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9ddaca96e997..788c52407180 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -71,7 +71,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516} - Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] -- Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} - Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] *Auditbeat*