From 6b611f0fad2a5a6dec229c02f6aa97f9e52c9da5 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 26 Jul 2016 17:09:27 +0200 Subject: [PATCH 1/2] Add missing IsConst to event format string --- libbeat/common/fmtstr/formatevents.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libbeat/common/fmtstr/formatevents.go b/libbeat/common/fmtstr/formatevents.go index ae80dff6f3a..ba0a3dfcf6d 100644 --- a/libbeat/common/fmtstr/formatevents.go +++ b/libbeat/common/fmtstr/formatevents.go @@ -201,6 +201,11 @@ func (fs *EventFormatString) Eval(out *bytes.Buffer, event common.MapStr) error return fs.formatter.Eval(ctx, out) } +// IsConst checks the format string always returning the same constant string +func (fs *EventFormatString) IsConst() bool { + return fs.formatter.IsConst() +} + // collectFields tries to extract and convert all required fields into an array // of strings. func (fs *EventFormatString) collectFields( From d37ab010f9d4667542c2c994bc05a787975b69bc Mon Sep 17 00:00:00 2001 From: urso Date: Sun, 24 Jul 2016 20:30:36 +0200 Subject: [PATCH 2/2] Add generic string-selector from events --- libbeat/outputs/outil/outil.go | 1 + libbeat/outputs/outil/select.go | 302 +++++++++++++++++++++++++++ libbeat/outputs/outil/select_test.go | 268 ++++++++++++++++++++++++ 3 files changed, 571 insertions(+) create mode 100644 libbeat/outputs/outil/outil.go create mode 100644 libbeat/outputs/outil/select.go create mode 100644 libbeat/outputs/outil/select_test.go diff --git a/libbeat/outputs/outil/outil.go b/libbeat/outputs/outil/outil.go new file mode 100644 index 00000000000..b296f305f24 --- /dev/null +++ b/libbeat/outputs/outil/outil.go @@ -0,0 +1 @@ +package outil diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go new file mode 100644 index 00000000000..7efaf92850a --- /dev/null +++ b/libbeat/outputs/outil/select.go @@ -0,0 +1,302 @@ +package outil + +import ( + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/fmtstr" + "github.com/elastic/beats/libbeat/processors" +) + +type Selector struct { + sel selector +} + +type Settings struct { + // single selector key and default option keyword + Key string + + // multi-selector key in config + MultiKey string + + // if enabled a selector `key` in config will be generated, if `key` is present + EnableSingleOnly bool + + // Fail building selector if `key` and `multiKey` are missing + FailEmpty bool +} + +type selector interface { + sel(evt common.MapStr) (string, error) +} + +type emptySelector struct{} + +type listSelector struct { + selectors []selector +} + +type condSelector struct { + s selector + cond *processors.Condition +} + +type constSelector struct { + s string +} + +type fmtSelector struct { + f fmtstr.EventFormatString + otherwise string +} + +type mapSelector struct { + from selector + otherwise string + to map[string]string +} + +var nilSelector selector = &emptySelector{} + +// Select runs configured selector against the current event. +// If no matching selector is found, an empty string is returned. +// It's up to the caller to decide if an empty string is an error +// or an expected result. +func (s Selector) Select(evt common.MapStr) (string, error) { + return s.sel.sel(evt) +} + +func BuildSelector(cfg *common.Config, settings Settings) (Selector, error) { + var sel []selector + + key := settings.Key + multiKey := settings.MultiKey + found := false + + if cfg.HasField(multiKey) { + found = true + sub, err := cfg.Child(multiKey, -1) + if err != nil { + return Selector{}, err + } + + var table []*common.Config + if err := sub.Unpack(&table); err != nil { + return Selector{}, err + } + + for _, config := range table { + action, err := buildSingle(config, key) + if err != nil { + return Selector{}, err + } + + if action != nilSelector { + sel = append(sel, action) + } + } + } + + if settings.EnableSingleOnly && cfg.HasField(key) { + found = true + + // expect event-format-string + str, err := cfg.String(key, -1) + if err != nil { + return Selector{}, err + } + + fmtstr, err := fmtstr.CompileEvent(str) + if err != nil { + return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) + } + + sel = append(sel, &fmtSelector{f: *fmtstr}) + } + + if settings.FailEmpty && !found { + if settings.EnableSingleOnly { + return Selector{}, fmt.Errorf("missing required '%v' or '%v' in %v", + key, multiKey, cfg.Path()) + } + + return Selector{}, fmt.Errorf("missing required '%v' in %v", + multiKey, cfg.Path()) + } + + switch len(sel) { + case 0: + return Selector{nilSelector}, nil + case 1: + return Selector{sel[0]}, nil + default: + return Selector{&listSelector{sel}}, nil + } +} + +func buildSingle(cfg *common.Config, key string) (selector, error) { + // TODO: check for unknown fields + + // 3. extract required key-word handler + if !cfg.HasField(key) { + return nil, fmt.Errorf("missing %v", cfg.PathOf(key)) + } + + str, err := cfg.String(key, -1) + if err != nil { + return nil, err + } + + evtfmt, err := fmtstr.CompileEvent(str) + if err != nil { + return nil, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) + } + + // 2. extract optional `default` value + var otherwise string + if cfg.HasField("default") { + tmp, err := cfg.String("default", -1) + if err != nil { + return nil, err + } + otherwise = tmp + } + + // 3. extract optional `mapping` + mapping := struct { + Table map[string]string `config:"mappings"` + }{nil} + if cfg.HasField("mappings") { + if err := cfg.Unpack(&mapping); err != nil { + return nil, err + } + } + + // 4. extract conditional + var cond *processors.Condition + if cfg.HasField("when") { + sub, err := cfg.Child("when", -1) + if err != nil { + return nil, err + } + + condConfig := processors.ConditionConfig{} + if err := sub.Unpack(&condConfig); err != nil { + return nil, err + } + + tmp, err := processors.NewCondition(&condConfig) + if err != nil { + return nil, err + } + + cond = tmp + } + + // 5. build selector from available fields + var sel selector + if len(mapping.Table) > 0 { + if evtfmt.IsConst() { + str, err := evtfmt.Run(common.MapStr{}) + if err != nil { + return nil, err + } + + str = mapping.Table[str] + if str == "" { + str = otherwise + } + + if str == "" { + sel = nilSelector + } else { + sel = &constSelector{str} + } + } else { + sel = &mapSelector{ + from: &fmtSelector{f: *evtfmt}, + to: mapping.Table, + otherwise: otherwise, + } + } + } else { + if evtfmt.IsConst() { + str, err := evtfmt.Run(common.MapStr{}) + if err != nil { + return nil, err + } + sel = &constSelector{str} + } else { + sel = &fmtSelector{f: *evtfmt, otherwise: otherwise} + } + } + if cond != nil && sel != nilSelector { + sel = &condSelector{s: sel, cond: cond} + } + + return sel, nil +} + +func (s *emptySelector) sel(evt common.MapStr) (string, error) { + return "", nil +} + +func (s *listSelector) sel(evt common.MapStr) (string, error) { + for _, sub := range s.selectors { + n, err := sub.sel(evt) + if err != nil { // TODO: try + return n, err + } + + if n != "" { + return n, nil + } + } + + return "", nil +} + +func (s *condSelector) sel(evt common.MapStr) (string, error) { + if !s.cond.Check(evt) { + return "", nil + } + return s.s.sel(evt) +} + +func (s *constSelector) sel(_ common.MapStr) (string, error) { + return s.s, nil +} + +func (s *fmtSelector) sel(evt common.MapStr) (string, error) { + n, err := s.f.Run(evt) + if err != nil { + // err will be set if not all keys present in event -> + // return empty selector result and ignore error + return s.otherwise, nil + } + + if n == "" { + return s.otherwise, nil + } + return n, nil +} + +func (s *mapSelector) sel(evt common.MapStr) (string, error) { + n, err := s.from.sel(evt) + if err != nil { + if s.otherwise == "" { + return "", err + } + return s.otherwise, nil + } + + if n == "" { + return s.otherwise, nil + } + + n = s.to[n] + if n == "" { + return s.otherwise, nil + } + return n, nil +} diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go new file mode 100644 index 00000000000..fa683897fe8 --- /dev/null +++ b/libbeat/outputs/outil/select_test.go @@ -0,0 +1,268 @@ +package outil + +import ( + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/stretchr/testify/assert" +) + +type node map[string]interface{} + +func TestSelector(t *testing.T) { + tests := []struct { + title string + config string + event common.MapStr + expected string + }{ + { + "constant key", + `key: value`, + common.MapStr{}, + "value", + }, + { + "format string key", + `key: '%{[key]}'`, + common.MapStr{"key": "value"}, + "value", + }, + { + "key with empty keys", + `{key: value, keys: }`, + common.MapStr{}, + "value", + }, + { + "constant in multi key", + `keys: [key: 'value']`, + common.MapStr{}, + "value", + }, + { + "format string in multi key", + `keys: [key: '%{[key]}']`, + common.MapStr{"key": "value"}, + "value", + }, + { + "missing format string key with default in rule", + `keys: + - key: '%{[key]}' + default: value`, + common.MapStr{}, + "value", + }, + { + "empty format string key with default in rule", + `keys: + - key: '%{[key]}' + default: value`, + common.MapStr{"key": ""}, + "value", + }, + { + "missing format string key with constant in next rule", + `keys: + - key: '%{[key]}' + - key: value`, + common.MapStr{}, + "value", + }, + { + "missing format string key with constant in top-level rule", + `{ key: value, keys: [key: '%{[key]}']}`, + common.MapStr{}, + "value", + }, + { + "apply mapping", + `keys: + - key: '%{[key]}' + mappings: + v: value`, + common.MapStr{"key": "v"}, + "value", + }, + { + "apply mapping with default on empty key", + `keys: + - key: '%{[key]}' + default: value + mappings: + v: 'v'`, + common.MapStr{"key": ""}, + "value", + }, + { + "apply mapping with default on empty lookup", + `keys: + - key: '%{[key]}' + default: value + mappings: + v: ''`, + common.MapStr{"key": "v"}, + "value", + }, + { + "apply mapping without match", + `keys: + - key: '%{[key]}' + mappings: + v: '' + - key: value`, + common.MapStr{"key": "x"}, + "value", + }, + { + "mapping with constant key", + `keys: + - key: k + mappings: + k: value`, + common.MapStr{}, + "value", + }, + { + "mapping with missing constant key", + `keys: + - key: unknown + mappings: {k: wrong} + - key: value`, + common.MapStr{}, + "value", + }, + { + "mapping with missing constant key, but default", + `keys: + - key: unknown + default: value + mappings: {k: wrong}`, + common.MapStr{}, + "value", + }, + { + "matching condition", + `keys: + - key: value + when.equals.test: test`, + common.MapStr{"test": "test"}, + "value", + }, + { + "failing condition", + `keys: + - key: wrong + when.equals.test: test + - key: value`, + common.MapStr{"test": "x"}, + "value", + }, + } + + for i, test := range tests { + t.Logf("run (%v): %v", i, test.title) + + cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") + if err != nil { + t.Error(err) + continue + } + + sel, err := BuildSelector(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) + if err != nil { + t.Error(err) + continue + } + + actual, err := sel.Select(test.event) + if err != nil { + t.Error(err) + continue + } + + assert.Equal(t, test.expected, actual) + } +} + +func TestSelectorInitFail(t *testing.T) { + tests := []struct { + title string + config string + }{ + { + "keys missing", + `test: no key`, + }, + { + "invalid keys type", + `keys: 5`, + }, + { + "invaid keys element type", + `keys: [5]`, + }, + { + "invalid key type", + `key: {}`, + }, + { + "missing key in list", + `keys: [default: value]`, + }, + { + "invalid key type in list", + `keys: [key: {}]`, + }, + { + "fail on invalid format string", + `key: '%{[abc}'`, + }, + { + "fail on invalid format string in list", + `keys: [key: '%{[abc}']`, + }, + { + "default value type mismatch", + `keys: [{key: ok, default: {}}]`, + }, + { + "mappings type mismatch", + `keys: + - key: '%{[k]}' + mappings: {v: {}}`, + }, + { + "condition empty", + `keys: + - key: value + when:`, + }, + } + + for i, test := range tests { + t.Logf("run (%v): %v", i, test.title) + + cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") + if err != nil { + t.Error(err) + continue + } + + _, err = BuildSelector(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) + + assert.Error(t, err) + t.Log(err) + } +}