From b0a75811cac914acea87d7b438598ff2b4333c72 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 12 Apr 2016 17:06:27 -0600 Subject: [PATCH] Adds support for removing/keeping tags from metrics closes #706 --- Godeps | 1 + agent/accumulator.go | 1 + docs/LICENSE_OF_DEPENDENCIES.md | 3 +- internal/config/config.go | 52 ++++++++-- internal/internal.go | 56 ----------- internal/internal_test.go | 41 -------- internal/models/filter.go | 163 +++++++++++++++++++++++--------- 7 files changed, 166 insertions(+), 151 deletions(-) diff --git a/Godeps b/Godeps index 14430ea5d0aef..71057f497f813 100644 --- a/Godeps +++ b/Godeps @@ -16,6 +16,7 @@ github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367 github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee +github.com/gobwas/glob d877f6352135181470c40c73ebb81aefa22115fa github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032 github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2 diff --git a/agent/accumulator.go b/agent/accumulator.go index 7ec22cd7f4ad2..6b2ffde2d4374 100644 --- a/agent/accumulator.go +++ b/agent/accumulator.go @@ -96,6 +96,7 @@ func (ac *accumulator) AddFields( tags[k] = v } } + ac.inputConfig.Filter.FilterTags(tags) result := make(map[string]interface{}) for k, v := range fields { diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index c8f3b0926a2ff..d448872f68a1b 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -28,6 +28,5 @@ - github.com/wvanbergen/kazoo-go [MIT LICENSE](https://github.com/wvanbergen/kazoo-go/blob/master/MIT-LICENSE) - gopkg.in/dancannon/gorethink.v1 [APACHE LICENSE](https://github.com/dancannon/gorethink/blob/v1.1.2/LICENSE) - gopkg.in/mgo.v2 [BSD LICENSE](https://github.com/go-mgo/mgo/blob/v2/LICENSE) -- golang.org/x/crypto/* [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE) -- internal Glob function [MIT LICENSE](https://github.com/ryanuber/go-glob/blob/master/LICENSE) +- golang.org/x/crypto/ [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE) diff --git a/internal/config/config.go b/internal/config/config.go index cfd6c959349f4..5d08369644083 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -580,9 +580,9 @@ func (c *Config) addInput(name string, table *ast.Table) error { // buildFilter builds a Filter // (tagpass/tagdrop/namepass/namedrop/fieldpass/fielddrop) to -// be inserted into the internal_models.OutputConfig/internal_models.InputConfig to be used for prefix -// filtering on tags and measurements -func buildFilter(tbl *ast.Table) internal_models.Filter { +// be inserted into the internal_models.OutputConfig/internal_models.InputConfig +// to be used for glob filtering on tags and measurements +func buildFilter(tbl *ast.Table) (internal_models.Filter, error) { f := internal_models.Filter{} if node, ok := tbl.Fields["namepass"]; ok { @@ -681,6 +681,33 @@ func buildFilter(tbl *ast.Table) internal_models.Filter { } } + if node, ok := tbl.Fields["tagexclude"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + f.TagExclude = append(f.TagExclude, str.Value) + } + } + } + } + } + + if node, ok := tbl.Fields["taginclude"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + f.TagInclude = append(f.TagInclude, str.Value) + } + } + } + } + } + if err := f.CompileFilter(); err != nil { + return f, err + } + delete(tbl.Fields, "namedrop") delete(tbl.Fields, "namepass") delete(tbl.Fields, "fielddrop") @@ -689,7 +716,9 @@ func buildFilter(tbl *ast.Table) internal_models.Filter { delete(tbl.Fields, "pass") delete(tbl.Fields, "tagdrop") delete(tbl.Fields, "tagpass") - return f + delete(tbl.Fields, "tagexclude") + delete(tbl.Fields, "taginclude") + return f, nil } // buildInput parses input specific items from the ast.Table, @@ -748,7 +777,11 @@ func buildInput(name string, tbl *ast.Table) (*internal_models.InputConfig, erro delete(tbl.Fields, "name_override") delete(tbl.Fields, "interval") delete(tbl.Fields, "tags") - cp.Filter = buildFilter(tbl) + var err error + cp.Filter, err = buildFilter(tbl) + if err != nil { + return cp, err + } return cp, nil } @@ -864,13 +897,18 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error return serializers.NewSerializer(c) } -// buildOutput parses output specific items from the ast.Table, builds the filter and returns an +// buildOutput parses output specific items from the ast.Table, +// builds the filter and returns an // internal_models.OutputConfig to be inserted into internal_models.RunningInput // Note: error exists in the return for future calls that might require error func buildOutput(name string, tbl *ast.Table) (*internal_models.OutputConfig, error) { + filter, err := buildFilter(tbl) + if err != nil { + return nil, err + } oc := &internal_models.OutputConfig{ Name: name, - Filter: buildFilter(tbl), + Filter: filter, } // Outputs don't support FieldDrop/FieldPass, so set to NameDrop/NamePass if len(oc.Filter.FieldDrop) > 0 { diff --git a/internal/internal.go b/internal/internal.go index ff73aae84fb6c..4b8e1536ff86f 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -139,59 +139,3 @@ func SnakeCase(in string) string { return string(out) } - -// Glob will test a string pattern, potentially containing globs, against a -// subject string. The result is a simple true/false, determining whether or -// not the glob pattern matched the subject text. -// -// Adapted from https://github.com/ryanuber/go-glob/blob/master/glob.go -// thanks Ryan Uber! -func Glob(pattern, measurement string) bool { - // Empty pattern can only match empty subject - if pattern == "" { - return measurement == pattern - } - - // If the pattern _is_ a glob, it matches everything - if pattern == "*" { - return true - } - - parts := strings.Split(pattern, "*") - - if len(parts) == 1 { - // No globs in pattern, so test for match - return pattern == measurement - } - - leadingGlob := strings.HasPrefix(pattern, "*") - trailingGlob := strings.HasSuffix(pattern, "*") - end := len(parts) - 1 - - for i, part := range parts { - switch i { - case 0: - if leadingGlob { - continue - } - if !strings.HasPrefix(measurement, part) { - return false - } - case end: - if len(measurement) > 0 { - return trailingGlob || strings.HasSuffix(measurement, part) - } - default: - if !strings.Contains(measurement, part) { - return false - } - } - - // Trim evaluated text from measurement as we loop over the pattern. - idx := strings.Index(measurement, part) + len(part) - measurement = measurement[idx:] - } - - // All parts of the pattern matched - return true -} diff --git a/internal/internal_test.go b/internal/internal_test.go index e4a5eed14341c..7ff64e87bb0fe 100644 --- a/internal/internal_test.go +++ b/internal/internal_test.go @@ -2,47 +2,6 @@ package internal import "testing" -func testGlobMatch(t *testing.T, pattern, subj string) { - if !Glob(pattern, subj) { - t.Errorf("%s should match %s", pattern, subj) - } -} - -func testGlobNoMatch(t *testing.T, pattern, subj string) { - if Glob(pattern, subj) { - t.Errorf("%s should not match %s", pattern, subj) - } -} - -func TestEmptyPattern(t *testing.T) { - testGlobMatch(t, "", "") - testGlobNoMatch(t, "", "test") -} - -func TestPatternWithoutGlobs(t *testing.T) { - testGlobMatch(t, "test", "test") -} - -func TestGlob(t *testing.T) { - for _, pattern := range []string{ - "*test", // Leading glob - "this*", // Trailing glob - "*is*a*", // Lots of globs - "**test**", // Double glob characters - "**is**a***test*", // Varying number of globs - } { - testGlobMatch(t, pattern, "this_is_a_test") - } - - for _, pattern := range []string{ - "test*", // Implicit substring match should fail - "*is", // Partial match should fail - "*no*", // Globs without a match between them should fail - } { - testGlobNoMatch(t, pattern, "this_is_a_test") - } -} - type SnakeTest struct { input string output string diff --git a/internal/models/filter.go b/internal/models/filter.go index e2b1377f441e0..d78492a5dbf0a 100644 --- a/internal/models/filter.go +++ b/internal/models/filter.go @@ -1,33 +1,104 @@ package internal_models import ( + "fmt" "strings" + "github.com/gobwas/glob" + "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/internal" ) // TagFilter is the name of a tag, and the values on which to filter type TagFilter struct { Name string Filter []string + filter glob.Glob } // Filter containing drop/pass and tagdrop/tagpass rules type Filter struct { NameDrop []string + nameDrop glob.Glob NamePass []string + namePass glob.Glob FieldDrop []string + fieldDrop glob.Glob FieldPass []string + fieldPass glob.Glob TagDrop []TagFilter TagPass []TagFilter + TagExclude []string + tagExclude glob.Glob + TagInclude []string + tagInclude glob.Glob + IsActive bool } -func (f Filter) ShouldMetricPass(metric telegraf.Metric) bool { +// Compile all Filter lists into glob.Glob objects. +func (f *Filter) CompileFilter() error { + var err error + f.nameDrop, err = compileFilter(f.NameDrop) + if err != nil { + return fmt.Errorf("Error compiling 'namedrop', %s", err) + } + f.namePass, err = compileFilter(f.NamePass) + if err != nil { + return fmt.Errorf("Error compiling 'namepass', %s", err) + } + + f.fieldDrop, err = compileFilter(f.FieldDrop) + if err != nil { + return fmt.Errorf("Error compiling 'fielddrop', %s", err) + } + f.fieldPass, err = compileFilter(f.FieldPass) + if err != nil { + return fmt.Errorf("Error compiling 'fieldpass', %s", err) + } + + f.tagExclude, err = compileFilter(f.TagExclude) + if err != nil { + return fmt.Errorf("Error compiling 'tagexclude', %s", err) + } + f.tagInclude, err = compileFilter(f.TagInclude) + if err != nil { + return fmt.Errorf("Error compiling 'taginclude', %s", err) + } + + for i, _ := range f.TagDrop { + f.TagDrop[i].filter, err = compileFilter(f.TagDrop[i].Filter) + if err != nil { + return fmt.Errorf("Error compiling 'tagdrop', %s", err) + } + } + for i, _ := range f.TagPass { + f.TagPass[i].filter, err = compileFilter(f.TagPass[i].Filter) + if err != nil { + return fmt.Errorf("Error compiling 'tagpass', %s", err) + } + } + return nil +} + +func compileFilter(filter []string) (glob.Glob, error) { + if len(filter) == 0 { + return nil, nil + } + var g glob.Glob + var err error + if len(filter) == 1 { + g, err = glob.Compile(filter[0]) + } else { + g, err = glob.Compile("{" + strings.Join(filter, ",") + "}") + } + return g, err +} + +func (f *Filter) ShouldMetricPass(metric telegraf.Metric) bool { if f.ShouldNamePass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) { return true } @@ -36,70 +107,51 @@ func (f Filter) ShouldMetricPass(metric telegraf.Metric) bool { // ShouldFieldsPass returns true if the metric should pass, false if should drop // based on the drop/pass filter parameters -func (f Filter) ShouldNamePass(key string) bool { - if f.NamePass != nil { - for _, pat := range f.NamePass { - // TODO remove HasPrefix check, leaving it for now for legacy support. - // Cam, 2015-12-07 - if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { - return true - } +func (f *Filter) ShouldNamePass(key string) bool { + if f.namePass != nil { + if f.namePass.Match(key) { + return true } return false } - if f.NameDrop != nil { - for _, pat := range f.NameDrop { - // TODO remove HasPrefix check, leaving it for now for legacy support. - // Cam, 2015-12-07 - if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { - return false - } + if f.nameDrop != nil { + if f.nameDrop.Match(key) { + return false } - - return true } return true } // ShouldFieldsPass returns true if the metric should pass, false if should drop // based on the drop/pass filter parameters -func (f Filter) ShouldFieldsPass(key string) bool { - if f.FieldPass != nil { - for _, pat := range f.FieldPass { - // TODO remove HasPrefix check, leaving it for now for legacy support. - // Cam, 2015-12-07 - if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { - return true - } +func (f *Filter) ShouldFieldsPass(key string) bool { + if f.fieldPass != nil { + if f.fieldPass.Match(key) { + return true } return false } - if f.FieldDrop != nil { - for _, pat := range f.FieldDrop { - // TODO remove HasPrefix check, leaving it for now for legacy support. - // Cam, 2015-12-07 - if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { - return false - } + if f.fieldDrop != nil { + if f.fieldDrop.Match(key) { + return false } - - return true } return true } // ShouldTagsPass returns true if the metric should pass, false if should drop // based on the tagdrop/tagpass filter parameters -func (f Filter) ShouldTagsPass(tags map[string]string) bool { +func (f *Filter) ShouldTagsPass(tags map[string]string) bool { if f.TagPass != nil { for _, pat := range f.TagPass { + if pat.filter == nil { + continue + } if tagval, ok := tags[pat.Name]; ok { - for _, filter := range pat.Filter { - if internal.Glob(filter, tagval) { - return true - } + if pat.filter.Match(tagval) { + return true } } } @@ -108,11 +160,12 @@ func (f Filter) ShouldTagsPass(tags map[string]string) bool { if f.TagDrop != nil { for _, pat := range f.TagDrop { + if pat.filter == nil { + continue + } if tagval, ok := tags[pat.Name]; ok { - for _, filter := range pat.Filter { - if internal.Glob(filter, tagval) { - return false - } + if pat.filter.Match(tagval) { + return false } } } @@ -121,3 +174,23 @@ func (f Filter) ShouldTagsPass(tags map[string]string) bool { return true } + +// Apply TagInclude and TagExclude filters. +// modifies the tags map in-place. +func (f *Filter) FilterTags(tags map[string]string) { + if f.tagInclude != nil { + for k, _ := range tags { + if !f.tagInclude.Match(k) { + delete(tags, k) + } + } + } + + if f.tagExclude != nil { + for k, _ := range tags { + if f.tagExclude.Match(k) { + delete(tags, k) + } + } + } +}