Adds support for removing/keeping tags from metrics

closes #706

Godeps

@@ -16,6 +16,7 @@ github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
 github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86
 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
+github.com/gobwas/glob d877f6352135181470c40c73ebb81aefa22115fa
 github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
 github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7
 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2

agent/accumulator.go

@@ -96,6 +96,7 @@ func (ac *accumulator) AddFields(
 			tags[k] = v
 		}
 	}
+	ac.inputConfig.Filter.FilterTags(tags)
 
 	result := make(map[string]interface{})
 	for k, v := range fields {

docs/LICENSE_OF_DEPENDENCIES.md

@@ -28,6 +28,5 @@
 - github.com/wvanbergen/kazoo-go [MIT LICENSE](https://github.com/wvanbergen/kazoo-go/blob/master/MIT-LICENSE)
 - gopkg.in/dancannon/gorethink.v1 [APACHE LICENSE](https://github.com/dancannon/gorethink/blob/v1.1.2/LICENSE)
 - gopkg.in/mgo.v2 [BSD LICENSE](https://github.com/go-mgo/mgo/blob/v2/LICENSE)
-- golang.org/x/crypto/* [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
-- internal Glob function [MIT LICENSE](https://github.com/ryanuber/go-glob/blob/master/LICENSE)
+- golang.org/x/crypto/ [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
 

internal/config/config.go

@@ -580,9 +580,9 @@ func (c *Config) addInput(name string, table *ast.Table) error {
 
 // buildFilter builds a Filter
 // (tagpass/tagdrop/namepass/namedrop/fieldpass/fielddrop) to
-// be inserted into the internal_models.OutputConfig/internal_models.InputConfig to be used for prefix
-// filtering on tags and measurements
-func buildFilter(tbl *ast.Table) internal_models.Filter {
+// be inserted into the internal_models.OutputConfig/internal_models.InputConfig
+// to be used for glob filtering on tags and measurements
+func buildFilter(tbl *ast.Table) (internal_models.Filter, error) {
 	f := internal_models.Filter{}
 
 	if node, ok := tbl.Fields["namepass"]; ok {
@@ -681,6 +681,33 @@ func buildFilter(tbl *ast.Table) internal_models.Filter {
 		}
 	}
 
+	if node, ok := tbl.Fields["tagexclude"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if ary, ok := kv.Value.(*ast.Array); ok {
+				for _, elem := range ary.Value {
+					if str, ok := elem.(*ast.String); ok {
+						f.TagExclude = append(f.TagExclude, str.Value)
+					}
+				}
+			}
+		}
+	}
+
+	if node, ok := tbl.Fields["taginclude"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if ary, ok := kv.Value.(*ast.Array); ok {
+				for _, elem := range ary.Value {
+					if str, ok := elem.(*ast.String); ok {
+						f.TagInclude = append(f.TagInclude, str.Value)
+					}
+				}
+			}
+		}
+	}
+	if err := f.CompileFilter(); err != nil {
+		return f, err
+	}
+
 	delete(tbl.Fields, "namedrop")
 	delete(tbl.Fields, "namepass")
 	delete(tbl.Fields, "fielddrop")
@@ -689,7 +716,9 @@ func buildFilter(tbl *ast.Table) internal_models.Filter {
 	delete(tbl.Fields, "pass")
 	delete(tbl.Fields, "tagdrop")
 	delete(tbl.Fields, "tagpass")
-	return f
+	delete(tbl.Fields, "tagexclude")
+	delete(tbl.Fields, "taginclude")
+	return f, nil
 }
 
 // buildInput parses input specific items from the ast.Table,
@@ -748,7 +777,11 @@ func buildInput(name string, tbl *ast.Table) (*internal_models.InputConfig, erro
 	delete(tbl.Fields, "name_override")
 	delete(tbl.Fields, "interval")
 	delete(tbl.Fields, "tags")
-	cp.Filter = buildFilter(tbl)
+	var err error
+	cp.Filter, err = buildFilter(tbl)
+	if err != nil {
+		return cp, err
+	}
 	return cp, nil
 }
 
@@ -864,13 +897,18 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error
 	return serializers.NewSerializer(c)
 }
 
-// buildOutput parses output specific items from the ast.Table, builds the filter and returns an
+// buildOutput parses output specific items from the ast.Table,
+// builds the filter and returns an
 // internal_models.OutputConfig to be inserted into internal_models.RunningInput
 // Note: error exists in the return for future calls that might require error
 func buildOutput(name string, tbl *ast.Table) (*internal_models.OutputConfig, error) {
+	filter, err := buildFilter(tbl)
+	if err != nil {
+		return nil, err
+	}
 	oc := &internal_models.OutputConfig{
 		Name:   name,
-		Filter: buildFilter(tbl),
+		Filter: filter,
 	}
 	// Outputs don't support FieldDrop/FieldPass, so set to NameDrop/NamePass
 	if len(oc.Filter.FieldDrop) > 0 {

internal/internal.go

@@ -139,59 +139,3 @@ func SnakeCase(in string) string {
 
 	return string(out)
 }
-
-// Glob will test a string pattern, potentially containing globs, against a
-// subject string. The result is a simple true/false, determining whether or
-// not the glob pattern matched the subject text.
-//
-// Adapted from https://github.com/ryanuber/go-glob/blob/master/glob.go
-// thanks Ryan Uber!
-func Glob(pattern, measurement string) bool {
-	// Empty pattern can only match empty subject
-	if pattern == "" {
-		return measurement == pattern
-	}
-
-	// If the pattern _is_ a glob, it matches everything
-	if pattern == "*" {
-		return true
-	}
-
-	parts := strings.Split(pattern, "*")
-
-	if len(parts) == 1 {
-		// No globs in pattern, so test for match
-		return pattern == measurement
-	}
-
-	leadingGlob := strings.HasPrefix(pattern, "*")
-	trailingGlob := strings.HasSuffix(pattern, "*")
-	end := len(parts) - 1
-
-	for i, part := range parts {
-		switch i {
-		case 0:
-			if leadingGlob {
-				continue
-			}
-			if !strings.HasPrefix(measurement, part) {
-				return false
-			}
-		case end:
-			if len(measurement) > 0 {
-				return trailingGlob || strings.HasSuffix(measurement, part)
-			}
-		default:
-			if !strings.Contains(measurement, part) {
-				return false
-			}
-		}
-
-		// Trim evaluated text from measurement as we loop over the pattern.
-		idx := strings.Index(measurement, part) + len(part)
-		measurement = measurement[idx:]
-	}
-
-	// All parts of the pattern matched
-	return true
-}

internal/internal_test.go

@@ -2,47 +2,6 @@ package internal
 
 import "testing"
 
-func testGlobMatch(t *testing.T, pattern, subj string) {
-	if !Glob(pattern, subj) {
-		t.Errorf("%s should match %s", pattern, subj)
-	}
-}
-
-func testGlobNoMatch(t *testing.T, pattern, subj string) {
-	if Glob(pattern, subj) {
-		t.Errorf("%s should not match %s", pattern, subj)
-	}
-}
-
-func TestEmptyPattern(t *testing.T) {
-	testGlobMatch(t, "", "")
-	testGlobNoMatch(t, "", "test")
-}
-
-func TestPatternWithoutGlobs(t *testing.T) {
-	testGlobMatch(t, "test", "test")
-}
-
-func TestGlob(t *testing.T) {
-	for _, pattern := range []string{
-		"*test",           // Leading glob
-		"this*",           // Trailing glob
-		"*is*a*",          // Lots of globs
-		"**test**",        // Double glob characters
-		"**is**a***test*", // Varying number of globs
-	} {
-		testGlobMatch(t, pattern, "this_is_a_test")
-	}
-
-	for _, pattern := range []string{
-		"test*", // Implicit substring match should fail
-		"*is",   // Partial match should fail
-		"*no*",  // Globs without a match between them should fail
-	} {
-		testGlobNoMatch(t, pattern, "this_is_a_test")
-	}
-}
-
 type SnakeTest struct {
 	input  string
 	output string