feat: migrate grok to new parser style (#11408)

influxdata · Jun 30, 2022 · 56a3f18 · 56a3f18
1 parent f8766bc
commit 56a3f18
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 62 deletions.
diff --git a/plugins/inputs/file/file_test.go b/plugins/inputs/file/file_test.go
@@ -15,8 +15,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/parsers"
 	"github.com/influxdata/telegraf/plugins/parsers/csv"
+	"github.com/influxdata/telegraf/plugins/parsers/grok"
 	"github.com/influxdata/telegraf/plugins/parsers/json"
 	"github.com/influxdata/telegraf/testutil"
 )
@@ -90,12 +90,15 @@ func TestGrokParser(t *testing.T) {
 	err := r.Init()
 	require.NoError(t, err)
 
-	parserConfig := parsers.Config{
-		DataFormat:   "grok",
-		GrokPatterns: []string{"%{COMMON_LOG_FORMAT}"},
-	}
+	r.SetParserFunc(func() (telegraf.Parser, error) {
+		parser := &grok.Parser{
+			Patterns: []string{"%{COMMON_LOG_FORMAT}"},
+			Log:      testutil.Logger{},
+		}
+		err := parser.Init()
 
-	r.SetParserFunc(func() (telegraf.Parser, error) { return parsers.NewParser(&parserConfig) })
+		return parser, err
+	})
 
 	err = r.Gather(&acc)
 	require.NoError(t, err)

diff --git a/plugins/inputs/logparser/logparser.go b/plugins/inputs/logparser/logparser.go
@@ -17,6 +17,7 @@ import (
 	"github.com/influxdata/telegraf/models"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/influxdata/telegraf/plugins/parsers"
+	"github.com/influxdata/telegraf/plugins/parsers/grok"
 )
 
 // DO NOT REMOVE THE NEXT TWO LINES! This is required to embed the sampleConfig data.
@@ -117,22 +118,20 @@ func (l *LogParserPlugin) Start(acc telegraf.Accumulator) error {
 	}
 
 	// Looks for fields which implement LogParser interface
-	config := &parsers.Config{
-		MetricName:             mName,
-		GrokPatterns:           l.GrokConfig.Patterns,
-		GrokNamedPatterns:      l.GrokConfig.NamedPatterns,
-		GrokCustomPatterns:     l.GrokConfig.CustomPatterns,
-		GrokCustomPatternFiles: l.GrokConfig.CustomPatternFiles,
-		GrokTimezone:           l.GrokConfig.Timezone,
-		GrokUniqueTimestamp:    l.GrokConfig.UniqueTimestamp,
-		DataFormat:             "grok",
+	parser := grok.Parser{
+		Measurement:        mName,
+		Patterns:           l.GrokConfig.Patterns,
+		NamedPatterns:      l.GrokConfig.NamedPatterns,
+		CustomPatterns:     l.GrokConfig.CustomPatterns,
+		CustomPatternFiles: l.GrokConfig.CustomPatternFiles,
+		Timezone:           l.GrokConfig.Timezone,
+		UniqueTimestamp:    l.GrokConfig.UniqueTimestamp,
 	}
-
-	var err error
-	l.GrokParser, err = parsers.NewParser(config)
+	err := parser.Init()
 	if err != nil {
 		return err
 	}
+	l.GrokParser = &parser
 	models.SetLoggerOnPlugin(l.GrokParser, l.Log)
 
 	l.wg.Add(1)

diff --git a/plugins/inputs/logparser/logparser_test.go b/plugins/inputs/logparser/logparser_test.go
@@ -24,7 +24,7 @@ func TestStartNoParsers(t *testing.T) {
 	}
 
 	acc := testutil.Accumulator{}
-	require.Error(t, logparser.Start(&acc))
+	require.NoError(t, logparser.Start(&acc))
 }
 
 func TestGrokParseLogFilesNonExistPattern(t *testing.T) {

diff --git a/plugins/inputs/tail/tail_test.go b/plugins/inputs/tail/tail_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/influxdata/telegraf/config"
 	"github.com/influxdata/telegraf/plugins/parsers"
 	"github.com/influxdata/telegraf/plugins/parsers/csv"
+	"github.com/influxdata/telegraf/plugins/parsers/grok"
 	"github.com/influxdata/telegraf/plugins/parsers/influx"
 	"github.com/influxdata/telegraf/plugins/parsers/json"
 	"github.com/influxdata/telegraf/testutil"
@@ -306,13 +307,13 @@ func TestGrokParseLogFilesWithMultilineTailerCloseFlushesMultilineBuffer(t *test
 }
 
 func createGrokParser() (parsers.Parser, error) {
-	grokConfig := &parsers.Config{
-		MetricName:             "tail_grok",
-		GrokPatterns:           []string{"%{TEST_LOG_MULTILINE}"},
-		GrokCustomPatternFiles: []string{filepath.Join(testdataDir, "test-patterns")},
-		DataFormat:             "grok",
+	parser := &grok.Parser{
+		Measurement:        "tail_grok",
+		Patterns:           []string{"%{TEST_LOG_MULTILINE}"},
+		CustomPatternFiles: []string{filepath.Join(testdataDir, "test-patterns")},
+		Log:                testutil.Logger{},
 	}
-	parser, err := parsers.NewParser(grokConfig)
+	err := parser.Init()
 	return parser, err
 }
 

diff --git a/plugins/parsers/all/all.go b/plugins/parsers/all/all.go
@@ -7,6 +7,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/parsers/dropwizard"
 	_ "github.com/influxdata/telegraf/plugins/parsers/form_urlencoded"
 	_ "github.com/influxdata/telegraf/plugins/parsers/graphite"
+	_ "github.com/influxdata/telegraf/plugins/parsers/grok"
 	_ "github.com/influxdata/telegraf/plugins/parsers/json"
 	_ "github.com/influxdata/telegraf/plugins/parsers/json_v2"
 	_ "github.com/influxdata/telegraf/plugins/parsers/logfmt"

diff --git a/plugins/parsers/grok/parser.go b/plugins/parsers/grok/parser.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/metric"
+	"github.com/influxdata/telegraf/plugins/parsers"
 )
 
 var timeLayouts = map[string]string{
@@ -66,17 +67,17 @@ var (
 
 // Parser is the primary struct to handle and grok-patterns defined in the config toml
 type Parser struct {
-	Patterns []string
+	Patterns []string `toml:"grok_patterns"`
 	// namedPatterns is a list of internally-assigned names to the patterns
 	// specified by the user in Patterns.
 	// They will look like:
 	//   GROK_INTERNAL_PATTERN_0, GROK_INTERNAL_PATTERN_1, etc.
-	NamedPatterns      []string
-	CustomPatterns     string
-	CustomPatternFiles []string
-	Measurement        string
-	DefaultTags        map[string]string
-	Log                telegraf.Logger `toml:"-"`
+	NamedPatterns      []string          `toml:"grok_named_patterns"`
+	CustomPatterns     string            `toml:"grok_custom_patterns"`
+	CustomPatternFiles []string          `toml:"grok_custom_pattern_files"`
+	Measurement        string            `toml:"-"`
+	DefaultTags        map[string]string `toml:"-"`
+	Log                telegraf.Logger   `toml:"-"`
 
 	// Timezone is an optional component to help render log dates to
 	// your chosen zone.
@@ -85,11 +86,11 @@ type Parser struct {
 	// 1. Local             -- interpret based on machine localtime
 	// 2. "America/Chicago" -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 	// 3. UTC               -- or blank/unspecified, will return timestamp in UTC
-	Timezone string
+	Timezone string `toml:"grok_timezone"`
 	loc      *time.Location
 
 	// UniqueTimestamp when set to "disable", timestamp will not incremented if there is a duplicate.
-	UniqueTimestamp string
+	UniqueTimestamp string `toml:"grok_unique_timestamp"`
 
 	// typeMap is a map of patterns -> capture name -> modifier,
 	//   ie, {
@@ -558,3 +559,43 @@ func (t *tsModder) tsMod(ts time.Time) time.Time {
 	}
 	return ts.Add(t.incr*t.incrn + t.rollover)
 }
+
+// InitFromConfig is a compatibility function to construct the parser the old way
+func (p *Parser) InitFromConfig(config *parsers.Config) error {
+	p.Measurement = config.MetricName
+	p.DefaultTags = config.DefaultTags
+	p.CustomPatterns = config.GrokCustomPatterns
+	p.CustomPatternFiles = config.GrokCustomPatternFiles
+	p.NamedPatterns = config.GrokNamedPatterns
+	p.Patterns = config.GrokPatterns
+	p.Timezone = config.GrokTimezone
+	p.UniqueTimestamp = config.GrokUniqueTimestamp
+
+	return p.Init()
+}
+
+func (p *Parser) Init() error {
+	if len(p.Patterns) == 0 {
+		p.Patterns = []string{"%{COMBINED_LOG_FORMAT}"}
+	}
+
+	if p.UniqueTimestamp == "" {
+		p.UniqueTimestamp = "auto"
+	}
+
+	if p.Timezone == "" {
+		p.Timezone = "Canada/Eastern"
+	}
+
+	return p.Compile()
+}
+
+func init() {
+	parsers.Add("grok",
+		func(defaultMetricName string) telegraf.Parser {
+			return &Parser{
+				Measurement: defaultMetricName,
+			}
+		},
+	)
+}
diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/parsers/grok"
 	"github.com/influxdata/telegraf/plugins/parsers/influx"
 	"github.com/influxdata/telegraf/plugins/parsers/influx/influx_upstream"
 	"github.com/influxdata/telegraf/plugins/parsers/nagios"
@@ -206,15 +205,6 @@ func NewParser(config *Config) (Parser, error) {
 		}
 	case "nagios":
 		parser, err = NewNagiosParser()
-	case "grok":
-		parser, err = newGrokParser(
-			config.MetricName,
-			config.GrokPatterns,
-			config.GrokNamedPatterns,
-			config.GrokCustomPatterns,
-			config.GrokCustomPatternFiles,
-			config.GrokTimezone,
-			config.GrokUniqueTimestamp)
 	case "prometheus":
 		parser, err = NewPrometheusParser(
 			config.DefaultTags,
@@ -240,24 +230,6 @@ func NewParser(config *Config) (Parser, error) {
 	return parser, err
 }
 
-func newGrokParser(metricName string,
-	patterns []string, nPatterns []string,
-	cPatterns string, cPatternFiles []string,
-	tZone string, uniqueTimestamp string) (Parser, error) {
-	parser := grok.Parser{
-		Measurement:        metricName,
-		Patterns:           patterns,
-		NamedPatterns:      nPatterns,
-		CustomPatterns:     cPatterns,
-		CustomPatternFiles: cPatternFiles,
-		Timezone:           tZone,
-		UniqueTimestamp:    uniqueTimestamp,
-	}
-
-	err := parser.Compile()
-	return &parser, err
-}
-
 func NewNagiosParser() (Parser, error) {
 	return &nagios.NagiosParser{}, nil
 }