From 9c706ee1f3123b35f1554cfb5515bc787306dfe5 Mon Sep 17 00:00:00 2001 From: adam Date: Fri, 1 Sep 2023 10:30:17 -0700 Subject: [PATCH 1/3] fix: remove incorrect forced addition of timestamp to fields --- plugins/parsers/avro/parser.go | 17 ----------------- .../avro/testdata/json-format/message.json | 2 +- .../avro/testdata/json-format/telegraf.conf | 4 ++-- .../expected.out | 2 +- .../telegraf.conf | 6 ------ 5 files changed, 4 insertions(+), 27 deletions(-) diff --git a/plugins/parsers/avro/parser.go b/plugins/parsers/avro/parser.go index 0b7d95dc953bd..d3904b8955431 100644 --- a/plugins/parsers/avro/parser.go +++ b/plugins/parsers/avro/parser.go @@ -173,23 +173,6 @@ func (p *Parser) createMetric(data map[string]interface{}, schema string) (teleg // get what you asked for. fieldList = p.Fields - // Except...if you specify the timestamp field, and it's - // not listed in your fields, you'll get it anyway. - // This will randomize your field ordering, which isn't - // ideal. If you care, list the timestamp field. - if p.Timestamp != "" { - // quick list-to-set-to-list implementation - fieldSet := make(map[string]bool) - for k := range fieldList { - fieldSet[fieldList[k]] = true - } - fieldSet[p.Timestamp] = true - var newList []string - for s := range fieldSet { - newList = append(newList, s) - } - fieldList = newList - } } else { for k := range data { // Otherwise, that which is not a tag is a field diff --git a/plugins/parsers/avro/testdata/json-format/message.json b/plugins/parsers/avro/testdata/json-format/message.json index 5420b8718326d..4c7104a0a03a6 100644 --- a/plugins/parsers/avro/testdata/json-format/message.json +++ b/plugins/parsers/avro/testdata/json-format/message.json @@ -4,4 +4,4 @@ "up_time": 1166984904, "cpu_utilization": 14.0, "memory_utilization": 20.0 -} \ No newline at end of file +} diff --git a/plugins/parsers/avro/testdata/json-format/telegraf.conf b/plugins/parsers/avro/testdata/json-format/telegraf.conf index d61ece9898760..ddf8ccff9760f 100644 --- a/plugins/parsers/avro/testdata/json-format/telegraf.conf +++ b/plugins/parsers/avro/testdata/json-format/telegraf.conf @@ -5,7 +5,7 @@ avro_format = "json" avro_measurement = "Switch" avro_tags = ["switch_wwn"] - avro_fields = ["up_time", "cpu_utilization", "memory_utilization"] + avro_fields = ["up_time", "cpu_utilization", "memory_utilization", "statistics_collection_time"] avro_timestamp = "statistics_collection_time" avro_timestamp_format = "unix_ms" avro_schema = ''' @@ -22,4 +22,4 @@ {"name": "memory_utilization", "type": "float", "default": 0, "doc": "Memory Utilization in %"} ] } - ''' \ No newline at end of file + ''' diff --git a/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/expected.out b/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/expected.out index c831d4ad47627..f5479404f4a94 100644 --- a/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/expected.out +++ b/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/expected.out @@ -1 +1 @@ -measurement,tag=test_tag field=19i,timestamp=1664296121000000i 1664296121000000 +measurement,tag=test_tag field=19i 1664296121000000 diff --git a/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/telegraf.conf b/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/telegraf.conf index 257a9ea696192..d5c58355ebd92 100644 --- a/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/telegraf.conf +++ b/plugins/parsers/avro/testdata/supplied_timestamp_fields_unspecified/telegraf.conf @@ -4,8 +4,6 @@ avro_measurement = "measurement" avro_tags = [ "tag" ] avro_fields = [ "field" ] - avro_timestamp = "timestamp" - avro_timestamp_format = "unix_us" avro_schema = ''' { "type":"record", @@ -19,10 +17,6 @@ { "name":"field", "type":"long" - }, - { - "name":"timestamp", - "type":"long" } ] } From 29ae4d2e9e739d9083b3daa7ae391f55b5082fac Mon Sep 17 00:00:00 2001 From: adam Date: Fri, 1 Sep 2023 11:10:02 -0700 Subject: [PATCH 2/3] Fix linting --- plugins/parsers/avro/parser.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/parsers/avro/parser.go b/plugins/parsers/avro/parser.go index d3904b8955431..55f9072fa7381 100644 --- a/plugins/parsers/avro/parser.go +++ b/plugins/parsers/avro/parser.go @@ -172,7 +172,6 @@ func (p *Parser) createMetric(data map[string]interface{}, schema string) (teleg // If you have specified your fields in the config, you // get what you asked for. fieldList = p.Fields - } else { for k := range data { // Otherwise, that which is not a tag is a field From a7905838bb978c0dc2772e23e88ab6d7438ef68b Mon Sep 17 00:00:00 2001 From: adam Date: Fri, 1 Sep 2023 13:02:06 -0700 Subject: [PATCH 3/3] take timestamp from raw input data, not from field --- plugins/parsers/avro/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/parsers/avro/parser.go b/plugins/parsers/avro/parser.go index 55f9072fa7381..83aebc2007779 100644 --- a/plugins/parsers/avro/parser.go +++ b/plugins/parsers/avro/parser.go @@ -237,7 +237,7 @@ func (p *Parser) createMetric(data map[string]interface{}, schema string) (teleg } var timestamp time.Time if p.Timestamp != "" { - rawTime := fmt.Sprintf("%v", fields[p.Timestamp]) + rawTime := fmt.Sprintf("%v", data[p.Timestamp]) var err error timestamp, err = internal.ParseTimestamp(p.TimestampFormat, rawTime, nil) if err != nil {