From 3aa126f4f82faafeb60527370c0da40d9afe17bc Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Mon, 20 Nov 2017 23:22:29 +0100 Subject: [PATCH] Add index by week number to Elasticsearch output (#3490) --- plugins/outputs/elasticsearch/README.md | 3 ++- plugins/outputs/elasticsearch/elasticsearch.go | 16 ++++++++++++---- .../outputs/elasticsearch/elasticsearch_test.go | 5 +++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/outputs/elasticsearch/README.md b/plugins/outputs/elasticsearch/README.md index b69631ba81832..d2c84a8d3285c 100644 --- a/plugins/outputs/elasticsearch/README.md +++ b/plugins/outputs/elasticsearch/README.md @@ -172,6 +172,7 @@ This plugin will format the events in the following way: # %m - month (01..12) # %d - day of month (e.g., 01) # %H - hour (00..23) + # %V - week of the year (ISO week) (01..53) index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -220,6 +221,6 @@ Integer values collected that are bigger than 2^63 and smaller than 1e21 (or in ```{"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"failed to parse"}],"type":"mapper_parsing_exception","reason":"failed to parse","caused_by":{"type":"illegal_state_exception","reason":"No matching token for number_type [BIG_INTEGER]"}},"status":400}``` -The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist. +The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist. This issue is caused by the way Elasticsearch tries to detect integer fields, and by how golang encodes numbers in JSON. There is no clear workaround for this at the moment. \ No newline at end of file diff --git a/plugins/outputs/elasticsearch/elasticsearch.go b/plugins/outputs/elasticsearch/elasticsearch.go index 31a702e55f3ce..a9fd5a49139f1 100644 --- a/plugins/outputs/elasticsearch/elasticsearch.go +++ b/plugins/outputs/elasticsearch/elasticsearch.go @@ -3,15 +3,16 @@ package elasticsearch import ( "context" "fmt" - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/internal" - "github.com/influxdata/telegraf/plugins/outputs" - "gopkg.in/olivere/elastic.v5" "log" "net/http" "strconv" "strings" "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/outputs" + "gopkg.in/olivere/elastic.v5" ) type Elasticsearch struct { @@ -58,6 +59,7 @@ var sampleConfig = ` # %m - month (01..12) # %d - day of month (e.g., 01) # %H - hour (00..23) + # %V - week of the year (ISO week) (01..53) index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -301,6 +303,7 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri "%m", eventTime.UTC().Format("01"), "%d", eventTime.UTC().Format("02"), "%H", eventTime.UTC().Format("15"), + "%V", getISOWeek(eventTime.UTC()), ) indexName = dateReplacer.Replace(indexName) @@ -310,6 +313,11 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri } +func getISOWeek(eventTime time.Time) string { + _, week := eventTime.ISOWeek() + return strconv.Itoa(week) +} + func (a *Elasticsearch) SampleConfig() string { return sampleConfig } diff --git a/plugins/outputs/elasticsearch/elasticsearch_test.go b/plugins/outputs/elasticsearch/elasticsearch_test.go index 9000676d9130c..dadd94da96755 100644 --- a/plugins/outputs/elasticsearch/elasticsearch_test.go +++ b/plugins/outputs/elasticsearch/elasticsearch_test.go @@ -120,6 +120,11 @@ func TestGetIndexName(t *testing.T) { "indexname-%y-%m", "indexname-14-12", }, + { + time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + "indexname-%Y-%V", + "indexname-2014-49", + }, } for _, test := range tests { indexName := e.GetIndexName(test.IndexName, test.EventTime)