From a45355dfc564134c8504a5941d353432b4b674b2 Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 11:03:39 +0200 Subject: [PATCH 01/13] Add add_locale processor --- libbeat/processors/add_locale/add_locale.go | 50 +++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 libbeat/processors/add_locale/add_locale.go diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go new file mode 100644 index 00000000000..4c6825c3bb2 --- /dev/null +++ b/libbeat/processors/add_locale/add_locale.go @@ -0,0 +1,50 @@ +package actions + +import ( + "time" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/processors" + "github.com/pkg/errors" +) + +type addLocale struct { + timezone string +} + +// init registers the add_locale processor. +func init() { + processors.RegisterPlugin("add_locale", + configChecked(newAddLocale, + requireFields("timezone"))) +} + +func newAddLocale(c common.Config) (processors.Processor, error) { + config := struct { + TimeZone string `config:"timezone"` + }{} + + err := c.Unpack(&config) + if err != nil { + return nil, errors.Wrap(err, "failed to unpack add_locale config") + } + + l := addLocale{timezone: config.TimeZone} + + return l, nil +} + +func (l addLocale) Run(event common.MapStr) (common.MapStr, error) { + zone, err := time.LoadLocation(l.timezone) + + if err != nil { + return event, err + } + + event.Put("beat.timezone", zone.String()) + return event, nil +} + +func (l addLocale) String() string { + return "add_locale=" + l.timezone +} From b2ad649ec843556e544011332b4fa32286d7283c Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 13:50:09 +0200 Subject: [PATCH 02/13] register processor --- libbeat/beat/beat.go | 1 + libbeat/processors/add_locale/add_locale.go | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index be9289b5fa8..a5099c84fc5 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -63,6 +63,7 @@ import ( // Register default processors. _ "github.com/elastic/beats/libbeat/processors/actions" _ "github.com/elastic/beats/libbeat/processors/add_cloud_metadata" + _ "github.com/elastic/beats/libbeat/processors/add_locale" // Register default monitoring reporting _ "github.com/elastic/beats/libbeat/monitoring/report/elasticsearch" diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go index 4c6825c3bb2..e3c0afed727 100644 --- a/libbeat/processors/add_locale/add_locale.go +++ b/libbeat/processors/add_locale/add_locale.go @@ -12,11 +12,8 @@ type addLocale struct { timezone string } -// init registers the add_locale processor. func init() { - processors.RegisterPlugin("add_locale", - configChecked(newAddLocale, - requireFields("timezone"))) + processors.RegisterPlugin("add_locale", newAddLocale) } func newAddLocale(c common.Config) (processors.Processor, error) { From 88cc4e74cf79fa4137abd105035d290f7911941d Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 14:26:23 +0200 Subject: [PATCH 03/13] Add tests --- .../processors/add_locale/add_locale_test.go | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 libbeat/processors/add_locale/add_locale_test.go diff --git a/libbeat/processors/add_locale/add_locale_test.go b/libbeat/processors/add_locale/add_locale_test.go new file mode 100644 index 00000000000..2f3e5c8d129 --- /dev/null +++ b/libbeat/processors/add_locale/add_locale_test.go @@ -0,0 +1,60 @@ +package actions + +import ( + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +func TestExportTimeZone(t *testing.T) { + var testConfig, _ = common.NewConfigFrom(map[string]interface{}{ + "timezone": "America/Curacao", + }) + + input := common.MapStr{} + + actual := getActualValue(t, testConfig, input) + + expected := common.MapStr{ + "beat": map[string]string{ + "timezone": "America/Curacao", + }, + } + + assert.Equal(t, expected.String(), actual.String()) +} + +func TestExportDefaultTimeZone(t *testing.T) { + var testConfig, _ = common.NewConfigFrom(map[string]interface{}{ + "timezone": "", + }) + input := common.MapStr{} + + actual := getActualValue(t, testConfig, input) + + expected := common.MapStr{ + "beat": map[string]string{ + "timezone": "UTC", + }, + } + + assert.Equal(t, expected.String(), actual.String()) +} + +func getActualValue(t *testing.T, config *common.Config, input common.MapStr) common.MapStr { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + p, err := newAddLocale(*config) + if err != nil { + logp.Err("Error initializing add_locale") + t.Fatal(err) + } + + actual, err := p.Run(input) + + return actual +} From 7d2782e72a15560a222496d1f405cc869fda66a1 Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 15:24:09 +0200 Subject: [PATCH 04/13] Change handling of getting local timezone --- libbeat/processors/add_locale/add_locale.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go index e3c0afed727..951e8eed366 100644 --- a/libbeat/processors/add_locale/add_locale.go +++ b/libbeat/processors/add_locale/add_locale.go @@ -18,7 +18,6 @@ func init() { func newAddLocale(c common.Config) (processors.Processor, error) { config := struct { - TimeZone string `config:"timezone"` }{} err := c.Unpack(&config) @@ -26,19 +25,16 @@ func newAddLocale(c common.Config) (processors.Processor, error) { return nil, errors.Wrap(err, "failed to unpack add_locale config") } - l := addLocale{timezone: config.TimeZone} + zone, _ := time.Now().In(time.Local).Zone() + + l := addLocale{timezone: zone} return l, nil } func (l addLocale) Run(event common.MapStr) (common.MapStr, error) { - zone, err := time.LoadLocation(l.timezone) - - if err != nil { - return event, err - } + event.Put("beat.timezone", l.timezone) - event.Put("beat.timezone", zone.String()) return event, nil } From 7f84cb35d1d331913ec99edb5469ea1f913c95f9 Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 15:29:02 +0200 Subject: [PATCH 05/13] change test --- .../processors/add_locale/add_locale_test.go | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/libbeat/processors/add_locale/add_locale_test.go b/libbeat/processors/add_locale/add_locale_test.go index 2f3e5c8d129..34b20e3f3a9 100644 --- a/libbeat/processors/add_locale/add_locale_test.go +++ b/libbeat/processors/add_locale/add_locale_test.go @@ -2,6 +2,7 @@ package actions import ( "testing" + "time" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -9,34 +10,17 @@ import ( ) func TestExportTimeZone(t *testing.T) { - var testConfig, _ = common.NewConfigFrom(map[string]interface{}{ - "timezone": "America/Curacao", - }) + var testConfig = common.NewConfig() input := common.MapStr{} - actual := getActualValue(t, testConfig, input) - - expected := common.MapStr{ - "beat": map[string]string{ - "timezone": "America/Curacao", - }, - } - - assert.Equal(t, expected.String(), actual.String()) -} - -func TestExportDefaultTimeZone(t *testing.T) { - var testConfig, _ = common.NewConfigFrom(map[string]interface{}{ - "timezone": "", - }) - input := common.MapStr{} + zone, _ := time.Now().In(time.Local).Zone() actual := getActualValue(t, testConfig, input) expected := common.MapStr{ "beat": map[string]string{ - "timezone": "UTC", + "timezone": zone, }, } From dd74400daf1c24aa0c3b7d20c87a68369367f740 Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 15:44:29 +0200 Subject: [PATCH 06/13] Remove unnecassary config --- libbeat/processors/add_locale/add_locale.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go index 951e8eed366..d66650ce8e5 100644 --- a/libbeat/processors/add_locale/add_locale.go +++ b/libbeat/processors/add_locale/add_locale.go @@ -5,7 +5,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" - "github.com/pkg/errors" ) type addLocale struct { @@ -17,14 +16,6 @@ func init() { } func newAddLocale(c common.Config) (processors.Processor, error) { - config := struct { - }{} - - err := c.Unpack(&config) - if err != nil { - return nil, errors.Wrap(err, "failed to unpack add_locale config") - } - zone, _ := time.Now().In(time.Local).Zone() l := addLocale{timezone: zone} From eb4dea6a3e466960a20c8dc2807a525e9969f94f Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 15:50:49 +0200 Subject: [PATCH 07/13] Simplified get zone --- libbeat/processors/add_locale/add_locale.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go index d66650ce8e5..3342458e6e6 100644 --- a/libbeat/processors/add_locale/add_locale.go +++ b/libbeat/processors/add_locale/add_locale.go @@ -16,7 +16,7 @@ func init() { } func newAddLocale(c common.Config) (processors.Processor, error) { - zone, _ := time.Now().In(time.Local).Zone() + zone, _ := time.Now().Zone() l := addLocale{timezone: zone} From a06dd06dcbf8e05b398260703f21ba82632821bc Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 4 Apr 2017 16:33:26 +0200 Subject: [PATCH 08/13] Update changelog --- CHANGELOG.asciidoc | 1 + filebeat/filebeat.full.yml | 5 +++++ heartbeat/heartbeat.full.yml | 5 +++++ libbeat/_meta/config.full.yml | 5 +++++ metricbeat/metricbeat.full.yml | 5 +++++ packetbeat/packetbeat.full.yml | 5 +++++ winlogbeat/winlogbeat.full.yml | 5 +++++ 7 files changed, 31 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index bd30c011fd8..df61c44d6b3 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -85,6 +85,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Disabled date detection in Elasticsearch index templates. Date fields must be explicitly defined in index templates. {pull}3528[3528] - Using environment variables in the configuration file is now GA, instead of experimental. {pull}3525[3525] - Initialize a beats UUID from file on startup. {pull}3615[3615] +- Add new `add_locale` processor to export the local timezone with an event. {pull}3902[3902] *Filebeat* diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index f3caf49222a..b7c383779e8 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -419,6 +419,11 @@ filebeat.prospectors: #processors: #- add_cloud_metadata: # +# The following example enriches each event with the local timezone. +# +#processors: +#- add_locale: +# #================================ Outputs ====================================== diff --git a/heartbeat/heartbeat.full.yml b/heartbeat/heartbeat.full.yml index a30820786b9..d6355208c21 100644 --- a/heartbeat/heartbeat.full.yml +++ b/heartbeat/heartbeat.full.yml @@ -267,6 +267,11 @@ heartbeat.scheduler: #processors: #- add_cloud_metadata: # +# The following example enriches each event with the local timezone. +# +#processors: +#- add_locale: +# #================================ Outputs ====================================== diff --git a/libbeat/_meta/config.full.yml b/libbeat/_meta/config.full.yml index dfd7e493b44..547d92882ba 100644 --- a/libbeat/_meta/config.full.yml +++ b/libbeat/_meta/config.full.yml @@ -69,6 +69,11 @@ #processors: #- add_cloud_metadata: # +# The following example enriches each event with the local timezone. +# +#processors: +#- add_locale: +# #================================ Outputs ====================================== diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index e2811154ac1..90d4d0bfd52 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -401,6 +401,11 @@ metricbeat.modules: #processors: #- add_cloud_metadata: # +# The following example enriches each event with the local timezone. +# +#processors: +#- add_locale: +# #================================ Outputs ====================================== diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 24b5b64a1a6..dfe32e1ef80 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -524,6 +524,11 @@ packetbeat.protocols: #processors: #- add_cloud_metadata: # +# The following example enriches each event with the local timezone. +# +#processors: +#- add_locale: +# #================================ Outputs ====================================== diff --git a/winlogbeat/winlogbeat.full.yml b/winlogbeat/winlogbeat.full.yml index 5851054c52d..d1abe2ea86f 100644 --- a/winlogbeat/winlogbeat.full.yml +++ b/winlogbeat/winlogbeat.full.yml @@ -104,6 +104,11 @@ winlogbeat.event_logs: #processors: #- add_cloud_metadata: # +# The following example enriches each event with the local timezone. +# +#processors: +#- add_locale: +# #================================ Outputs ====================================== From e6353be7a4b0d619c51cc5f921a99b5bea97027e Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Wed, 5 Apr 2017 12:57:16 +0200 Subject: [PATCH 09/13] Add description for config.asciidoc --- libbeat/docs/processors-config.asciidoc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libbeat/docs/processors-config.asciidoc b/libbeat/docs/processors-config.asciidoc index b7a71bc17c9..c8ba6e70d28 100644 --- a/libbeat/docs/processors-config.asciidoc +++ b/libbeat/docs/processors-config.asciidoc @@ -253,6 +253,7 @@ not: The supported processors are: * <> + * <> * <> * <> * <> @@ -340,6 +341,23 @@ _GCE_ } ------------------------------------------------------------------------------- +[[add-locale]] +=== add_locale + +The `add_locale` processor enriches each event with the machine's time zone. + +The simple configuration below enables the processor. + +[source,yaml] +------------------------------------------------------------------------------- +processors: +- add_locale: +------------------------------------------------------------------------------- + +NOTE: Please consider that `add_locale` differentiate between DST and regular time. +For example `CET` and `CEST`. + + [[decode-json-fields]] === decode_json_fields From 6914c9a25b43aa9179e20a451d07f6c4323dbc33 Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Wed, 5 Apr 2017 13:51:22 +0200 Subject: [PATCH 10/13] Fix grammar config.asciidoc --- libbeat/docs/processors-config.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/docs/processors-config.asciidoc b/libbeat/docs/processors-config.asciidoc index c8ba6e70d28..277bd7ec685 100644 --- a/libbeat/docs/processors-config.asciidoc +++ b/libbeat/docs/processors-config.asciidoc @@ -354,7 +354,7 @@ processors: - add_locale: ------------------------------------------------------------------------------- -NOTE: Please consider that `add_locale` differentiate between DST and regular time. +NOTE: Please consider that `add_locale` differentiates between DST and regular time. For example `CET` and `CEST`. From ed0196789afcc85a4ac0702d7b4979911d435acd Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Wed, 5 Apr 2017 18:07:50 +0200 Subject: [PATCH 11/13] Add timezone to fields.common.yml --- libbeat/_meta/fields.common.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libbeat/_meta/fields.common.yml b/libbeat/_meta/fields.common.yml index e21047c38b5..0992f0ce645 100644 --- a/libbeat/_meta/fields.common.yml +++ b/libbeat/_meta/fields.common.yml @@ -15,6 +15,10 @@ description: > The hostname as returned by the operating system on which the Beat is running. + - name: beat.timezone + description: > + The timezone as returned by the operating system on which the Beat is + running. - name: beat.version description: > The version of the beat that generated this event. From 94dca42c5f639e898de6e6c3d3f412cf5f448a84 Mon Sep 17 00:00:00 2001 From: maddin2016 Date: Fri, 7 Apr 2017 08:35:55 +0200 Subject: [PATCH 12/13] Fetch timezone in run method --- libbeat/processors/add_locale/add_locale.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go index 3342458e6e6..9c7ecef5407 100644 --- a/libbeat/processors/add_locale/add_locale.go +++ b/libbeat/processors/add_locale/add_locale.go @@ -7,28 +7,23 @@ import ( "github.com/elastic/beats/libbeat/processors" ) -type addLocale struct { - timezone string -} +type addLocale struct{} func init() { processors.RegisterPlugin("add_locale", newAddLocale) } func newAddLocale(c common.Config) (processors.Processor, error) { - zone, _ := time.Now().Zone() - - l := addLocale{timezone: zone} - - return l, nil + return addLocale{}, nil } func (l addLocale) Run(event common.MapStr) (common.MapStr, error) { - event.Put("beat.timezone", l.timezone) + zone, _ := time.Now().Zone() + event.Put("beat.timezone", zone) return event, nil } func (l addLocale) String() string { - return "add_locale=" + l.timezone + return "add_locale" } From f5f4b4cb40b99d142fbb7e43c0a4af819424aa41 Mon Sep 17 00:00:00 2001 From: maddin2016 Date: Fri, 7 Apr 2017 09:05:17 +0200 Subject: [PATCH 13/13] Add benchmark test --- filebeat/docs/fields.asciidoc | 6 ++++++ heartbeat/docs/fields.asciidoc | 6 ++++++ libbeat/processors/add_locale/add_locale_test.go | 15 +++++++++++++++ metricbeat/docs/fields.asciidoc | 6 ++++++ packetbeat/docs/fields.asciidoc | 6 ++++++ winlogbeat/docs/fields.asciidoc | 6 ++++++ 6 files changed, 45 insertions(+) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index eb5550bec47..de7655f0358 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -304,6 +304,12 @@ The name of the Beat sending the log messages. If the Beat name is set in the co The hostname as returned by the operating system on which the Beat is running. +[float] +=== beat.timezone + +The timezone as returned by the operating system on which the Beat is running. + + [float] === beat.version diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index 7bee2fdef6b..58b338ca337 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -36,6 +36,12 @@ The name of the Beat sending the log messages. If the Beat name is set in the co The hostname as returned by the operating system on which the Beat is running. +[float] +=== beat.timezone + +The timezone as returned by the operating system on which the Beat is running. + + [float] === beat.version diff --git a/libbeat/processors/add_locale/add_locale_test.go b/libbeat/processors/add_locale/add_locale_test.go index 34b20e3f3a9..3d2b31a9396 100644 --- a/libbeat/processors/add_locale/add_locale_test.go +++ b/libbeat/processors/add_locale/add_locale_test.go @@ -42,3 +42,18 @@ func getActualValue(t *testing.T, config *common.Config, input common.MapStr) co return actual } + +func BenchmarkConstruct(b *testing.B) { + var testConfig = common.NewConfig() + + input := common.MapStr{} + + p, err := newAddLocale(*testConfig) + if err != nil { + b.Fatal(err) + } + + for i := 0; i < b.N; i++ { + _, err = p.Run(input) + } +} diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 119925546a6..9e920ca5d95 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -383,6 +383,12 @@ The name of the Beat sending the log messages. If the Beat name is set in the co The hostname as returned by the operating system on which the Beat is running. +[float] +=== beat.timezone + +The timezone as returned by the operating system on which the Beat is running. + + [float] === beat.version diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 5009ceb677b..e553e41afd3 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -375,6 +375,12 @@ The name of the Beat sending the log messages. If the Beat name is set in the co The hostname as returned by the operating system on which the Beat is running. +[float] +=== beat.timezone + +The timezone as returned by the operating system on which the Beat is running. + + [float] === beat.version diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index 7826b431a75..7085b9e7512 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -37,6 +37,12 @@ The name of the Beat sending the log messages. If the Beat name is set in the co The hostname as returned by the operating system on which the Beat is running. +[float] +=== beat.timezone + +The timezone as returned by the operating system on which the Beat is running. + + [float] === beat.version