From eaeaf64f25c9eb4bb32eb376217794bd3a05fdb7 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 23 Nov 2022 15:26:47 -0500 Subject: [PATCH 1/5] Register the add_formatted_index as a usage processor for the processors list. --- libbeat/cmd/instance/imports_common.go | 1 + libbeat/common/fmtstr/formattimestamp.go | 15 +++++++ .../add_formatted_index.go | 31 +++++++++++++-- .../processors/add_formatted_index/config.go | 39 +++++++++++++++++++ .../docs/add_formatted_index.asciidoc | 23 +++++++++++ .../enterprisesearch-xpack.yml.disabled | 11 ------ 6 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 libbeat/processors/add_formatted_index/config.go create mode 100644 libbeat/processors/add_formatted_index/docs/add_formatted_index.asciidoc delete mode 100644 x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go index 540bef793ab5..be4174c0ea07 100644 --- a/libbeat/cmd/instance/imports_common.go +++ b/libbeat/cmd/instance/imports_common.go @@ -23,6 +23,7 @@ import ( _ "github.com/elastic/beats/v7/libbeat/monitoring/report/elasticsearch" // Register default monitoring reporting _ "github.com/elastic/beats/v7/libbeat/processors/actions" // Register default processors. _ "github.com/elastic/beats/v7/libbeat/processors/add_cloud_metadata" + _ "github.com/elastic/beats/v7/libbeat/processors/add_formatted_index" _ "github.com/elastic/beats/v7/libbeat/processors/add_host_metadata" _ "github.com/elastic/beats/v7/libbeat/processors/add_id" _ "github.com/elastic/beats/v7/libbeat/processors/add_locale" diff --git a/libbeat/common/fmtstr/formattimestamp.go b/libbeat/common/fmtstr/formattimestamp.go index 652fae018d77..30e6df8b35e6 100644 --- a/libbeat/common/fmtstr/formattimestamp.go +++ b/libbeat/common/fmtstr/formattimestamp.go @@ -75,6 +75,21 @@ func (fs *TimestampFormatString) Run(timestamp time.Time) (string, error) { return fs.eventFormatString.Run(placeholderEvent) } +// RunEvent executes the format string returning a new expanded string or an error +// if execution or event field expansion fails. +func (fs *TimestampFormatString) RunEvent(event *beat.Event) (string, error) { + return fs.eventFormatString.Run(event) +} + func (fs *TimestampFormatString) String() string { return fs.eventFormatString.expression } + +// Unpack tries to initialize the TimestampFormatString from provided value +// (which must be a string). Unpack method satisfies go-ucfg.Unpacker interface +// required by config.C, in order to use TimestampFormatString with +// `common.(*Config).Unpack()`. +func (fs *TimestampFormatString) Unpack(v interface{}) error { + fs.eventFormatString = &EventFormatString{} + return fs.eventFormatString.Unpack(v) +} diff --git a/libbeat/processors/add_formatted_index/add_formatted_index.go b/libbeat/processors/add_formatted_index/add_formatted_index.go index 9022b69cff9c..ec4d403e9235 100644 --- a/libbeat/processors/add_formatted_index/add_formatted_index.go +++ b/libbeat/processors/add_formatted_index/add_formatted_index.go @@ -19,29 +19,51 @@ package add_formatted_index import ( "fmt" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/common/fmtstr" + "github.com/elastic/beats/v7/libbeat/processors" + conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" ) +func init() { + processors.RegisterPlugin("add_formatted_index", NewC) +} + // AddFormattedIndex is a Processor to set an event's "raw_index" metadata field // with a given TimestampFormatString. The elasticsearch output interprets // that field as specifying the (raw string) index the event should be sent to; // in other outputs it is just included in the metadata. type AddFormattedIndex struct { formatString *fmtstr.TimestampFormatString + configString *fmtstr.TimestampFormatString } // New returns a new AddFormattedIndex processor. func New(formatString *fmtstr.TimestampFormatString) *AddFormattedIndex { - return &AddFormattedIndex{formatString} + return &AddFormattedIndex{formatString: formatString} +} + +// NewC constructs a new AddFormattedIndex processor from configuration +func NewC(cfg *conf.C) (processors.Processor, error) { + var c config + if err := cfg.Unpack(&c); err != nil { + return nil, err + } + + return &AddFormattedIndex{configString: c.Index}, nil } // Run runs the processor. func (p *AddFormattedIndex) Run(event *beat.Event) (*beat.Event, error) { - index, err := p.formatString.Run(event.Timestamp) + var index string + var err error + if p.configString != nil { + index, err = p.configString.RunEvent(event) + } else { + index, err = p.formatString.Run(event.Timestamp) + } if err != nil { return nil, err } @@ -54,5 +76,8 @@ func (p *AddFormattedIndex) Run(event *beat.Event) (*beat.Event, error) { } func (p *AddFormattedIndex) String() string { + if p.configString != nil { + fmt.Sprintf("add_index_pattern=%v", p.configString) + } return fmt.Sprintf("add_index_pattern=%v", p.formatString) } diff --git a/libbeat/processors/add_formatted_index/config.go b/libbeat/processors/add_formatted_index/config.go new file mode 100644 index 000000000000..6b4e6c8f2b9f --- /dev/null +++ b/libbeat/processors/add_formatted_index/config.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package add_formatted_index + +import ( + "errors" + + "github.com/elastic/beats/v7/libbeat/common/fmtstr" +) + +// configuration for AddFormattedIndex processor. +type config struct { + Index *fmtstr.TimestampFormatString `config:"index"` // Index formatted string value +} + +// Validate ensures that the configuration is valid. +func (c *config) Validate() error { + // Validate type of ID generator + if c.Index == nil { + return errors.New("index field is required") + } + + return nil +} diff --git a/libbeat/processors/add_formatted_index/docs/add_formatted_index.asciidoc b/libbeat/processors/add_formatted_index/docs/add_formatted_index.asciidoc new file mode 100644 index 000000000000..312625a27253 --- /dev/null +++ b/libbeat/processors/add_formatted_index/docs/add_formatted_index.asciidoc @@ -0,0 +1,23 @@ +[[add-locale]] +=== Add formatted index + +++++ +add_formatted_index +++++ + +The `add_formatted_index` processor allows the destination index for the event to +be changed based on a formatted string that can use values from fields defined on +the event. + +For example, this configuration uses a custom field, fields.log_type, to set the index: + +[source,yaml] +------------------------------------------------------------------------------- +processors: + - add_formatted_index: + index: "%{[fields.log_type]}-%{[agent.version]}-%{+yyyy.MM.dd}" +------------------------------------------------------------------------------- + +With this configuration, all events with log_type: normal are sent to an index named +normal-7.10.2-2022-11-18, and all events with log_type: critical are sent to an index +named critical-7.10.2-2022-11-18. diff --git a/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled b/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled deleted file mode 100644 index 0af7916573a0..000000000000 --- a/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: enterprisesearch -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-enterprisesearch.html - -- module: enterprisesearch - xpack.enabled: true - metricsets: ["health", "stats"] - enabled: true - period: 10s - hosts: ["http://localhost:3002"] - #username: "user" - #password: "secret" From 4cf6b29e235a94a6f2f8866c093895da7b01caaf Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 23 Nov 2022 15:32:18 -0500 Subject: [PATCH 2/5] Re-add removed file. --- .../modules.d/enterprisesearch-xpack.yml.disabled | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled diff --git a/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled b/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled new file mode 100644 index 000000000000..0af7916573a0 --- /dev/null +++ b/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled @@ -0,0 +1,11 @@ +# Module: enterprisesearch +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-enterprisesearch.html + +- module: enterprisesearch + xpack.enabled: true + metricsets: ["health", "stats"] + enabled: true + period: 10s + hosts: ["http://localhost:3002"] + #username: "user" + #password: "secret" From 5d7e0decf2faf29502aaf4c393ba8bd40f4485e9 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 23 Nov 2022 15:33:58 -0500 Subject: [PATCH 3/5] Add changelog entry. --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e46b86b33edb..8e5cee7a8691 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -118,6 +118,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Beats will now attempt to recover if a lockfile has not been removed {pull}[33169] - Add `http.pprof` config options for enabling block and mutex profiling. {issue}33572[33572] {pull}33576[33576] - Added append Processor which will append concrete values or values from a field to target. {issue}29934[29934] {pull}33364[33364] +- Add `add_formatted_index` processor that allows the resulting index for an event to be changed based on content from the event. {pull}33800[33800] *Auditbeat* From 8e555732a464e6d844021f1c68158574b7a071cd Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 28 Nov 2022 11:14:00 -0500 Subject: [PATCH 4/5] Add missing return in String(). --- libbeat/processors/add_formatted_index/add_formatted_index.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/add_formatted_index/add_formatted_index.go b/libbeat/processors/add_formatted_index/add_formatted_index.go index ec4d403e9235..c97e8e1dee40 100644 --- a/libbeat/processors/add_formatted_index/add_formatted_index.go +++ b/libbeat/processors/add_formatted_index/add_formatted_index.go @@ -19,6 +19,7 @@ package add_formatted_index import ( "fmt" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/common/fmtstr" @@ -77,7 +78,7 @@ func (p *AddFormattedIndex) Run(event *beat.Event) (*beat.Event, error) { func (p *AddFormattedIndex) String() string { if p.configString != nil { - fmt.Sprintf("add_index_pattern=%v", p.configString) + return fmt.Sprintf("add_index_pattern=%v", p.configString) } return fmt.Sprintf("add_index_pattern=%v", p.formatString) } From c07993ec7c74b8f3c9640261045e3e9e451987fa Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 28 Nov 2022 11:57:21 -0500 Subject: [PATCH 5/5] Change implementation to use a boolean. --- .../add_formatted_index/add_formatted_index.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/libbeat/processors/add_formatted_index/add_formatted_index.go b/libbeat/processors/add_formatted_index/add_formatted_index.go index c97e8e1dee40..6419c28a191a 100644 --- a/libbeat/processors/add_formatted_index/add_formatted_index.go +++ b/libbeat/processors/add_formatted_index/add_formatted_index.go @@ -38,12 +38,12 @@ func init() { // in other outputs it is just included in the metadata. type AddFormattedIndex struct { formatString *fmtstr.TimestampFormatString - configString *fmtstr.TimestampFormatString + fullEvent bool } // New returns a new AddFormattedIndex processor. func New(formatString *fmtstr.TimestampFormatString) *AddFormattedIndex { - return &AddFormattedIndex{formatString: formatString} + return &AddFormattedIndex{formatString, false} } // NewC constructs a new AddFormattedIndex processor from configuration @@ -53,15 +53,15 @@ func NewC(cfg *conf.C) (processors.Processor, error) { return nil, err } - return &AddFormattedIndex{configString: c.Index}, nil + return &AddFormattedIndex{c.Index, true}, nil } // Run runs the processor. func (p *AddFormattedIndex) Run(event *beat.Event) (*beat.Event, error) { var index string var err error - if p.configString != nil { - index, err = p.configString.RunEvent(event) + if p.fullEvent { + index, err = p.formatString.RunEvent(event) } else { index, err = p.formatString.Run(event.Timestamp) } @@ -77,8 +77,5 @@ func (p *AddFormattedIndex) Run(event *beat.Event) (*beat.Event, error) { } func (p *AddFormattedIndex) String() string { - if p.configString != nil { - return fmt.Sprintf("add_index_pattern=%v", p.configString) - } return fmt.Sprintf("add_index_pattern=%v", p.formatString) }