Add filtering option for prometheus collector

metricbeat/module/prometheus/collector/collector.go

@@ -18,7 +18,10 @@
 package collector
 
 import (
+	"regexp"
+
 	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
 
 	"github.com/elastic/beats/libbeat/common"
 	p "github.com/elastic/beats/metricbeat/helper/prometheus"
@@ -49,20 +52,36 @@ func init() {
 // MetricSet for fetching prometheus data
 type MetricSet struct {
 	mb.BaseMetricSet
-	prometheus p.Prometheus
+	prometheus     p.Prometheus
+	includeMetrics []*regexp.Regexp
+	ignoreMetrics  []*regexp.Regexp
 }
 
 // New creates a new metricset
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	config := defaultConfig
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
 	prometheus, err := p.NewPrometheusClient(base)
 	if err != nil {
 		return nil, err
 	}
 
-	return &MetricSet{
+	ms := &MetricSet{
 		BaseMetricSet: base,
 		prometheus:    prometheus,
-	}, nil
+	}
+	ms.ignoreMetrics, err = compilePatternList(config.MetricsFilters.IgnoreMetrics)
+	if err != nil {
+		return nil, err
+	}
+	ms.includeMetrics, err = compilePatternList(config.MetricsFilters.IncludeMetrics)
+	if err != nil {
+		return nil, err
+	}
+
+	return ms, nil
 }
 
 // Fetch fetches data and reports it
@@ -81,6 +100,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 	}
 
 	for _, family := range families {
+		if m.skipFamily(family){
+			continue
+		}
 		promEvents := getPromEventsFromMetricFamily(family)
 
 		for _, promEvent := range promEvents {
@@ -140,3 +162,58 @@ func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) {
 	}
 
 }
+
+func (m *MetricSet) skipFamily(family *dto.MetricFamily) bool {
+	// example:
+	//	include_metrics:
+	//		- node_*
+	//	exclude_metrics:
+	//		- node_disk_*
+	//
+	// This would mean that we want to keep only the metrics that start with node_ prefix but
+	// are not related to disk so we exclude node_disk_* metrics from them.
+
+	if family == nil {
+		return true
+	}
+
+	// if include_metrics are defined, check if this metric should be included
+	if len(m.includeMetrics) > 0  {
+		if !matchMetricFamily(*family.Name, m.includeMetrics) {
+			return true
+		}
+	}
+	// now exclude the metric if it matches any of the given patterns
+	if len(m.ignoreMetrics) > 0 {
+		if matchMetricFamily(*family.Name, m.ignoreMetrics) {
+			return true
+		}
+	}
+	return false
+}
+
+func compilePatternList(patterns *[]string) ([]*regexp.Regexp, error){
+	var compiledPatterns []*regexp.Regexp
+	compiledPatterns = []*regexp.Regexp{}
+	if patterns != nil {
+		for _, pattern := range *patterns {
+			r, err := regexp.Compile(pattern)
+			if err != nil {
+				return nil, err
+			}
+			compiledPatterns = append(compiledPatterns, r)
+		}
+		return compiledPatterns, nil
+	}
+	return []*regexp.Regexp{}, nil
+}
+
+func matchMetricFamily(family string, matchMetrics []*regexp.Regexp) bool {
+	for _, checkMetric := range matchMetrics {
+		matched := checkMetric.MatchString(family)
+		if matched {
+			return true
+		}
+	}
+	return false
+}

metricbeat/module/prometheus/collector/config.go

@@ -0,0 +1,38 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package collector
+
+type metricsetConfig struct {
+	MetricsFilters MetricFilters `config:"metrics_filters"`
+}
+
+type MetricFilters struct {
+	IncludeMetrics *[]string `config:"include_metrics"`
+	IgnoreMetrics  *[]string `config:"ignore_metrics"`
+}
+
+var defaultConfig = metricsetConfig{
+	MetricsFilters: MetricFilters{
+		IncludeMetrics: nil,
+		IgnoreMetrics:  nil},
+}
+
+func (c *metricsetConfig) Validate() error {
+	// validate configuration here
+	return nil
+}