Add filtering option for prometheus collector

metricbeat/helper/prometheus/prometheus.go

@@ -98,13 +98,48 @@ func (p *prometheus) GetFamilies() ([]*dto.MetricFamily, error) {
 			}
 			return nil, errors.Wrap(err, "decoding of metric family failed")
 		} else {
+			//families = p.filterFamilies(families)
 			families = append(families, mf)
 		}
 	}
 
 	return families, nil
 }
 
+//func (p *prometheus) filterFamilies(families []*dto.MetricFamily) []*dto.MetricFamily {
+//	var ignoreMetrics *[]string
+//	// ignoreMetrics := &[]string{"go_*"}
+//	var includeMetrics *[]string
+//	includeMetrics = &[]string{"go_*"}
+//	filteredFamilies := []*dto.MetricFamily{}
+//	for _, family := range families {
+//		if includeMetrics != nil {
+//			if matchMetricFamily(*family.Name, includeMetrics){
+//				filteredFamilies = append(filteredFamilies, family)
+//			} else {
+//				continue
+//			}
+//		} else if ignoreMetrics != nil {
+//			if matchMetricFamily(*family.Name, ignoreMetrics){
+//				continue
+//			} else {
+//				filteredFamilies = append(filteredFamilies, family)
+//			}
+//		}
+//	}
+//	return filteredFamilies
+//}
+//
+//func matchMetricFamily(family string, matchMetrics *[]string) bool {
+//	for _, checkMetric := range *matchMetrics {
+//		matched, _ := regexp.MatchString(checkMetric, family)
+//		if matched {
+//			return true
+//		}
+//	}
+//	return false
+//}
+
 // MetricsMapping defines mapping settings for Prometheus metrics, to be used with `GetProcessedMetrics`
 type MetricsMapping struct {
 	// Metrics translates from prometheus metric name to Metricbeat fields

metricbeat/module/prometheus/collector/collector.go

@@ -18,6 +18,8 @@
 package collector
 
 import (
+	"regexp"
+
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -49,19 +51,27 @@ func init() {
 // MetricSet for fetching prometheus data
 type MetricSet struct {
 	mb.BaseMetricSet
-	prometheus p.Prometheus
+	prometheus     p.Prometheus
+	includeMetrics *[]string
+	ignoreMetrics  *[]string
 }
 
 // New creates a new metricset
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	config := defaultConfig
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
 	prometheus, err := p.NewPrometheusClient(base)
 	if err != nil {
 		return nil, err
 	}
 
 	return &MetricSet{
-		BaseMetricSet: base,
-		prometheus:    prometheus,
+		BaseMetricSet:  base,
+		prometheus:     prometheus,
+		ignoreMetrics:  config.MetricsFilters.IgnoreMetrics,
+		includeMetrics: config.MetricsFilters.IncludeMetrics,
 	}, nil
 }
 
@@ -81,6 +91,16 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 	}
 
 	for _, family := range families {
+		if m.includeMetrics != nil {
+			if !matchMetricFamily(*family.Name, m.includeMetrics) {
+				continue
+			}
+		} else if m.ignoreMetrics != nil {
+			if matchMetricFamily(*family.Name, m.ignoreMetrics) {
+				continue
+			}
+		}
+
 		promEvents := getPromEventsFromMetricFamily(family)
 
 		for _, promEvent := range promEvents {
@@ -140,3 +160,13 @@ func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) {
 	}
 
 }
+
+func matchMetricFamily(family string, matchMetrics *[]string) bool {
+	for _, checkMetric := range *matchMetrics {
+		matched, _ := regexp.MatchString(checkMetric, family)
+		if matched {
+			return true
+		}
+	}
+	return false
+}

metricbeat/module/prometheus/collector/config.go

@@ -0,0 +1,45 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package collector
+
+import (
+	"github.com/elastic/beats/libbeat/logp"
+)
+
+type metricsetConfig struct {
+	MetricsFilters MetricFilters `config:"metrics_filters"`
+}
+
+type MetricFilters struct {
+	IncludeMetrics *[]string `config:"include_metrics"`
+	IgnoreMetrics  *[]string `config:"ignore_metrics"`
+}
+
+var defaultConfig = metricsetConfig{
+	MetricsFilters: MetricFilters{
+		IncludeMetrics: nil,
+		IgnoreMetrics:  nil},
+}
+
+func (c *metricsetConfig) Validate() error {
+	if c.MetricsFilters.IncludeMetrics != nil && c.MetricsFilters.IgnoreMetrics == nil {
+		logp.Debug("prometheus", "include_metrics and ignore_metrics are complementary and cannot be used together. Falling back to using only include_metrics")
+		c.MetricsFilters.IgnoreMetrics = nil
+	}
+	return nil
+}