Add filesystem.ignore_types to ignore filesystem types (elastic#4823) (…

…elastic#4906)

* Add filesystem.ignore_types to ignore filesystem types

Add `filesystem.ignore_types` to the system module for ignoring filesystems
in the `filesystem` and `fsstat` metricsets. The new configuration option
accepts a list of filesystem types.

    metricbeat.modules:
    - module: system
      metricsets: [filesystem, fsstat]
      filesystem.ignore_types: [nfs, smbfs, proc, cgroups]

Closes elastic#4685

(cherry picked from commit 70caf0e)

CHANGELOG.asciidoc

@@ -87,6 +87,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di
 - Add http server metricset to support push metrics via http. {pull}4770[4770]
 - Make config object public for graphite and http server {pull}4820[4820]
 - Add system uptime metricset. {issue}[4848[4848]
+- Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685]
 
 *Packetbeat*
 

metricbeat/metricbeat.reference.yml

@@ -52,6 +52,11 @@ metricbeat.modules:
   cpu.metrics:  ["percentages"]  # The other available options are normalized_percentages and ticks.
   core.metrics: ["percentages"]  # The other available option is ticks.
 
+  # A list of filesystem types to ignore. The filesystem metricset will not
+  # collect data from filesystems matching any of the specified types, and
+  # fsstats will not include data from these filesystems in its summary stats.
+  #filesystem.ignore_types: []
+
   # These options allow you to filter out all processes that are not
   # in the top N by CPU or memory, in order to reduce the number of documents created.
   # If both the `by_cpu` and `by_memory` options are used, the union of the two sets

metricbeat/module/system/_meta/config.reference.yml

@@ -20,6 +20,11 @@
   cpu.metrics:  ["percentages"]  # The other available options are normalized_percentages and ticks.
   core.metrics: ["percentages"]  # The other available option is ticks.
 
+  # A list of filesystem types to ignore. The filesystem metricset will not
+  # collect data from filesystems matching any of the specified types, and
+  # fsstats will not include data from these filesystems in its summary stats.
+  #filesystem.ignore_types: []
+
   # These options allow you to filter out all processes that are not
   # in the top N by CPU or memory, in order to reduce the number of documents created.
   # If both the `by_cpu` and `by_memory` options are used, the union of the two sets

metricbeat/module/system/filesystem/_meta/docs.asciidoc

@@ -11,15 +11,35 @@ This metricset is available on:
 - OpenBSD
 - Windows
 
+[float]
+=== Configuration
+
+*`filesystem.ignore_types`* - A list of filesystem types to ignore. Metrics will
+not be collected from filesystems matching these types. This setting also
+affects the `fsstats` metricset.
+
 [float]
 === Filtering
 
 Often there are mounted filesystems that you do not want Metricbeat to report
-metrics on. A simple strategy to deal with these filesystems is to configure a
-drop_event filter that matches the `mount_point` or `type` using a regular
-expression.
+metrics on. One option is to configure Metricbeat to ignore specific filesystem
+types. This can be accomplished by configuring `filesystem.ignore_types` with
+a list of filesystem types to ignore. In this example we are ignoring three
+types of filesystems.
+
+[source,yaml]
+----
+metricbeat.modules:
+  - module: system
+    period: 30s
+    metricsets: ["filesystem"]
+    filesystem.ignore_types: [nfs, smbfs, autofs]
+----
 
-Below is an example.
+Another strategy to deal with these filesystems is to configure a `drop_event`
+filter that matches the `mount_point` using a regular expression. This type of
+filtering occurs after the data has been collected so it can be less efficient
+than the previous method.
 
 [source,yaml]
 ----

metricbeat/module/system/filesystem/filesystem.go

@@ -22,12 +22,19 @@ func init() {
 // MetricSet for fetching filesystem metrics.
 type MetricSet struct {
 	mb.BaseMetricSet
+	config Config
 }
 
 // New creates and returns a new instance of MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	var config Config
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
 		BaseMetricSet: base,
+		config:        config,
 	}, nil
 }
 
@@ -39,6 +46,10 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 		return nil, errors.Wrap(err, "filesystem list")
 	}
 
+	if len(m.config.IgnoreTypes) > 0 {
+		fss = Filter(fss, BuildTypeFilter(m.config.IgnoreTypes...))
+	}
+
 	filesSystems := make([]common.MapStr, 0, len(fss))
 	for _, fs := range fss {
 		fsStat, err := GetFileSystemStat(fs)

metricbeat/module/system/filesystem/helper.go

@@ -13,6 +13,10 @@ import (
 	sigar "github.com/elastic/gosigar"
 )
 
+type Config struct {
+	IgnoreTypes []string `config:"filesystem.ignore_types"`
+}
+
 type FileSystemStat struct {
 	sigar.FileSystemUsage
 	DevName     string  `json:"device_name"`
@@ -91,3 +95,35 @@ func GetFilesystemEvent(fsStat *FileSystemStat) common.MapStr {
 		},
 	}
 }
+
+// Predicate is a function predicate for use with filesystems. It returns true
+// if the argument matches the predicate.
+type Predicate func(*sigar.FileSystem) bool
+
+// Filter returns a filtered list of filesystems. The in parameter
+// is used as the backing storage for the returned slice and is therefore
+// modified in this operation.
+func Filter(in []sigar.FileSystem, p Predicate) []sigar.FileSystem {
+	out := in[:0]
+	for _, fs := range in {
+		if p(&fs) {
+			out = append(out, fs)
+		}
+	}
+	return out
+}
+
+// BuildTypeFilter returns a predicate that returns false if the given
+// filesystem has a type that matches one of the ignoreType values.
+func BuildTypeFilter(ignoreType ...string) Predicate {
+	return func(fs *sigar.FileSystem) bool {
+		for _, fsType := range ignoreType {
+			// XXX (andrewkroh): SysTypeName appears to be used for non-Windows
+			// and TypeName is used exclusively for Windows.
+			if fs.SysTypeName == fsType || fs.TypeName == fsType {
+				return false
+			}
+		}
+		return true
+	}
+}

metricbeat/module/system/filesystem/helper_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	sigar "github.com/elastic/gosigar"
 )
 
 func TestFileSystemList(t *testing.T) {
@@ -44,3 +46,18 @@ func TestFileSystemList(t *testing.T) {
 		}
 	}
 }
+
+func TestFilter(t *testing.T) {
+	in := []sigar.FileSystem{
+		{SysTypeName: "nfs"},
+		{SysTypeName: "ext4"},
+		{SysTypeName: "proc"},
+		{SysTypeName: "smb"},
+	}
+
+	out := Filter(in, BuildTypeFilter("nfs", "smb", "proc"))
+
+	if assert.Len(t, out, 1) {
+		assert.Equal(t, "ext4", out[0].SysTypeName)
+	}
+}

metricbeat/module/system/fsstat/_meta/docs.asciidoc

@@ -9,3 +9,10 @@ This metricset is available on:
 - macOS
 - OpenBSD
 - Windows
+
+[float]
+=== Configuration
+
+*`filesystem.ignore_types`* - A list of filesystem types to ignore. Metrics will
+not be collected from filesystems matching these types. This setting also
+affects the `filesystem` metricset.

metricbeat/module/system/fsstat/fsstat.go

@@ -23,12 +23,19 @@ func init() {
 // MetricSet for fetching a summary of filesystem stats.
 type MetricSet struct {
 	mb.BaseMetricSet
+	config filesystem.Config
 }
 
 // New creates and returns a new instance of MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	var config filesystem.Config
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
 		BaseMetricSet: base,
+		config:        config,
 	}, nil
 }
 
@@ -40,6 +47,10 @@ func (m *MetricSet) Fetch() (common.MapStr, error) {
 		return nil, errors.Wrap(err, "filesystem list")
 	}
 
+	if len(m.config.IgnoreTypes) > 0 {
+		fss = filesystem.Filter(fss, filesystem.BuildTypeFilter(m.config.IgnoreTypes...))
+	}
+
 	// These values are optional and could also be calculated by Kibana
 	var totalFiles, totalSize, totalSizeFree, totalSizeUsed uint64
 	dict := map[string]bool{}