diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f567ab345750..9eb45c7cbf86 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -63,6 +63,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di *Metricbeat* +- Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685] + *Packetbeat* *Winlogbeat* diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4638e7475732..67e4331deec9 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -51,6 +51,11 @@ metricbeat.modules: cpu.metrics: ["percentages"] # The other available options are normalized_percentages and ticks. core.metrics: ["percentages"] # The other available option is ticks. + # A list of filesystem types to ignore. The filesystem metricset will not + # collect data from filesystems matching any of the specified types, and + # fsstats will not include data from these filesystems in its summary stats. + #filesystem.ignore_types: [] + # These options allow you to filter out all processes that are not # in the top N by CPU or memory, in order to reduce the number of documents created. # If both the `by_cpu` and `by_memory` options are used, the union of the two sets diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index 93928f7c1858..8505d01a204f 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -19,6 +19,11 @@ cpu.metrics: ["percentages"] # The other available options are normalized_percentages and ticks. core.metrics: ["percentages"] # The other available option is ticks. + # A list of filesystem types to ignore. The filesystem metricset will not + # collect data from filesystems matching any of the specified types, and + # fsstats will not include data from these filesystems in its summary stats. + #filesystem.ignore_types: [] + # These options allow you to filter out all processes that are not # in the top N by CPU or memory, in order to reduce the number of documents created. # If both the `by_cpu` and `by_memory` options are used, the union of the two sets diff --git a/metricbeat/module/system/filesystem/_meta/docs.asciidoc b/metricbeat/module/system/filesystem/_meta/docs.asciidoc index ee216a01ffde..a66af4263889 100644 --- a/metricbeat/module/system/filesystem/_meta/docs.asciidoc +++ b/metricbeat/module/system/filesystem/_meta/docs.asciidoc @@ -11,13 +11,35 @@ This metricset is available on: - OpenBSD - Windows +[float] +=== Configuration + +*`filesystem.ignore_types`* - A list of filesystem types to ignore. Metrics will +not be collected from filesystems matching these types. This setting also +affects the `fsstats` metricset. + [float] === Filtering Often there are mounted filesystems that you do not want Metricbeat to report -metrics on. A simple strategy to deal with these filesystems is to configure a -drop_event filter that matches the `mount_point` using a regular expression. -Below is an example. +metrics on. One option is to configure Metricbeat to ignore specific filesystem +types. This can be accomplished by configuring `filesystem.ignore_types` with +a list of filesystem types to ignore. In this example we are ignoring three +types of filesystems. + +[source,yaml] +---- +metricbeat.modules: + - module: system + period: 30s + metricsets: ["filesystem"] + filesystem.ignore_types: [nfs, smbfs, autofs] +---- + +Another strategy to deal with these filesystems is to configure a `drop_event` +filter that matches the `mount_point` using a regular expression. This type of +filtering occurs after the data has been collected so it can be less efficient +than the previous method. [source,yaml] ---- diff --git a/metricbeat/module/system/filesystem/filesystem.go b/metricbeat/module/system/filesystem/filesystem.go index 24426db8bb12..7bf133be8ab6 100644 --- a/metricbeat/module/system/filesystem/filesystem.go +++ b/metricbeat/module/system/filesystem/filesystem.go @@ -22,12 +22,19 @@ func init() { // MetricSet for fetching filesystem metrics. type MetricSet struct { mb.BaseMetricSet + config Config } // New creates and returns a new instance of MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + var config Config + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + return &MetricSet{ BaseMetricSet: base, + config: config, }, nil } @@ -39,6 +46,10 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) { return nil, errors.Wrap(err, "filesystem list") } + if len(m.config.IgnoreTypes) > 0 { + fss = Filter(fss, BuildTypeFilter(m.config.IgnoreTypes...)) + } + filesSystems := make([]common.MapStr, 0, len(fss)) for _, fs := range fss { fsStat, err := GetFileSystemStat(fs) diff --git a/metricbeat/module/system/filesystem/helper.go b/metricbeat/module/system/filesystem/helper.go index c7f6159a27b3..2ad83d931c0c 100644 --- a/metricbeat/module/system/filesystem/helper.go +++ b/metricbeat/module/system/filesystem/helper.go @@ -11,6 +11,10 @@ import ( sigar "github.com/elastic/gosigar" ) +type Config struct { + IgnoreTypes []string `config:"filesystem.ignore_types"` +} + type FileSystemStat struct { sigar.FileSystemUsage DevName string `json:"device_name"` @@ -79,3 +83,35 @@ func GetFilesystemEvent(fsStat *FileSystemStat) common.MapStr { }, } } + +// Predicate is a function predicate for use with filesystems. It returns true +// if the argument matches the predicate. +type Predicate func(*sigar.FileSystem) bool + +// Filter returns a filtered list of filesystems. The in parameter +// is used as the backing storage for the returned slice and is therefore +// modified in this operation. +func Filter(in []sigar.FileSystem, p Predicate) []sigar.FileSystem { + out := in[:0] + for _, fs := range in { + if p(&fs) { + out = append(out, fs) + } + } + return out +} + +// BuildTypeFilter returns a predicate that returns false if the given +// filesystem has a type that matches one of the ignoreType values. +func BuildTypeFilter(ignoreType ...string) Predicate { + return func(fs *sigar.FileSystem) bool { + for _, fsType := range ignoreType { + // XXX (andrewkroh): SysTypeName appears to be used for non-Windows + // and TypeName is used exclusively for Windows. + if fs.SysTypeName == fsType || fs.TypeName == fsType { + return false + } + } + return true + } +} diff --git a/metricbeat/module/system/filesystem/helper_test.go b/metricbeat/module/system/filesystem/helper_test.go index 8c5e37228b26..084974a198e2 100644 --- a/metricbeat/module/system/filesystem/helper_test.go +++ b/metricbeat/module/system/filesystem/helper_test.go @@ -9,6 +9,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + sigar "github.com/elastic/gosigar" ) func TestFileSystemList(t *testing.T) { @@ -40,3 +42,18 @@ func TestFileSystemList(t *testing.T) { } } } + +func TestFilter(t *testing.T) { + in := []sigar.FileSystem{ + {SysTypeName: "nfs"}, + {SysTypeName: "ext4"}, + {SysTypeName: "proc"}, + {SysTypeName: "smb"}, + } + + out := Filter(in, BuildTypeFilter("nfs", "smb", "proc")) + + if assert.Len(t, out, 1) { + assert.Equal(t, "ext4", out[0].SysTypeName) + } +} diff --git a/metricbeat/module/system/fsstat/_meta/docs.asciidoc b/metricbeat/module/system/fsstat/_meta/docs.asciidoc index 16cbe772f052..a595521dcf6c 100644 --- a/metricbeat/module/system/fsstat/_meta/docs.asciidoc +++ b/metricbeat/module/system/fsstat/_meta/docs.asciidoc @@ -9,3 +9,10 @@ This metricset is available on: - Linux - OpenBSD - Windows + +[float] +=== Configuration + +*`filesystem.ignore_types`* - A list of filesystem types to ignore. Metrics will +not be collected from filesystems matching these types. This setting also +affects the `filesystem` metricset. diff --git a/metricbeat/module/system/fsstat/fsstat.go b/metricbeat/module/system/fsstat/fsstat.go index 21eb4870e150..aec4069f3f59 100644 --- a/metricbeat/module/system/fsstat/fsstat.go +++ b/metricbeat/module/system/fsstat/fsstat.go @@ -23,12 +23,19 @@ func init() { // MetricSet for fetching a summary of filesystem stats. type MetricSet struct { mb.BaseMetricSet + config filesystem.Config } // New creates and returns a new instance of MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + var config filesystem.Config + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + return &MetricSet{ BaseMetricSet: base, + config: config, }, nil } @@ -40,6 +47,10 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { return nil, errors.Wrap(err, "filesystem list") } + if len(m.config.IgnoreTypes) > 0 { + fss = filesystem.Filter(fss, filesystem.BuildTypeFilter(m.config.IgnoreTypes...)) + } + // These values are optional and could also be calculated by Kibana var totalFiles, totalSize, totalSizeFree, totalSizeUsed uint64 dict := map[string]bool{}