From 8f4f51f7157722de716666ea80bb83af17d1c864 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Tue, 16 May 2017 15:39:01 +0200 Subject: [PATCH] Change Metricbeat default config to reduce disk space This implements several changes from #4112: * reduce the frequency for the fsstats and filesystem * filter out some filesystems by mount point * set one shard by default * enable best_compression by default --- CHANGELOG.asciidoc | 1 + metricbeat/Makefile | 1 + metricbeat/_meta/setup.yml | 5 +++ metricbeat/docs/modules/system.asciidoc | 37 ++++++-------------- metricbeat/metricbeat.yml | 42 +++++++++-------------- metricbeat/module/system/_meta/config.yml | 37 ++++++-------------- setup.yml | 1 + 7 files changed, 46 insertions(+), 78 deletions(-) create mode 100644 metricbeat/_meta/setup.yml create mode 100644 setup.yml diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7077afc1ce3..4ae2ca531d7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -73,6 +73,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d - Add `kube-state-metrics` based metrics to `kubernetes` module {pull}4253[4253] - Add debug logging to Jolokia JMX metricset. {pull}4341[4341] - Add events metricset for kubernetes metricbeat module {pull}4315[4315] +- Change Metricbeat default configuration file to be better optimized for most users. {pull}4329[4329] *Packetbeat* diff --git a/metricbeat/Makefile b/metricbeat/Makefile index f9d8fcc81a8..7c405e6c6c3 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -53,6 +53,7 @@ collect-docs: python-env configs: python-env cat ${ES_BEATS}/metricbeat/_meta/common.yml > _meta/beat.yml . ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml + cat ${ES_BEATS}/metricbeat/_meta/setup.yml >> _meta/beat.yml cat ${ES_BEATS}/metricbeat/_meta/common.full.yml > _meta/beat.full.yml . ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.full.yml diff --git a/metricbeat/_meta/setup.yml b/metricbeat/_meta/setup.yml new file mode 100644 index 00000000000..c18143a7cd4 --- /dev/null +++ b/metricbeat/_meta/setup.yml @@ -0,0 +1,5 @@ +#==================== Elasticsearch template setting ========================== +setup.template.settings: + index.number_of_shards: 1 + index.codec: best_compression + #_source.enabled: false diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index 51fde9ce689..f54c32cf54b 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -86,45 +86,30 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: system + enabled: true + period: 10s metricsets: - # CPU stats - cpu - - # System Load stats - load - - # Per CPU core stats + - memory #- core - - # IO stats #- diskio - - # Per filesystem stats - - filesystem - - # File system summary stats - - fsstat - - # Memory stats - - memory - - # Network stats - network - - # Processes summary - process_summary - - # Per process stats - process - - # Sockets (linux only) #- socket - enabled: true - period: 10s processes: ['.*'] process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory +- module: system + enabled: true + period: 1m + metricsets: + - filesystem + - fsstat + filters: + - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' ---- [float] diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index a5b01343689..f5ac172f1f5 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -12,47 +12,37 @@ metricbeat.modules: #------------------------------- System Module ------------------------------- - module: system + enabled: true + period: 10s metricsets: - # CPU stats - cpu - - # System Load stats - load - - # Per CPU core stats + - memory #- core - - # IO stats #- diskio - - # Per filesystem stats - - filesystem - - # File system summary stats - - fsstat - - # Memory stats - - memory - - # Network stats - network - - # Processes summary - process_summary - - # Per process stats - process - - # Sockets (linux only) #- socket - enabled: true - period: 10s processes: ['.*'] process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory +- module: system + enabled: true + period: 1m + metricsets: + - filesystem + - fsstat + filters: + - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' +#==================== Elasticsearch template setting ========================== +setup.template.settings: + index.number_of_shards: 1 + index.codec: best_compression + #_source.enabled: false #================================ General ===================================== diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index d2db45f7014..c1c6a5ab5a9 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -1,40 +1,25 @@ - module: system + enabled: true + period: 10s metricsets: - # CPU stats - cpu - - # System Load stats - load - - # Per CPU core stats + - memory #- core - - # IO stats #- diskio - - # Per filesystem stats - - filesystem - - # File system summary stats - - fsstat - - # Memory stats - - memory - - # Network stats - network - - # Processes summary - process_summary - - # Per process stats - process - - # Sockets (linux only) #- socket - enabled: true - period: 10s processes: ['.*'] process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory +- module: system + enabled: true + period: 1m + metricsets: + - filesystem + - fsstat + filters: + - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' diff --git a/setup.yml b/setup.yml new file mode 100644 index 00000000000..168251f0630 --- /dev/null +++ b/setup.yml @@ -0,0 +1 @@ +#======================== Template options ===============================