From 1b778efe71d0b175d1b7ba35900242dbbad1f533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 22 Mar 2017 11:49:09 +0100 Subject: [PATCH] Remove `_all` field from template for ES >= 6.0 (#3778) ES 6.0 deprecates `_all`, see https://github.com/elastic/elasticsearch/pull/22144. `*.template-es6x.json` has been added in case Elasticsearch 6.X is detected --- CHANGELOG.asciidoc | 1 + dev-tools/packer/platforms/binary/run.sh.j2 | 1 + dev-tools/packer/platforms/centos/run.sh.j2 | 1 + dev-tools/packer/platforms/darwin/run.sh.j2 | 1 + dev-tools/packer/platforms/debian/run.sh.j2 | 1 + dev-tools/packer/platforms/windows/run.sh.j2 | 1 + dev-tools/packer/xgo-scripts/before_build.sh | 1 + filebeat/filebeat.full.yml | 8 + filebeat/filebeat.template-es6x.json | 594 +++ heartbeat/heartbeat.full.yml | 8 + heartbeat/heartbeat.template-es6x.json | 192 + libbeat/_meta/config.full.yml | 8 + libbeat/docs/outputconfig.asciidoc | 12 +- libbeat/libbeat.template-es6x.json | 87 + .../elasticsearch/client_integration_test.go | 7 + libbeat/outputs/elasticsearch/config.go | 8 +- libbeat/outputs/elasticsearch/output.go | 19 + libbeat/scripts/Makefile | 3 +- libbeat/scripts/generate_template.py | 18 +- libbeat/tests/system/test_base.py | 2 + metricbeat/metricbeat.full.yml | 8 + metricbeat/metricbeat.template-es6x.json | 4132 +++++++++++++++++ packetbeat/packetbeat.full.yml | 8 + packetbeat/packetbeat.template-es6x.json | 1397 ++++++ winlogbeat/winlogbeat.full.yml | 8 + winlogbeat/winlogbeat.template-es6x.json | 185 + 26 files changed, 6698 insertions(+), 13 deletions(-) create mode 100644 filebeat/filebeat.template-es6x.json create mode 100644 heartbeat/heartbeat.template-es6x.json create mode 100644 libbeat/libbeat.template-es6x.json create mode 100644 metricbeat/metricbeat.template-es6x.json create mode 100644 packetbeat/packetbeat.template-es6x.json create mode 100644 winlogbeat/winlogbeat.template-es6x.json diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 036f7be965b..d880171982f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -64,6 +64,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Add the option to load the sample dashboards during the Beat startup phase. {pull}3506[3506] - Disabled date detection in Elasticsearch index templates. Date fields must be explicitly defined in index templates. {pull}3528[3528] - Using environment variables in the configuration file is now GA, instead of experimental. {pull}3525[3525] +- Update index mappings to support future Elasticsearch 6.X. {pull}3778[3778] *Filebeat* diff --git a/dev-tools/packer/platforms/binary/run.sh.j2 b/dev-tools/packer/platforms/binary/run.sh.j2 index 9ae01a102b6..df442222dc0 100644 --- a/dev-tools/packer/platforms/binary/run.sh.j2 +++ b/dev-tools/packer/platforms/binary/run.sh.j2 @@ -18,6 +18,7 @@ cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.be cp {{.beat_name}}-linux.full.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.full.yml cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ +cp {{.beat_name}}.template-es6x.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ mkdir -p upload tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}} diff --git a/dev-tools/packer/platforms/centos/run.sh.j2 b/dev-tools/packer/platforms/centos/run.sh.j2 index d5df40c6e11..cf6cc589629 100644 --- a/dev-tools/packer/platforms/centos/run.sh.j2 +++ b/dev-tools/packer/platforms/centos/run.sh.j2 @@ -38,6 +38,7 @@ fpm --force -s dir -t rpm \ {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \ {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \ {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \ + {{.beat_name}}.template-es6x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es6x.json \ ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \ god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \ import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards diff --git a/dev-tools/packer/platforms/darwin/run.sh.j2 b/dev-tools/packer/platforms/darwin/run.sh.j2 index f246aeecafd..61f0924c0f9 100644 --- a/dev-tools/packer/platforms/darwin/run.sh.j2 +++ b/dev-tools/packer/platforms/darwin/run.sh.j2 @@ -18,6 +18,7 @@ cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_na cp {{.beat_name}}-darwin.full.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.full.yml cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-darwin-x86_64/ cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-darwin-x86_64/ +cp {{.beat_name}}.template-es6x.json /{{.beat_name}}-${VERSION}-darwin-x86_64/ mkdir -p upload tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64 diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2 index 95ff01792be..ef2e2825215 100644 --- a/dev-tools/packer/platforms/debian/run.sh.j2 +++ b/dev-tools/packer/platforms/debian/run.sh.j2 @@ -35,6 +35,7 @@ fpm --force -s dir -t deb \ {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \ {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \ {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \ + {{.beat_name}}.template-es6x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es6x.json \ ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \ god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \ import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards diff --git a/dev-tools/packer/platforms/windows/run.sh.j2 b/dev-tools/packer/platforms/windows/run.sh.j2 index d35643cd1d3..a2f392f13b1 100644 --- a/dev-tools/packer/platforms/windows/run.sh.j2 +++ b/dev-tools/packer/platforms/windows/run.sh.j2 @@ -19,6 +19,7 @@ cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.be cp {{.beat_name}}-win.full.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.full.yml cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ +cp {{.beat_name}}.template-es6x.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ diff --git a/dev-tools/packer/xgo-scripts/before_build.sh b/dev-tools/packer/xgo-scripts/before_build.sh index 9ebcaaede3e..b9c3787f0b2 100755 --- a/dev-tools/packer/xgo-scripts/before_build.sh +++ b/dev-tools/packer/xgo-scripts/before_build.sh @@ -15,6 +15,7 @@ PREFIX=/build # Copy template cp $BEAT_NAME.template.json $PREFIX/$BEAT_NAME.template.json cp $BEAT_NAME.template-es2x.json $PREFIX/$BEAT_NAME.template-es2x.json +cp $BEAT_NAME.template-es6x.json $PREFIX/$BEAT_NAME.template-es6x.json # linux cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index fd68d4590da..281d362b1ca 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -510,6 +510,14 @@ output.elasticsearch: # Path to the Elasticsearch 2.x version of the template file. #template.versions.2x.path: "${path.config}/filebeat.template-es2x.json" + # If set to true, filebeat checks the Elasticsearch version at connect time, and if it + # is 6.x, it loads the file specified by the template.versions.6x.path setting. The + # default is true. + #template.versions.6x.enabled: true + + # Path to the Elasticsearch 6.x version of the template file. + #template.versions.6x.path: "${path.config}/filebeat.template-es6x.json" + # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/filebeat/filebeat.template-es6x.json b/filebeat/filebeat.template-es6x.json new file mode 100644 index 00000000000..6db516fba7c --- /dev/null +++ b/filebeat/filebeat.template-es6x.json @@ -0,0 +1,594 @@ +{ + "mappings": { + "_default_": { + "_meta": { + "version": "5.4.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "apache2": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "type": "long" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "fields": { + "properties": {} + }, + "fileset": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "input_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "type": "long" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "offset": { + "type": "long" + }, + "read_timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "properties": { + "gid": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "event": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "useradd": { + "properties": { + "gid": { + "type": "long" + }, + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "type": "long" + } + } + } + } + }, + "syslog": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + }, + "template": "filebeat-*" +} \ No newline at end of file diff --git a/heartbeat/heartbeat.full.yml b/heartbeat/heartbeat.full.yml index 3155f7aa015..f0a1d502b00 100644 --- a/heartbeat/heartbeat.full.yml +++ b/heartbeat/heartbeat.full.yml @@ -358,6 +358,14 @@ output.elasticsearch: # Path to the Elasticsearch 2.x version of the template file. #template.versions.2x.path: "${path.config}/heartbeat.template-es2x.json" + # If set to true, heartbeat checks the Elasticsearch version at connect time, and if it + # is 6.x, it loads the file specified by the template.versions.6x.path setting. The + # default is true. + #template.versions.6x.enabled: true + + # Path to the Elasticsearch 6.x version of the template file. + #template.versions.6x.path: "${path.config}/heartbeat.template-es6x.json" + # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/heartbeat/heartbeat.template-es6x.json b/heartbeat/heartbeat.template-es6x.json new file mode 100644 index 00000000000..d1cb1845a35 --- /dev/null +++ b/heartbeat/heartbeat.template-es6x.json @@ -0,0 +1,192 @@ +{ + "mappings": { + "_default_": { + "_meta": { + "version": "5.4.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "error": { + "properties": { + "message": { + "norms": false, + "type": "text" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "properties": {} + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_rtt": { + "properties": { + "us": { + "type": "long" + } + } + }, + "icmp_rtt": { + "properties": { + "us": { + "type": "long" + } + } + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "monitor": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "resolve_rtt": { + "properties": { + "us": { + "type": "long" + } + } + }, + "response": { + "properties": { + "status": { + "type": "long" + } + } + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "socks5_connect_rtt": { + "properties": { + "us": { + "type": "long" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_connect_rtt": { + "properties": { + "us": { + "type": "long" + } + } + }, + "tls_handshake_rtt": { + "properties": { + "us": { + "type": "long" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "up": { + "type": "boolean" + }, + "url": { + "norms": false, + "type": "text" + }, + "validate_rtt": { + "properties": { + "us": { + "type": "long" + } + } + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + }, + "template": "heartbeat-*" +} \ No newline at end of file diff --git a/libbeat/_meta/config.full.yml b/libbeat/_meta/config.full.yml index 1a89c33cc87..2537364034f 100644 --- a/libbeat/_meta/config.full.yml +++ b/libbeat/_meta/config.full.yml @@ -160,6 +160,14 @@ output.elasticsearch: # Path to the Elasticsearch 2.x version of the template file. #template.versions.2x.path: "${path.config}/beatname.template-es2x.json" + # If set to true, beatname checks the Elasticsearch version at connect time, and if it + # is 6.x, it loads the file specified by the template.versions.6x.path setting. The + # default is true. + #template.versions.6x.enabled: true + + # Path to the Elasticsearch 6.x version of the template file. + #template.versions.6x.path: "${path.config}/beatname.template-es6x.json" + # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 713b462afc3..da8336a8316 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -283,6 +283,12 @@ Elasticsearch versions 2.x.y. The default is +{beatname_lc}.template-es2x.json+. *`2x.enabled`*:: If set to +false+, the +2x.path+ option is ignored and the default template is loaded regardless of the Elasticsearch version. +*`6x.path`*:: The path to the template file to load for +Elasticsearch versions 6.x.y. The default is +{beatname_lc}.template-es6x.json+. + +*`6x.enabled`*:: If set to +false+, the +6x.path+ option is ignored and the +default template is loaded regardless of the Elasticsearch version. + For example: ["source","yaml",subs="attributes,callouts"] @@ -430,11 +436,11 @@ The default value is true. [[hosts]] ===== hosts -The list of known Logstash servers to connect to. If load balancing is disabled, but -mutliple hosts are configured, one host is selected randomly (there is no precedence). +The list of known Logstash servers to connect to. If load balancing is disabled, but +mutliple hosts are configured, one host is selected randomly (there is no precedence). If one host becomes unreachable, another one is selected randomly. -All entries in this list can contain a port number. If no port number is given, the +All entries in this list can contain a port number. If no port number is given, the value specified for <> is used as the default port number. ===== compression_level diff --git a/libbeat/libbeat.template-es6x.json b/libbeat/libbeat.template-es6x.json new file mode 100644 index 00000000000..8afcfd0498a --- /dev/null +++ b/libbeat/libbeat.template-es6x.json @@ -0,0 +1,87 @@ +{ + "mappings": { + "_default_": { + "_meta": { + "version": "5.4.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "properties": {} + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + }, + "template": "libbeat-*" +} \ No newline at end of file diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index eedc4272906..db51c4051bb 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -48,6 +48,8 @@ func TestLoadTemplate(t *testing.T) { templatePath := absPath + "/libbeat.template.json" if strings.HasPrefix(client.Connection.version, "2.") { templatePath = absPath + "/libbeat.template-es2x.json" + } else if strings.HasPrefix(client.Connection.version, "6.") { + templatePath = absPath + "/libbeat.template-es6x.json" } content, err := readTemplate(templatePath) assert.Nil(t, err) @@ -116,6 +118,8 @@ func TestLoadBeatsTemplate(t *testing.T) { if strings.HasPrefix(client.Connection.version, "2.") { templatePath = absPath + "/" + beat + ".template-es2x.json" + } else if strings.HasPrefix(client.Connection.version, "6.") { + templatePath = absPath + "/" + beat + ".template-es6x.json" } content, err := readTemplate(templatePath) @@ -161,6 +165,8 @@ func TestOutputLoadTemplate(t *testing.T) { if strings.HasPrefix(client.Connection.version, "2.") { templatePath = "../../libbeat.template-es2x.json" + } else if strings.HasPrefix(client.Connection.version, "6.") { + templatePath = "../../libbeat.template-es6x.json" } tPath, err := filepath.Abs(templatePath) @@ -173,6 +179,7 @@ func TestOutputLoadTemplate(t *testing.T) { "name": "libbeat", "path": tPath, "versions.2x.enabled": false, + "versions.6x.enabled": false, }, } diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index c30f68702db..9fabadfc956 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -33,6 +33,7 @@ type Template struct { type TemplateVersions struct { Es2x TemplateVersion `config:"2x"` + Es6x TemplateVersion `config:"6x"` } type TemplateVersion struct { @@ -58,8 +59,11 @@ var ( TLS: nil, LoadBalance: true, Template: Template{ - Enabled: true, - Versions: TemplateVersions{Es2x: TemplateVersion{Enabled: true}}, + Enabled: true, + Versions: TemplateVersions{ + Es2x: TemplateVersion{Enabled: true}, + Es6x: TemplateVersion{Enabled: true}, + }, }, } ) diff --git a/libbeat/outputs/elasticsearch/output.go b/libbeat/outputs/elasticsearch/output.go index 20e7e1fe62a..6b312b0beda 100644 --- a/libbeat/outputs/elasticsearch/output.go +++ b/libbeat/outputs/elasticsearch/output.go @@ -31,6 +31,7 @@ type elasticsearchOutput struct { template map[string]interface{} template2x map[string]interface{} + template6x map[string]interface{} templateMutex sync.Mutex } @@ -248,6 +249,9 @@ func (out *elasticsearchOutput) readTemplate(config *Template) error { if config.Versions.Es2x.Path == "" { config.Versions.Es2x.Path = fmt.Sprintf("%s.template-es2x.json", out.beatName) } + if config.Versions.Es6x.Path == "" { + config.Versions.Es6x.Path = fmt.Sprintf("%s.template-es6x.json", out.beatName) + } // Look for the template in the configuration path, if it's not absolute templatePath := paths.Resolve(paths.Config, config.Path) @@ -270,6 +274,18 @@ func (out *elasticsearchOutput) readTemplate(config *Template) error { } out.template2x = template } + + if config.Versions.Es6x.Enabled { + // Read the version of the template compatible with ES 6.x + templatePath := paths.Resolve(paths.Config, config.Versions.Es6x.Path) + logp.Info("Loading template enabled for Elasticsearch 6.x. Reading template file: %v", templatePath) + + template, err := readTemplate(templatePath) + if err != nil { + return fmt.Errorf("Error loading template %s: %v", templatePath, err) + } + out.template6x = template + } } return nil } @@ -312,6 +328,9 @@ func (out *elasticsearchOutput) loadTemplate(config Template, client *Client) er if config.Versions.Es2x.Enabled && strings.HasPrefix(client.Connection.version, "2.") { logp.Info("Detected Elasticsearch 2.x. Automatically selecting the 2.x version of the template") template = out.template2x + } else if config.Versions.Es6x.Enabled && strings.HasPrefix(client.Connection.version, "6.") { + logp.Info("Detected Elasticsearch 6.x. Automatically selecting the 6.x version of the template") + template = out.template6x } err := client.LoadTemplate(config.Name, template) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 37024f74544..804c7cbe032 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -246,8 +246,9 @@ update: python-env collect . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS} # Generate index templates - . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py $(PWD) ${BEAT_NAME} ${ES_BEATS} . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py --es2x $(PWD) ${BEAT_NAME} ${ES_BEATS} + . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py --es6x $(PWD) ${BEAT_NAME} ${ES_BEATS} + . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py $(PWD) ${BEAT_NAME} ${ES_BEATS} # Generate index-pattern echo "Generate index pattern" diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py index 20b4b5fab4c..07701bdec39 100644 --- a/libbeat/scripts/generate_template.py +++ b/libbeat/scripts/generate_template.py @@ -67,17 +67,17 @@ def fields_to_es_template(args, input, output, index, version): } } else: - # For ES 5.x, increase the limit on the max number of fields. + # For ES >= 5.x, increase the limit on the max number of fields. # In a typical scenario, most fields are not used, so increasing the # limit shouldn't be that bad. template["settings"]["index.mapping.total_fields.limit"] = 10000 - # should be done only for es5x. For es6x, any "_all" setting results - # in an error. - # TODO: https://github.com/elastic/beats/issues/3368 - template["mappings"]["_default_"]["_all"] = { - "norms": False - } + if not args.es6x: + # should be done only for es5x. For es6x, any "_all" setting results + # in an error. + template["mappings"]["_default_"]["_all"] = { + "norms": False + } properties = {} dynamic_templates = [] @@ -344,6 +344,8 @@ def fill_field_properties(args, field, defaults, path): description="Generates the templates for a Beat.") parser.add_argument("--es2x", action="store_true", help="Generate template for Elasticsearch 2.x.") + parser.add_argument("--es6x", action="store_true", + help="Generate template for Elasticsearch 6.x.") parser.add_argument("path", help="Path to the beat folder") parser.add_argument("beatname", help="The beat fname") parser.add_argument("es_beats", help="The path to the general beats folder") @@ -353,6 +355,8 @@ def fill_field_properties(args, field, defaults, path): target = args.path + "/" + args.beatname + ".template" if args.es2x: target += "-es2x" + elif args.es6x: + target += "-es6x" target += ".json" fields_yml = args.path + "/_meta/fields.generated.yml" diff --git a/libbeat/tests/system/test_base.py b/libbeat/tests/system/test_base.py index f9f14c2f1f0..f6e90899016 100644 --- a/libbeat/tests/system/test_base.py +++ b/libbeat/tests/system/test_base.py @@ -73,6 +73,8 @@ def test_config_test(self): f.write('{"template": true}') with open(self.working_dir + "/mockbeat.template-es2x.json", "w") as f: f.write('{"template": true}') + with open(self.working_dir + "/mockbeat.template-es6x.json", "w") as f: + f.write('{"template": true}') exit_code = self.run_beat( config="libbeat.yml", diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 04649c091d6..f56c93f7cc0 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -454,6 +454,14 @@ output.elasticsearch: # Path to the Elasticsearch 2.x version of the template file. #template.versions.2x.path: "${path.config}/metricbeat.template-es2x.json" + # If set to true, metricbeat checks the Elasticsearch version at connect time, and if it + # is 6.x, it loads the file specified by the template.versions.6x.path setting. The + # default is true. + #template.versions.6x.enabled: true + + # Path to the Elasticsearch 6.x version of the template file. + #template.versions.6x.path: "${path.config}/metricbeat.template-es6x.json" + # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/metricbeat/metricbeat.template-es6x.json b/metricbeat/metricbeat.template-es6x.json new file mode 100644 index 00000000000..77372e154d8 --- /dev/null +++ b/metricbeat/metricbeat.template-es6x.json @@ -0,0 +1,4132 @@ +{ + "mappings": { + "_default_": { + "_meta": { + "version": "5.4.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + }, + { + "system.process.cgroup.cpuacct.percpu": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "system.process.cgroup.cpuacct.percpu.*" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "apache": { + "properties": { + "status": { + "properties": { + "bytes_per_request": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "bytes_per_sec": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "connections": { + "properties": { + "async": { + "properties": { + "closing": { + "type": "long" + }, + "keep_alive": { + "type": "long" + }, + "writing": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "cpu": { + "properties": { + "children_system": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "children_user": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "load": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "system": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "user": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "load": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + } + } + }, + "requests_per_sec": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "scoreboard": { + "properties": { + "closing_connection": { + "type": "long" + }, + "dns_lookup": { + "type": "long" + }, + "gracefully_finishing": { + "type": "long" + }, + "idle_cleanup": { + "type": "long" + }, + "keepalive": { + "type": "long" + }, + "logging": { + "type": "long" + }, + "open_slot": { + "type": "long" + }, + "reading_request": { + "type": "long" + }, + "sending_reply": { + "type": "long" + }, + "starting_up": { + "type": "long" + }, + "total": { + "type": "long" + }, + "waiting_for_connection": { + "type": "long" + } + } + }, + "total_accesses": { + "type": "long" + }, + "total_kbytes": { + "type": "long" + }, + "uptime": { + "properties": { + "server_uptime": { + "type": "long" + }, + "uptime": { + "type": "long" + } + } + }, + "workers": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + } + } + } + } + } + } + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ceph": { + "properties": { + "cluster_disk": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "cluster_health": { + "properties": { + "overall_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "timechecks": { + "properties": { + "epoch": { + "type": "long" + }, + "round": { + "properties": { + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "long" + } + } + } + } + } + } + }, + "monitor_health": { + "properties": { + "available": { + "properties": { + "kb": { + "type": "long" + }, + "pct": { + "type": "long" + } + } + }, + "health": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_updated": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "store_stats": { + "properties": { + "last_updated": { + "type": "long" + }, + "log": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "misc": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "sst": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "kb": { + "type": "long" + } + } + }, + "used": { + "properties": { + "kb": { + "type": "long" + } + } + } + } + }, + "pool_disk": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "objects": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "kb": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "couchbase": { + "properties": { + "bucket": { + "properties": { + "data": { + "properties": { + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "disk": { + "properties": { + "fetches": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "item_count": { + "type": "long" + }, + "memory": { + "properties": { + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ops_per_sec": { + "type": "long" + }, + "quota": { + "properties": { + "ram": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "use": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cluster": { + "properties": { + "hdd": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "by_data": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "max_bucket_count": { + "type": "long" + }, + "quota": { + "properties": { + "index_memory": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "ram": { + "properties": { + "quota": { + "properties": { + "total": { + "properties": { + "per_node": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "used": { + "properties": { + "per_node": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "by_data": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "node": { + "properties": { + "cmd_get": { + "type": "long" + }, + "couch": { + "properties": { + "docs": { + "properties": { + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "disk_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "spatial": { + "properties": { + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "disk_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "views": { + "properties": { + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "disk_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "cpu_utilization_rate": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "current_items": { + "properties": { + "total": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "ep_bg_fetched": { + "type": "long" + }, + "get_hits": { + "type": "long" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "mcd_memory": { + "properties": { + "allocated": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "reserved": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "ops": { + "type": "long" + }, + "swap": { + "properties": { + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "vb_replica_curr_items": { + "type": "long" + } + } + } + } + }, + "docker": { + "properties": { + "container": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "properties": {} + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "properties": { + "root_fs": { + "type": "long" + }, + "rw": { + "type": "long" + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "properties": {} + } + } + }, + "cpu": { + "properties": { + "kernel": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "total": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "user": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "diskio": { + "properties": { + "reads": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "writes": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "healthcheck": { + "properties": { + "event": { + "properties": { + "end_date": { + "type": "date" + }, + "exit_code": { + "type": "long" + }, + "output": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_date": { + "type": "date" + } + } + }, + "failingstreak": { + "type": "long" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "image": { + "properties": { + "created": { + "type": "date" + }, + "id": { + "properties": { + "current": { + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "properties": {} + }, + "size": { + "properties": { + "regular": { + "type": "long" + }, + "virtual": { + "type": "long" + } + } + }, + "tags": { + "properties": {} + } + } + }, + "info": { + "properties": { + "containers": { + "properties": { + "paused": { + "type": "long" + }, + "running": { + "type": "long" + }, + "stopped": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "images": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "fail": { + "properties": { + "count": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "limit": { + "type": "long" + }, + "rss": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "max": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + } + } + }, + "fields": { + "properties": {} + }, + "haproxy": { + "properties": { + "info": { + "properties": { + "compress": { + "properties": { + "bps": { + "properties": { + "in": { + "type": "long" + }, + "out": { + "type": "long" + }, + "rate_limit": { + "type": "long" + } + } + } + } + }, + "connection": { + "properties": { + "current": { + "type": "long" + }, + "hard_max": { + "type": "long" + }, + "max": { + "type": "long" + }, + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "ssl": { + "properties": { + "current": { + "type": "long" + }, + "max": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "memory": { + "properties": { + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "pid": { + "type": "long" + }, + "pipes": { + "properties": { + "free": { + "type": "long" + }, + "max": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "process_num": { + "type": "long" + }, + "processes": { + "type": "long" + }, + "requests": { + "properties": { + "max": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "run_queue": { + "type": "long" + }, + "session": { + "properties": { + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "sockets": { + "properties": { + "max": { + "type": "long" + } + } + }, + "ssl": { + "properties": { + "backend": { + "properties": { + "key_rate": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "cache_misses": { + "type": "long" + }, + "cached_lookups": { + "type": "long" + }, + "frontend": { + "properties": { + "key_rate": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "session_reuse": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "tasks": { + "type": "long" + }, + "ulimit_n": { + "type": "long" + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "zlib_mem_usage": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "stat": { + "properties": { + "check": { + "properties": { + "agent": { + "properties": { + "last": { + "type": "long" + } + } + }, + "code": { + "type": "long" + }, + "down": { + "type": "long" + }, + "duration": { + "type": "long" + }, + "failed": { + "type": "long" + }, + "health": { + "properties": { + "fail": { + "type": "long" + }, + "last": { + "type": "long" + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "aborted": { + "type": "long" + } + } + }, + "component_type": { + "type": "long" + }, + "compressor": { + "properties": { + "bypassed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "response": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "connection": { + "properties": { + "retried": { + "type": "long" + }, + "time": { + "properties": { + "avg": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "downtime": { + "type": "long" + }, + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "last_change": { + "type": "long" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "process_id": { + "type": "long" + }, + "proxy": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "queue": { + "properties": { + "limit": { + "type": "long" + }, + "time": { + "properties": { + "avg": { + "type": "long" + } + } + } + } + }, + "request": { + "properties": { + "connection": { + "properties": { + "errors": { + "type": "long" + } + } + }, + "denied": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "queued": { + "properties": { + "current": { + "type": "long" + }, + "max": { + "type": "long" + } + } + }, + "rate": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "redispatched": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "response": { + "properties": { + "denied": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "http": { + "properties": { + "1xx": { + "type": "long" + }, + "2xx": { + "type": "long" + }, + "3xx": { + "type": "long" + }, + "4xx": { + "type": "long" + }, + "5xx": { + "type": "long" + }, + "other": { + "type": "long" + } + } + }, + "time": { + "properties": { + "avg": { + "type": "long" + } + } + } + } + }, + "selected": { + "properties": { + "total": { + "type": "long" + } + } + }, + "server": { + "properties": { + "aborted": { + "type": "long" + }, + "active": { + "type": "long" + }, + "backup": { + "type": "long" + }, + "id": { + "type": "long" + } + } + }, + "service_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "session": { + "properties": { + "current": { + "type": "long" + }, + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "throttle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "tracked": { + "properties": { + "id": { + "type": "long" + } + } + }, + "weight": { + "type": "long" + } + } + } + } + }, + "kafka": { + "properties": { + "consumergroup": { + "properties": { + "broker": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + } + } + }, + "client": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "member_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "meta": { + "norms": false, + "type": "text" + }, + "offset": { + "type": "long" + }, + "partition": { + "type": "long" + }, + "topic": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "partition": { + "properties": { + "broker": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + } + } + }, + "offset": { + "properties": { + "newest": { + "type": "long" + }, + "oldest": { + "type": "long" + } + } + }, + "partition": { + "properties": { + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "insync_replica": { + "type": "boolean" + }, + "isr": { + "properties": {} + }, + "leader": { + "type": "long" + }, + "replica": { + "type": "long" + } + } + }, + "topic": { + "properties": { + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "metricset": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "rtt": { + "type": "long" + } + } + }, + "mongodb": { + "properties": { + "dbstats": { + "properties": { + "avg_obj_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "collections": { + "type": "long" + }, + "data_file_version": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + } + } + }, + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "db": { + "ignore_above": 1024, + "type": "keyword" + }, + "extent_free_list": { + "properties": { + "num": { + "type": "long" + }, + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "file_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "index_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "indexes": { + "type": "long" + }, + "ns_size_mb": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "num_extents": { + "type": "long" + }, + "objects": { + "type": "long" + }, + "storage_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "status": { + "properties": { + "asserts": { + "properties": { + "msg": { + "type": "long" + }, + "regular": { + "type": "long" + }, + "rollovers": { + "type": "long" + }, + "user": { + "type": "long" + }, + "warning": { + "type": "long" + } + } + }, + "background_flushing": { + "properties": { + "average": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "flushes": { + "type": "long" + }, + "last": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "last_finished": { + "type": "date" + }, + "total": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "connections": { + "properties": { + "available": { + "type": "long" + }, + "current": { + "type": "long" + }, + "total_created": { + "type": "long" + } + } + }, + "extra_info": { + "properties": { + "heap_usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + } + } + }, + "journaling": { + "properties": { + "commits": { + "type": "long" + }, + "commits_in_write_lock": { + "type": "long" + }, + "compression": { + "type": "long" + }, + "early_commits": { + "type": "long" + }, + "journaled": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "times": { + "properties": { + "commits": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "commits_in_write_lock": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "dt": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "prep_log_buffer": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "remap_private_view": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_data_files": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_journal": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "write_to_data_files": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "local_time": { + "type": "date" + }, + "memory": { + "properties": { + "bits": { + "type": "long" + }, + "mapped": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "mapped_with_journal": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "resident": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "requests": { + "type": "long" + } + } + }, + "opcounters": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "opcounters_replicated": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "storage_engine": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "wired_tiger": { + "properties": { + "cache": { + "properties": { + "dirty": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "maximum": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "pages": { + "properties": { + "evicted": { + "type": "long" + }, + "read": { + "type": "long" + }, + "write": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "concurrent_transactions": { + "properties": { + "read": { + "properties": { + "available": { + "type": "long" + }, + "out": { + "type": "long" + }, + "total_tickets": { + "type": "long" + } + } + }, + "write": { + "properties": { + "available": { + "type": "long" + }, + "out": { + "type": "long" + }, + "total_tickets": { + "type": "long" + } + } + } + } + }, + "log": { + "properties": { + "flushes": { + "type": "long" + }, + "max_file_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "scans": { + "type": "long" + }, + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "syncs": { + "type": "long" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "writes": { + "type": "long" + } + } + } + } + }, + "write_backs_queued": { + "type": "boolean" + } + } + } + } + }, + "mysql": { + "properties": { + "status": { + "properties": { + "aborted": { + "properties": { + "clients": { + "type": "long" + }, + "connects": { + "type": "long" + } + } + }, + "binlog": { + "properties": { + "cache": { + "properties": { + "disk_use": { + "type": "long" + }, + "use": { + "type": "long" + } + } + } + } + }, + "bytes": { + "properties": { + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "command": { + "properties": { + "delete": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "select": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "connections": { + "type": "long" + }, + "created": { + "properties": { + "tmp": { + "properties": { + "disk_tables": { + "type": "long" + }, + "files": { + "type": "long" + }, + "tables": { + "type": "long" + } + } + } + } + }, + "delayed": { + "properties": { + "errors": { + "type": "long" + }, + "insert_threads": { + "type": "long" + }, + "writes": { + "type": "long" + } + } + }, + "flush_commands": { + "type": "long" + }, + "max_used_connections": { + "type": "long" + }, + "open": { + "properties": { + "files": { + "type": "long" + }, + "streams": { + "type": "long" + }, + "tables": { + "type": "long" + } + } + }, + "opened_tables": { + "type": "long" + }, + "threads": { + "properties": { + "cached": { + "type": "long" + }, + "connected": { + "type": "long" + }, + "created": { + "type": "long" + }, + "running": { + "type": "long" + } + } + } + } + } + } + }, + "nginx": { + "properties": { + "stubstatus": { + "properties": { + "accepts": { + "type": "long" + }, + "active": { + "type": "long" + }, + "current": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "handled": { + "type": "long" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "reading": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "waiting": { + "type": "long" + }, + "writing": { + "type": "long" + } + } + } + } + }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "connections": { + "properties": { + "accepted": { + "type": "long" + }, + "queued": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "processes": { + "properties": { + "active": { + "type": "long" + }, + "idle": { + "type": "long" + } + } + }, + "slow_requests": { + "type": "long" + } + } + } + } + }, + "postgresql": { + "properties": { + "activity": { + "properties": { + "application_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "backend_start": { + "type": "date" + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + } + } + }, + "database": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "oid": { + "type": "long" + } + } + }, + "pid": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_start": { + "type": "date" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_change": { + "type": "date" + }, + "transaction_start": { + "type": "date" + }, + "user": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "waiting": { + "type": "boolean" + } + } + }, + "bgwriter": { + "properties": { + "buffers": { + "properties": { + "allocated": { + "type": "long" + }, + "backend": { + "type": "long" + }, + "backend_fsync": { + "type": "long" + }, + "checkpoints": { + "type": "long" + }, + "clean": { + "type": "long" + }, + "clean_full": { + "type": "long" + } + } + }, + "checkpoints": { + "properties": { + "requested": { + "type": "long" + }, + "scheduled": { + "type": "long" + }, + "times": { + "properties": { + "sync": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "write": { + "properties": { + "ms": { + "type": "float" + } + } + } + } + } + } + }, + "stats_reset": { + "type": "date" + } + } + }, + "database": { + "properties": { + "blocks": { + "properties": { + "hit": { + "type": "long" + }, + "read": { + "type": "long" + }, + "time": { + "properties": { + "read": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "conflicts": { + "type": "long" + }, + "deadlocks": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "number_of_backends": { + "type": "long" + }, + "oid": { + "type": "long" + }, + "rows": { + "properties": { + "deleted": { + "type": "long" + }, + "fetched": { + "type": "long" + }, + "inserted": { + "type": "long" + }, + "returned": { + "type": "long" + }, + "updated": { + "type": "long" + } + } + }, + "stats_reset": { + "type": "date" + }, + "temporary": { + "properties": { + "bytes": { + "type": "long" + }, + "files": { + "type": "long" + } + } + }, + "transactions": { + "properties": { + "commit": { + "type": "long" + }, + "rollback": { + "type": "long" + } + } + } + } + } + } + }, + "prometheus": { + "properties": { + "stats": { + "properties": { + "notifications": { + "properties": { + "dropped": { + "type": "long" + }, + "queue_length": { + "type": "long" + } + } + }, + "processes": { + "properties": { + "open_fds": { + "type": "long" + } + } + }, + "storage": { + "properties": { + "chunks_to_persist": { + "type": "long" + } + } + } + } + } + } + }, + "redis": { + "properties": { + "info": { + "properties": { + "clients": { + "properties": { + "biggest_input_buf": { + "type": "long" + }, + "blocked": { + "type": "long" + }, + "connected": { + "type": "long" + }, + "longest_output_list": { + "type": "long" + } + } + }, + "cluster": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "cpu": { + "properties": { + "used": { + "properties": { + "sys": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "sys_children": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "user": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "user_children": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "memory": { + "properties": { + "allocator": { + "ignore_above": 1024, + "type": "keyword" + }, + "used": { + "properties": { + "lua": { + "type": "long" + }, + "peak": { + "type": "long" + }, + "rss": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "persistence": { + "properties": { + "aof": { + "properties": { + "bgrewrite": { + "properties": { + "last_status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "enabled": { + "type": "boolean" + }, + "rewrite": { + "properties": { + "current_time": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "in_progress": { + "type": "boolean" + }, + "last_time": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "scheduled": { + "type": "boolean" + } + } + }, + "write": { + "properties": { + "last_status": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "loading": { + "type": "boolean" + }, + "rdb": { + "properties": { + "bgsave": { + "properties": { + "current_time": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "in_progress": { + "type": "boolean" + }, + "last_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_time": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "last_save": { + "properties": { + "changes_since": { + "type": "long" + }, + "time": { + "type": "long" + } + } + } + } + } + } + }, + "replication": { + "properties": { + "backlog": { + "properties": { + "active": { + "type": "long" + }, + "first_byte_offset": { + "type": "long" + }, + "histlen": { + "type": "long" + }, + "size": { + "type": "long" + } + } + }, + "connected_slaves": { + "type": "long" + }, + "master_offset": { + "type": "long" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "arch_bits": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "config_file": { + "ignore_above": 1024, + "type": "keyword" + }, + "gcc_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "git_dirty": { + "ignore_above": 1024, + "type": "keyword" + }, + "git_sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "hz": { + "type": "long" + }, + "lru_clock": { + "type": "long" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "multiplexing_api": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "process_id": { + "type": "long" + }, + "run_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_port": { + "type": "long" + }, + "uptime": { + "type": "long" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "stats": { + "properties": { + "commands_processed": { + "type": "long" + }, + "connections": { + "properties": { + "received": { + "type": "long" + }, + "rejected": { + "type": "long" + } + } + }, + "instantaneous": { + "properties": { + "input_kbps": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ops_per_sec": { + "type": "long" + }, + "output_kbps": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "keys": { + "properties": { + "evicted": { + "type": "long" + }, + "expired": { + "type": "long" + } + } + }, + "keyspace": { + "properties": { + "hits": { + "type": "long" + }, + "misses": { + "type": "long" + } + } + }, + "latest_fork_usec": { + "type": "long" + }, + "migrate_cached_sockets": { + "type": "long" + }, + "net": { + "properties": { + "input": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "output": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "pubsub": { + "properties": { + "channels": { + "type": "long" + }, + "patterns": { + "type": "long" + } + } + }, + "sync": { + "properties": { + "full": { + "type": "long" + }, + "partial": { + "properties": { + "err": { + "type": "long" + }, + "ok": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "keyspace": { + "properties": { + "avg_ttl": { + "type": "long" + }, + "expires": { + "type": "long" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "keys": { + "type": "long" + } + } + } + } + }, + "system": { + "properties": { + "core": { + "properties": { + "id": { + "type": "long" + }, + "idle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "cores": { + "type": "long" + }, + "idle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "diskio": { + "properties": { + "io": { + "properties": { + "time": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "read": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "mount_point": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "fsstat": { + "properties": { + "count": { + "type": "long" + }, + "total_files": { + "type": "long" + }, + "total_size": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "type": "long" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "norm": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + } + } + } + } + }, + "memory": { + "properties": { + "actual": { + "properties": { + "free": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "free": { + "type": "long" + }, + "swap": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "process": { + "properties": { + "cgroup": { + "properties": { + "blkio": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "properties": { + "bytes": { + "type": "long" + }, + "ios": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "cfs": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "us": { + "type": "long" + } + } + }, + "shares": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "rt": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "runtime": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "stats": { + "properties": { + "periods": { + "type": "long" + }, + "throttled": { + "properties": { + "ns": { + "type": "long" + }, + "periods": { + "type": "long" + } + } + } + } + } + } + }, + "cpuacct": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "percpu": { + "properties": {} + }, + "stats": { + "properties": { + "system": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "user": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kmem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "kmem_tcp": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "mem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "memsw": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "active_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "active_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "cache": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memory_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memsw_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "major_page_faults": { + "type": "long" + }, + "mapped_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + }, + "pages_in": { + "type": "long" + }, + "pages_out": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "rss_huge": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "swap": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "unevictable": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cmdline": { + "ignore_above": 1024, + "type": "keyword" + }, + "cpu": { + "properties": { + "start_time": { + "type": "date" + }, + "system": { + "type": "long" + }, + "total": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "type": "long" + } + } + }, + "env": { + "properties": {} + }, + "fd": { + "properties": { + "limit": { + "properties": { + "hard": { + "type": "long" + }, + "soft": { + "type": "long" + } + } + }, + "open": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "rss": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "share": { + "type": "long" + }, + "size": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "socket": { + "properties": { + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "local": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "process": { + "properties": { + "cmdline": { + "ignore_above": 1024, + "type": "keyword" + }, + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "exe": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + } + } + }, + "remote": { + "properties": { + "etld_plus_one": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_error": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "user": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "zookeeper": { + "properties": { + "mntr": { + "properties": { + "approximate_data_size": { + "type": "long" + }, + "ephemerals_count": { + "type": "long" + }, + "followers": { + "type": "long" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "latency": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "max_file_descriptor_count": { + "type": "long" + }, + "num_alive_connections": { + "type": "long" + }, + "open_file_descriptor_count": { + "type": "long" + }, + "outstanding_requests": { + "type": "long" + }, + "packets": { + "properties": { + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "pending_syncs": { + "type": "long" + }, + "server_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "synced_followers": { + "type": "long" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "watch_count": { + "type": "long" + }, + "znode_count": { + "type": "long" + } + } + } + } + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + }, + "template": "metricbeat-*" +} \ No newline at end of file diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 35d8aa0960a..0f78db43e2e 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -614,6 +614,14 @@ output.elasticsearch: # Path to the Elasticsearch 2.x version of the template file. #template.versions.2x.path: "${path.config}/packetbeat.template-es2x.json" + # If set to true, packetbeat checks the Elasticsearch version at connect time, and if it + # is 6.x, it loads the file specified by the template.versions.6x.path setting. The + # default is true. + #template.versions.6x.enabled: true + + # Path to the Elasticsearch 6.x version of the template file. + #template.versions.6x.path: "${path.config}/packetbeat.template-es6x.json" + # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/packetbeat/packetbeat.template-es6x.json b/packetbeat/packetbeat.template-es6x.json new file mode 100644 index 00000000000..2ae387af7cf --- /dev/null +++ b/packetbeat/packetbeat.template-es6x.json @@ -0,0 +1,1397 @@ +{ + "mappings": { + "_default_": { + "_meta": { + "version": "5.4.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "amqp": { + "properties": { + "app-id": { + "ignore_above": 1024, + "type": "keyword" + }, + "arguments": { + "properties": {} + }, + "auto-delete": { + "type": "boolean" + }, + "class-id": { + "type": "long" + }, + "consumer-count": { + "type": "long" + }, + "consumer-tag": { + "ignore_above": 1024, + "type": "keyword" + }, + "content-encoding": { + "ignore_above": 1024, + "type": "keyword" + }, + "content-type": { + "ignore_above": 1024, + "type": "keyword" + }, + "correlation-id": { + "ignore_above": 1024, + "type": "keyword" + }, + "delivery-mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "delivery-tag": { + "type": "long" + }, + "durable": { + "type": "boolean" + }, + "exchange": { + "ignore_above": 1024, + "type": "keyword" + }, + "exchange-type": { + "ignore_above": 1024, + "type": "keyword" + }, + "exclusive": { + "type": "boolean" + }, + "expiration": { + "ignore_above": 1024, + "type": "keyword" + }, + "headers": { + "properties": {} + }, + "if-empty": { + "type": "boolean" + }, + "if-unused": { + "type": "boolean" + }, + "immediate": { + "type": "boolean" + }, + "mandatory": { + "type": "boolean" + }, + "message-count": { + "type": "long" + }, + "message-id": { + "ignore_above": 1024, + "type": "keyword" + }, + "method-id": { + "type": "long" + }, + "multiple": { + "type": "boolean" + }, + "no-ack": { + "type": "boolean" + }, + "no-local": { + "type": "boolean" + }, + "no-wait": { + "type": "boolean" + }, + "passive": { + "type": "boolean" + }, + "priority": { + "type": "long" + }, + "queue": { + "ignore_above": 1024, + "type": "keyword" + }, + "redelivered": { + "type": "boolean" + }, + "reply-code": { + "type": "long" + }, + "reply-text": { + "ignore_above": 1024, + "type": "keyword" + }, + "reply-to": { + "ignore_above": 1024, + "type": "keyword" + }, + "routing-key": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user-id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes_in": { + "type": "long" + }, + "bytes_out": { + "type": "long" + }, + "cassandra": { + "properties": { + "request": { + "properties": { + "headers": { + "properties": { + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "op": { + "ignore_above": 1024, + "type": "keyword" + }, + "stream": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "authentication": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "type": "long" + }, + "details": { + "properties": { + "alive": { + "type": "long" + }, + "arg_types": { + "ignore_above": 1024, + "type": "keyword" + }, + "blockfor": { + "type": "long" + }, + "data_present": { + "type": "boolean" + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + }, + "keyspace": { + "ignore_above": 1024, + "type": "keyword" + }, + "num_failures": { + "ignore_above": 1024, + "type": "keyword" + }, + "read_consistency": { + "ignore_above": 1024, + "type": "keyword" + }, + "received": { + "type": "long" + }, + "required": { + "type": "long" + }, + "stmt_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "table": { + "ignore_above": 1024, + "type": "keyword" + }, + "write_type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "msg": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "change": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "schema_change": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "change": { + "ignore_above": 1024, + "type": "keyword" + }, + "keyspace": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "object": { + "ignore_above": 1024, + "type": "keyword" + }, + "table": { + "ignore_above": 1024, + "type": "keyword" + }, + "target": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "headers": { + "properties": { + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "op": { + "ignore_above": 1024, + "type": "keyword" + }, + "stream": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "result": { + "properties": { + "keyspace": { + "ignore_above": 1024, + "type": "keyword" + }, + "prepared": { + "properties": { + "prepared_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "req_meta": { + "properties": { + "col_count": { + "type": "long" + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "keyspace": { + "ignore_above": 1024, + "type": "keyword" + }, + "paging_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "pkey_columns": { + "type": "long" + }, + "table": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resp_meta": { + "properties": { + "col_count": { + "type": "long" + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "keyspace": { + "ignore_above": 1024, + "type": "keyword" + }, + "paging_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "pkey_columns": { + "type": "long" + }, + "table": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "rows": { + "properties": { + "meta": { + "properties": { + "col_count": { + "type": "long" + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "keyspace": { + "ignore_above": 1024, + "type": "keyword" + }, + "paging_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "pkey_columns": { + "type": "long" + }, + "table": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "num_rows": { + "type": "long" + } + } + }, + "schema_change": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "change": { + "ignore_above": 1024, + "type": "keyword" + }, + "keyspace": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "object": { + "ignore_above": 1024, + "type": "keyword" + }, + "table": { + "ignore_above": 1024, + "type": "keyword" + }, + "target": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "supported": { + "properties": {} + }, + "warnings": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "client_geoip": { + "properties": { + "location": { + "type": "geo_point" + } + } + }, + "client_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "client_location": { + "type": "geo_point" + }, + "client_port": { + "ignore_above": 1024, + "type": "keyword" + }, + "client_proc": { + "ignore_above": 1024, + "type": "keyword" + }, + "client_server": { + "ignore_above": 1024, + "type": "keyword" + }, + "client_service": { + "ignore_above": 1024, + "type": "keyword" + }, + "connection_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "connecttime": { + "type": "long" + }, + "cpu_time": { + "type": "long" + }, + "dest": { + "properties": { + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip_location": { + "type": "geo_point" + }, + "ipv6": { + "ignore_above": 1024, + "type": "keyword" + }, + "ipv6_location": { + "type": "geo_point" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "outer_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "outer_ip_location": { + "type": "geo_point" + }, + "outer_ipv6": { + "ignore_above": 1024, + "type": "keyword" + }, + "outer_ipv6_location": { + "type": "geo_point" + }, + "port": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "net_bytes_total": { + "type": "long" + }, + "net_packets_total": { + "type": "long" + } + } + } + } + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "dns": { + "properties": { + "additionals": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "additionals_count": { + "type": "long" + }, + "answers": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "answers_count": { + "type": "long" + }, + "authorities": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "authorities_count": { + "type": "long" + }, + "flags": { + "properties": { + "authentic_data": { + "type": "boolean" + }, + "authoritative": { + "type": "boolean" + }, + "checking_disabled": { + "type": "boolean" + }, + "recursion_available": { + "type": "boolean" + }, + "recursion_desired": { + "type": "boolean" + }, + "truncated_response": { + "type": "boolean" + } + } + }, + "id": { + "type": "long" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "opt": { + "properties": { + "do": { + "type": "boolean" + }, + "ext_rcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "udp_size": { + "type": "long" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "question": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "etld_plus_one": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response_code": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "dnstime": { + "type": "long" + }, + "domloadtime": { + "type": "long" + }, + "fields": { + "properties": {} + }, + "final": { + "ignore_above": 1024, + "type": "keyword" + }, + "flow_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "norms": false, + "type": "text" + }, + "headers": { + "properties": {} + }, + "params": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "headers": { + "properties": {} + }, + "phrase": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "icmp": { + "properties": { + "request": { + "properties": { + "code": { + "type": "long" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "type": "long" + } + } + }, + "response": { + "properties": { + "code": { + "type": "long" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "icmp_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_time": { + "type": "date" + }, + "loadtime": { + "type": "long" + }, + "memcache": { + "properties": { + "protocol_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "properties": { + "automove": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "cas_unique": { + "type": "long" + }, + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "count_values": { + "type": "long" + }, + "delta": { + "type": "long" + }, + "dest_class": { + "type": "long" + }, + "exptime": { + "type": "long" + }, + "flags": { + "type": "long" + }, + "initial": { + "type": "long" + }, + "keys": { + "properties": {} + }, + "line": { + "ignore_above": 1024, + "type": "keyword" + }, + "noreply": { + "type": "boolean" + }, + "opaque": { + "type": "long" + }, + "opcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "opcode_value": { + "type": "long" + }, + "quiet": { + "type": "boolean" + }, + "raw_args": { + "ignore_above": 1024, + "type": "keyword" + }, + "sleep_us": { + "type": "long" + }, + "source_class": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "values": { + "properties": {} + }, + "vbucket": { + "type": "long" + }, + "verbosity": { + "type": "long" + } + } + }, + "response": { + "properties": { + "bytes": { + "type": "long" + }, + "cas_unique": { + "type": "long" + }, + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "count_values": { + "type": "long" + }, + "error_msg": { + "ignore_above": 1024, + "type": "keyword" + }, + "flags": { + "type": "long" + }, + "keys": { + "properties": {} + }, + "opaque": { + "type": "long" + }, + "opcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "opcode_value": { + "type": "long" + }, + "stats": { + "properties": {} + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_code": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "long" + }, + "values": { + "properties": {} + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "mongodb": { + "properties": { + "cursorId": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "fullCollectionName": { + "ignore_above": 1024, + "type": "keyword" + }, + "numberReturned": { + "type": "long" + }, + "numberToReturn": { + "type": "long" + }, + "numberToSkip": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "returnFieldsSelector": { + "ignore_above": 1024, + "type": "keyword" + }, + "selector": { + "ignore_above": 1024, + "type": "keyword" + }, + "startingFrom": { + "ignore_above": 1024, + "type": "keyword" + }, + "update": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mysql": { + "properties": { + "affected_rows": { + "type": "long" + }, + "error_code": { + "type": "long" + }, + "error_message": { + "ignore_above": 1024, + "type": "keyword" + }, + "insert_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "iserror": { + "type": "boolean" + }, + "num_fields": { + "ignore_above": 1024, + "type": "keyword" + }, + "num_rows": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "nfs": { + "properties": { + "minor_version": { + "type": "long" + }, + "opcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "notes": { + "ignore_above": 1024, + "type": "keyword" + }, + "outer_vlan": { + "ignore_above": 1024, + "type": "keyword" + }, + "params": { + "norms": false, + "type": "text" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pgsql": { + "properties": { + "error_code": { + "type": "long" + }, + "error_message": { + "ignore_above": 1024, + "type": "keyword" + }, + "error_severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "iserror": { + "type": "boolean" + }, + "num_fields": { + "ignore_above": 1024, + "type": "keyword" + }, + "num_rows": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "port": { + "ignore_above": 1024, + "type": "keyword" + }, + "proc": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "real_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "redis": { + "properties": { + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "return_value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "release": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "norms": false, + "type": "text" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + }, + "response": { + "norms": false, + "type": "text" + }, + "responsetime": { + "type": "long" + }, + "rpc": { + "properties": { + "auth_flavor": { + "ignore_above": 1024, + "type": "keyword" + }, + "call_size": { + "type": "long" + }, + "cred": { + "properties": { + "gid": { + "type": "long" + }, + "gids": { + "ignore_above": 1024, + "type": "keyword" + }, + "machinename": { + "ignore_above": 1024, + "type": "keyword" + }, + "stamp": { + "type": "long" + }, + "uid": { + "type": "long" + } + } + }, + "reply_size": { + "type": "long" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "time": { + "type": "long" + }, + "time_str": { + "ignore_above": 1024, + "type": "keyword" + }, + "xid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "properties": { + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip_location": { + "type": "geo_point" + }, + "ipv6": { + "ignore_above": 1024, + "type": "keyword" + }, + "ipv6_location": { + "type": "geo_point" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "outer_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "outer_ip_location": { + "type": "geo_point" + }, + "outer_ipv6": { + "ignore_above": 1024, + "type": "keyword" + }, + "outer_ipv6_location": { + "type": "geo_point" + }, + "port": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "net_bytes_total": { + "type": "long" + }, + "net_packets_total": { + "type": "long" + } + } + } + } + }, + "start_time": { + "type": "date" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "thrift": { + "properties": { + "exceptions": { + "ignore_above": 1024, + "type": "keyword" + }, + "params": { + "ignore_above": 1024, + "type": "keyword" + }, + "return_value": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + }, + "template": "packetbeat-*" +} \ No newline at end of file diff --git a/winlogbeat/winlogbeat.full.yml b/winlogbeat/winlogbeat.full.yml index 9b43f04aa09..260dc9411a9 100644 --- a/winlogbeat/winlogbeat.full.yml +++ b/winlogbeat/winlogbeat.full.yml @@ -195,6 +195,14 @@ output.elasticsearch: # Path to the Elasticsearch 2.x version of the template file. #template.versions.2x.path: "${path.config}/winlogbeat.template-es2x.json" + # If set to true, winlogbeat checks the Elasticsearch version at connect time, and if it + # is 6.x, it loads the file specified by the template.versions.6x.path setting. The + # default is true. + #template.versions.6x.enabled: true + + # Path to the Elasticsearch 6.x version of the template file. + #template.versions.6x.path: "${path.config}/winlogbeat.template-es6x.json" + # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/winlogbeat/winlogbeat.template-es6x.json b/winlogbeat/winlogbeat.template-es6x.json new file mode 100644 index 00000000000..b941492bb32 --- /dev/null +++ b/winlogbeat/winlogbeat.template-es6x.json @@ -0,0 +1,185 @@ +{ + "mappings": { + "_default_": { + "_meta": { + "version": "5.4.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "activity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "computer_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_data": { + "properties": {} + }, + "event_id": { + "type": "long" + }, + "fields": { + "properties": {} + }, + "keywords": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "message_error": { + "ignore_above": 1024, + "type": "keyword" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "opcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "process_id": { + "type": "long" + }, + "provider_guid": { + "ignore_above": 1024, + "type": "keyword" + }, + "record_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "related_activity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "task": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread_id": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_data": { + "properties": {} + }, + "version": { + "type": "long" + }, + "xml": { + "norms": false, + "type": "text" + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + }, + "template": "winlogbeat-*" +} \ No newline at end of file