Add GC fileset to Elasticsearch Filebeat module

CHANGELOG.asciidoc

@@ -188,6 +188,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Correctly join partial log lines when using `docker` input. {pull}6967[6967]
 - Add support for TLS with client authentication to the TCP input {pull}7056[7056]
 - Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209]
+- Add GC fileset to the Elasticsearch module. {pull}7305[7305]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -800,6 +800,23 @@ elasticsearch Module
 
 
 
+[float]
+== gc fields
+
+Contains fields for the Elasticsearch JVM garbage collection logs.
+
+
+
+*`elasticsearch.gc.message`*::
++
+--
+type: text
+
+Original GC log message.
+
+
+--
+
 [float]
 == server fields
 

filebeat/module/elasticsearch/gc/_meta/fields.yml

@@ -0,0 +1,9 @@
+- name: gc
+  type: group
+  description: >
+    Contains fields for the Elasticsearch JVM garbage collection logs.
+  fields:
+    - name: message
+      type: text
+      description: >
+        Original GC log message.

filebeat/module/elasticsearch/gc/config/gc.yml

@@ -0,0 +1,15 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+multiline:
+  pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  negate: true
+  match: after
+
+  fields:
+    service.name: "elasticsearch"
+
+  fields_under_root: true

filebeat/module/elasticsearch/gc/ingest/pipeline.json

@@ -0,0 +1,37 @@
+{
+  "description": "Pipeline for parsing Elasticsearch JVM garbage collection logs",
+  "processors": [
+    {
+      "grok": {
+        "field": "message",
+        "patterns": [
+          "%{TIMESTAMP_ISO8601:timestamp}",
+          "%{GREEDYMULTILINE:elasticsearch.gc.message}"
+        ],
+        "pattern_definitions": {
+          "GREEDYMULTILINE": "(.|\n)*"
+        }
+      }
+    },
+    {
+      "rename": {
+        "field": "@timestamp",
+        "target_field": "event.created"
+      }
+    },
+    {
+      "rename": {
+        "field": "timestamp",
+        "target_field": "@timestamp"
+      }
+    }
+  ],
+  "on_failure": [
+    {
+      "set": {
+        "field": "error.message",
+        "value": "{{ _ingest.on_failure_message }}"
+      }
+    }
+  ]
+}

filebeat/module/elasticsearch/gc/manifest.yml

@@ -0,0 +1,11 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/elasticsearch/gc.log.[0-9]*
+    os.darwin: []
+    os.windows: []
+
+ingest_pipeline: ingest/pipeline.json
+input: config/gc.yml

filebeat/module/elasticsearch/gc/test/test.log

@@ -0,0 +1,4 @@
+2018-06-11T02:00:11.097+0000: 1396558.467: [GC (Allocation Failure) 2018-06-11T02:00:11.097+0000: 1396558.467: [ParNew
+Desired survivor size 8716288 bytes, new threshold 6 (max 6)
+- age   1:    1142528 bytes,    1142528 total
+: 153138K->12252K(153344K), 0.0077785 secs] 680068K->540560K(1031552K), 0.0078609 secs] [Times: user=0.02 sys=0.00, real=0.01 secs] 

filebeat/module/elasticsearch/gc/test/test.log-expected.json

@@ -0,0 +1,34 @@
+[
+  {
+    "_index": "filebeat-7.0.0-alpha1-2018.06.13",
+    "_type": "doc",
+    "_id": "QgKh-WMBTsDQhtAzVH5V",
+    "_score": 1,
+    "_source": {
+      "offset": 226,
+      "prospector": {
+        "type": "log"
+      },
+      "source": "/Users/jbryan/es/filebeat/snapshot/filebeat-7.0.0-alpha1-SNAPSHOT-darwin-x86_64/test.log",
+      "message": "2018-06-11T02:00:11.097+0000: 1396558.467: [GC (Allocation Failure) 2018-06-11T02:00:11.097+0000: 1396558.467: [ParNew\nDesired survivor size 8716288 bytes, new threshold 6 (max 6)\n- age   1:    1142528 bytes,    1142528 total",
+      "input": {
+        "type": "log"
+      },
+      "@timestamp": "2018-06-11T02:00:11.097+0000",
+      "service": {
+        "name": "elasticsearch"
+      },
+      "beat": {
+        "hostname": "peanut.lan",
+        "name": "peanut.lan",
+        "version": "7.0.0-alpha1"
+      },
+      "host": {
+        "name": "peanut.lan"
+      },
+      "event": {
+        "created": "2018-06-13T14:51:23.265Z"
+      }
+    }
+  }
+]

filebeat/module/elasticsearch/module.yml

@@ -1,2 +1,4 @@
 dashboards:
 
+- id: Filebeat-elasticsearch-gc-Dashboard
+  file: Filebeat-elasticsearch-gc.json