From 7271c5515e0103c20b24de7b0a55bf8e1fce13c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <kvch@users.noreply.github.com>
Date: Fri, 5 Oct 2018 11:59:41 +0200
Subject: [PATCH] Support multiline logs in logstash/log fileset of Filebeat
 (#8562)

Multiline JDBC plugin logs were not parsed correctly. From now on the module is capable of aggregating log lines into a single multiline event and its pipeline can parse it correctly.
---
 CHANGELOG.asciidoc                                 |  1 +
 filebeat/module/logstash/log/config/log.yml        |  4 ++++
 .../module/logstash/log/ingest/pipeline-plain.json |  5 +++--
 .../module/logstash/log/test/logstash-plain.log    |  4 ++++
 .../log/test/logstash-plain.log-expected.json      | 14 ++++++++++++++
 5 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index fcaff833a09..f71d44c8371 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -63,6 +63,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Update CRI format to support partial/full tags. {pull}8265[8265]
 - Fix some errors happening when stopping syslog input. {pull}8347[8347]
 - Fix RFC3339 timezone and nanoseconds parsing with the syslog input. {pull}8346[8346]
+- Support multiline logs in logstash/log fileset of Filebeat. {pull}8562[8562]
 
 *Heartbeat*
 
diff --git a/filebeat/module/logstash/log/config/log.yml b/filebeat/module/logstash/log/config/log.yml
index 0afd17317d4..c960389e7ca 100644
--- a/filebeat/module/logstash/log/config/log.yml
+++ b/filebeat/module/logstash/log/config/log.yml
@@ -4,3 +4,7 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+multiline:
+  pattern: ^\[[0-9]{4}-[0-9]{2}-[0-9]{2}
+  negate: true
+  match: after
diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.json b/filebeat/module/logstash/log/ingest/pipeline-plain.json
index 82e9e6d2989..1ce1d969823 100644
--- a/filebeat/module/logstash/log/ingest/pipeline-plain.json
+++ b/filebeat/module/logstash/log/ingest/pipeline-plain.json
@@ -14,10 +14,11 @@
                 "field": "message",
                 "pattern_definitions": {
                     "LOGSTASH_CLASS_MODULE": "[\\w\\.]+\\s*",
-                    "LOGSTASH_LOGLEVEL": "INFO|ERROR|DEBUG|FATAL|WARN|TRACE"
+                    "LOGSTASH_LOGLEVEL": "INFO|ERROR|DEBUG|FATAL|WARN|TRACE",
+                    "GREEDYMULTILINE" : "(.|\n)*"
                 },
                 "patterns": [
-                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:logstash.log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\] %{GREEDYDATA:logstash.log.message}"
+                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:logstash.log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\] %{GREEDYMULTILINE:logstash.log.message}"
                 ]
             }
         },
diff --git a/filebeat/module/logstash/log/test/logstash-plain.log b/filebeat/module/logstash/log/test/logstash-plain.log
index 07e9bd6c58e..d65e2af70bb 100644
--- a/filebeat/module/logstash/log/test/logstash-plain.log
+++ b/filebeat/module/logstash/log/test/logstash-plain.log
@@ -1 +1,5 @@
 [2017-10-23T14:20:12,046][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
+[2017-11-20T03:55:00,318][INFO ][logstash.inputs.jdbc     ] (0.058950s) Select Name as [person.name]
+, Address as [person.address]
+from people
+
diff --git a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json
index 2157da6003d..c14a53e54e1 100644
--- a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json
+++ b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json
@@ -9,5 +9,19 @@
         "logstash.log.module": "logstash.modules.scaffold", 
         "offset": 0, 
         "prospector.type": "log"
+    }, 
+    {
+        "@timestamp": "2017-11-20T03:55:00,318", 
+        "fileset.module": "logstash", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "log.flags": [
+            "multiline"
+        ], 
+        "logstash.log.level": "INFO", 
+        "logstash.log.message": "(0.058950s) Select Name as [person.name]\n, Address as [person.address]\nfrom people\n", 
+        "logstash.log.module": "logstash.inputs.jdbc     ", 
+        "offset": 175, 
+        "prospector.type": "log"
     }
 ]
\ No newline at end of file