Fix parsing of apache trace log levels (#28717)

Apache levels may contain numbers as sublevels such as trace1.
elastic · Nov 23, 2021 · 78f3a3b · 78f3a3b
1 parent 9b154ad
commit 78f3a3b
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -185,6 +185,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for username in cisco asa security negotiation logs {pull}26975[26975]
 - Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
 - Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352]
+- Fix parsing of apache log levels including numbers. {pull}28717[28717]
 - Upgrade azure-eventhub sdk reference, contains potential checkpoint fixes. {pull}28919[28919]
 - Revert usageDetails api version to 2019-01-01. {pull}28995[28995]
 - Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]

diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml
@@ -11,10 +11,12 @@ processors:
     patterns:
     - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client
       %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}
-    - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\]
+    - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\]
       \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\](
       \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}
     pattern_definitions:
+      # Apache log level can have numeric sub-levels such as trace1.
+      APACHE_LOGLEVEL: '%{LOGLEVEL}[0-9]*'
       APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}'
     ignore_missing: true
 - grok:

diff --git a/filebeat/module/apache/error/test/sublevel.log b/filebeat/module/apache/error/test/sublevel.log
@@ -0,0 +1,2 @@
+[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'
+
diff --git a/filebeat/module/apache/error/test/sublevel.log-expected.json b/filebeat/module/apache/error/test/sublevel.log-expected.json
@@ -0,0 +1,21 @@
+[
+    {
+        "@timestamp": "2021-10-20T19:20:59.121-02:00",
+        "apache.error.module": "rewrite",
+        "event.category": "web",
+        "event.dataset": "apache.error",
+        "event.kind": "event",
+        "event.module": "apache",
+        "event.original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'",
+        "event.timezone": "-02:00",
+        "event.type": "info",
+        "fileset.name": "error",
+        "input.type": "log",
+        "log.level": "trace3",
+        "log.offset": 0,
+        "message": "mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'",
+        "process.pid": 121591,
+        "process.thread.id": 140413273032448,
+        "service.type": "apache"
+    }
+]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'