Cherry-pick elastic#12695 to 7.2: Add support for client addresses wi…

…th port in Apache error logs (elastic#12729) Client addresses can contain the port in the default log format, add support for this. (cherry picked from commit b81c58c) Co-authored-by: William VINCENT <willouuu@gmail.com>
leweafan · Jul 2, 2019 · c5c6606 · c5c6606
1 parent ed5784d
commit c5c6606
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -34,6 +34,8 @@ https://github.com/elastic/beats/compare/v7.2.0...7.2[Check the HEAD diff]
 
 *Filebeat*
 
+- Add support for client addresses with port in Apache error logs {pull}12695[12695]
+
 *Heartbeat*
 
 *Journalbeat*

diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json
@@ -5,8 +5,8 @@
       "grok": {
         "field": "message",
         "patterns": [
-          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}\\])? %{GREEDYDATA:message}",
-          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}\\])? %{GREEDYDATA:message}"
+          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}",
+          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"
         ],
         "pattern_definitions": {
           "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"

diff --git a/filebeat/module/apache/error/test/test.log b/filebeat/module/apache/error/test/test.log
@@ -1,3 +1,4 @@
 [Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico
 [Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'
 [Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico
+[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 123.123.123.123:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html
diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json
@@ -50,5 +50,29 @@
         "source.geo.region_iso_code": "US-GA",
         "source.geo.region_name": "Georgia",
         "source.ip": "72.15.99.187"
+    },
+    {
+        "@timestamp": "2019-06-27T06:58:09.169Z",
+        "apache.error.module": "include",
+        "ecs.version": "1.0.0",
+        "event.dataset": "apache.error",
+        "event.module": "apache",
+        "fileset.name": "error",
+        "input.type": "log",
+        "log.level": "warn",
+        "log.offset": 384,
+        "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html",
+        "process.pid": 15934,
+        "service.type": "apache",
+        "source.address": "123.123.123.123",
+        "source.geo.city_name": "Beijing",
+        "source.geo.continent_name": "Asia",
+        "source.geo.country_iso_code": "CN",
+        "source.geo.location.lat": 39.9288,
+        "source.geo.location.lon": 116.3889,
+        "source.geo.region_iso_code": "CN-BJ",
+        "source.geo.region_name": "Beijing",
+        "source.ip": "123.123.123.123",
+        "source.port": "12345"
     }
 ]