Parse more fields from elasticsearch audit log

filebeat/docs/fields.asciidoc

@@ -3995,12 +3995,21 @@ Where the request originated: rest (request originated from a REST API request),
 
 --
 
+*`elasticsearch.audit.realm`*::
++
+--
+type: keyword
+
+The authentication realm the authentication was validated against
+
+--
+
 *`elasticsearch.audit.user.realm`*::
 +
 --
 type: keyword
 
-The authentication realm
+The user's authentication realm, if authenticated
 
 --
 
@@ -4026,6 +4035,24 @@ The name of the action that was executed
 
 --
 
+*`elasticsearch.audit.url.params`*::
++
+--
+type: keyword
+
+example: {username=jacknich2}
+
+REST URI parameters
+
+*`elasticsearch.audit.url.params.text`*::
++
+--
+type: text
+
+--
+
+--
+
 *`elasticsearch.audit.indices`*::
 +
 --
@@ -4037,6 +4064,17 @@ Indices accessed by action
 
 --
 
+*`elasticsearch.audit.request.id`*::
++
+--
+type: keyword
+
+example: WzL_kb6VSvOhAq0twPvHOQ
+
+Unique ID of request
+
+--
+
 *`elasticsearch.audit.request.name`*::
 +
 --

filebeat/module/elasticsearch/audit/_meta/fields.yml

@@ -10,8 +10,12 @@
       description: "Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)"
       example: "local_node"
       type: keyword
+    - name: realm
+      description: "The authentication realm the authentication was validated against"
+      example": "default_file"
+      type: keyword
     - name: user.realm
-      description: "The authentication realm"
+      description: "The user's authentication realm, if authenticated"
       example": "active_directory"
       type: keyword
     - name: user.roles
@@ -22,10 +26,21 @@
       description: "The name of the action that was executed"
       example: "cluster:monitor/main"
       type: keyword
+    - name: url.params
+      description: "REST URI parameters"
+      example: "{username=jacknich2}"
+      type: keyword
+      multi_fields:
+        - name: text
+          type: text
     - name: indices
       description: "Indices accessed by action"
       example: [ "foo-2019.01.04", "foo-2019.01.03", "foo-2019.01.06" ]
       type: keyword
+    - name: request.id
+      description: "Unique ID of request"
+      example: "WzL_kb6VSvOhAq0twPvHOQ"
+      type: keyword
     - name: request.name
       description: "The type of request that was executed"
       example: "ClearScrollRequest"

filebeat/module/elasticsearch/audit/ingest/pipeline-json.json

@@ -119,6 +119,33 @@
                 "field": "elasticsearch.audit.user.name",
                 "target_field": "user.name"
             }
+        },
+        {
+            "dot_expander": {
+                "field": "request.method",
+                "path": "elasticsearch.audit"
+            }
+        },
+        {
+            "rename": {
+                "field": "elasticsearch.audit.request.method",
+                "target_field": "http.request.method",
+                "ignore_missing": true
+            }
+
+        },
+        {
+            "dot_expander": {
+                "field": "request.body",
+                "path": "elasticsearch.audit"
+            }
+        },
+        {
+            "rename": {
+                "field": "elasticsearch.audit.request.body",
+                "target_field": "http.request.body.content",
+                "ignore_missing": true
+            }
         }
     ],
     "on_failure": [

filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json

@@ -11,17 +11,18 @@
                     "ES_AUDIT_EVENT_TYPE": "\\[%{WORD:event.type}\\]",
                     "ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin.type}\\])?",
                     "ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:source.ip}\\])?",
-                    "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:user.name}\\])?",
-                    "ES_AUDIT_REALM": "(realm\\=\\[%{WORD:elasticsearch.audit.user.realm}\\])?",
+                    "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{DATA:user.name}\\])?",
+                    "ES_AUDIT_REALM": "(realm\\=\\[%{WORD:elasticsearch.audit.realm}\\])?",
                     "ES_AUDIT_ROLES": "(roles\\=\\[%{DATA:elasticsearch.audit.user.roles}\\])?",
                     "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}(\\[%{DATA:elasticsearch.audit.sub_action}\\])?\\])?",
                     "ES_AUDIT_URI": "(uri=\\[%{DATA:url.original}\\])?",
+                    "ES_AUDIT_URI_PARAMS": "(params=\\[%{DATA:elasticsearch.audit.url.params}\\])?",
                     "ES_AUDIT_INDICES": "(indices\\=\\[%{DATA:elasticsearch.audit.indices}\\])?",
                     "ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request.name}\\])?",
                     "ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:http.request.body.content}\\])?"
                 },
                 "patterns": [
-                    "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
+                    "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_URI_PARAMS},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
                 ]
             }
         },

filebeat/module/elasticsearch/audit/test/test-access.log

@@ -6,3 +6,4 @@
 [2018-06-19T05:55:26,898] [transport] [access_denied]	origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]
 [2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]	origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]
 [2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted]     origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]
+[2019-01-27T20:04:27,244] [node-0] [rest] [authentication_success]    origin_address=[::1], principal=[elastic-admin], realm=[default_file], uri=[/_xpack/security/user/jacknich2], params=[{username=jacknich2}], request_body=[{"metadata":{"intelligence":7},"full_name":"Jack Nicholson","roles":["admin","other_role1"],"email":"jacknich@example.com"}]

filebeat/module/elasticsearch/audit/test/test-access.log-expected.json

@@ -132,8 +132,8 @@
         ],
         "elasticsearch.audit.layer": "transport",
         "elasticsearch.audit.origin.type": "transport",
+        "elasticsearch.audit.realm": "active_directory",
         "elasticsearch.audit.request.name": "SearchFreeContextRequest",
-        "elasticsearch.audit.user.realm": "active_directory",
         "elasticsearch.audit.user.roles": [
             "kibana_user",
             "my_custom_role_1",
@@ -150,5 +150,25 @@
         "service.type": "elasticsearch",
         "source.ip": "192.168.2.1",
         "user.name": "username"
+    },
+    {
+        "@timestamp": "2019-01-27T20:04:27.244Z",
+        "ecs.version": "1.0.0-beta2",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.realm": "default_file",
+        "elasticsearch.audit.url.params": "{username=jacknich2}",
+        "elasticsearch.node.name": "node-0",
+        "event.dataset": "elasticsearch.audit",
+        "event.module": "elasticsearch",
+        "event.type": "authentication_success",
+        "fileset.name": "audit",
+        "http.request.body.content": "{\"metadata\":{\"intelligence\":7},\"full_name\":\"Jack Nicholson\",\"roles\":[\"admin\",\"other_role1\"",
+        "input.type": "log",
+        "log.offset": 1626,
+        "message": "[2019-01-27T20:04:27,244] [node-0] [rest] [authentication_success]    origin_address=[::1], principal=[elastic-admin], realm=[default_file], uri=[/_xpack/security/user/jacknich2], params=[{username=jacknich2}], request_body=[{\"metadata\":{\"intelligence\":7},\"full_name\":\"Jack Nicholson\",\"roles\":[\"admin\",\"other_role1\"],\"email\":\"jacknich@example.com\"}]",
+        "service.type": "elasticsearch",
+        "source.ip": "::1",
+        "url.original": "/_xpack/security/user/jacknich2",
+        "user.name": "elastic-admin"
     }
 ]

filebeat/module/elasticsearch/audit/test/test-audit.log

@@ -4,3 +4,4 @@
 {"@timestamp":"2018-10-31T09:35:11,430", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "action":"cluster:admin/xpack/security/realm/cache/clear[n]", "request.name":"Node"}
 {"@timestamp":"2018-10-31T09:35:12,303", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest","origin.address":"[::1]:61711", "action":"cluster:admin/xpack/security/user/change_password", "request.name":"ChangePasswordRequest"}
 {"@timestamp":"2018-10-31T09:35:12,314", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":[".security-6"]}
+{"@timestamp":"2019-01-27T20:15:10,380", "node.name":"node-0", "node.id":"y8fa3M5zSSGo1M_KJRMUXw", "event.type":"rest", "event.action":"authentication_success", "user.name":"elastic-admin", "origin.type":"rest", "origin.address":"[::1]:58955", "realm":"default_file", "url.path":"/_search", "request.method":"GET", "request.body":"\n{\n    \"query\" : {\n        \"term\" : { \"user\" : \"kimchy\" }\n    }\n}\n", "request.id":"WzL_kb6VSvOhAq0twPvHOQ"}

filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json

@@ -141,5 +141,30 @@
         "source.ip": "127.0.0.1",
         "source.port": 9300,
         "user.name": "_xpack_security"
+    },
+    {
+        "@timestamp": "2019-01-27T20:15:10.380Z",
+        "ecs.version": "1.0.0-beta2",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.origin.type": "rest",
+        "elasticsearch.audit.realm": "default_file",
+        "elasticsearch.audit.request.id": "WzL_kb6VSvOhAq0twPvHOQ",
+        "elasticsearch.node.id": "y8fa3M5zSSGo1M_KJRMUXw",
+        "elasticsearch.node.name": "node-0",
+        "event.action": "authentication_success",
+        "event.dataset": "elasticsearch.audit",
+        "event.module": "elasticsearch",
+        "fileset.name": "audit",
+        "http.request.body.content": "\n{\n    \"query\" : {\n        \"term\" : { \"user\" : \"kimchy\" }\n    }\n}\n",
+        "http.request.method": "GET",
+        "input.type": "log",
+        "log.offset": 2056,
+        "message": "{\"@timestamp\":\"2019-01-27T20:15:10,380\", \"node.name\":\"node-0\", \"node.id\":\"y8fa3M5zSSGo1M_KJRMUXw\", \"event.type\":\"rest\", \"event.action\":\"authentication_success\", \"user.name\":\"elastic-admin\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:58955\", \"realm\":\"default_file\", \"url.path\":\"/_search\", \"request.method\":\"GET\", \"request.body\":\"\\n{\\n    \\\"query\\\" : {\\n        \\\"term\\\" : { \\\"user\\\" : \\\"kimchy\\\" }\\n    }\\n}\\n\", \"request.id\":\"WzL_kb6VSvOhAq0twPvHOQ\"}",
+        "service.type": "elasticsearch",
+        "source.address": "[::1]:58955",
+        "source.ip": "::1",
+        "source.port": 58955,
+        "url.original": "/_search",
+        "user.name": "elastic-admin"
     }
 ]