Skip to content

Commit

Permalink
Parse more fields from elasticsearch audit log (#10385)
Browse files Browse the repository at this point in the history 
Follow up to #10352 per #10352 (comment):

> While working on this PR I realized that we don't have sample lines for the **structured** elasticsearch audit log containing a request body (which is supposed to be parsed into the `http.request.body.content` field). I'm working with `@albertzaharovits` to get such a sample and will incorporate it into follow up PRs (for `master` and `6.x`).

Accordingly, this PR adds sample lines to the structured and unstructured log file test fixtures for the `elasticsearch/audit` fileset and teaches the fileset to parse any new fields encountered in these sample lines.
  • Loading branch information
@ycombinator
ycombinator authored Jan 29, 2019
1 parent 58c0488 commit 7207729
Show file tree
Hide file tree
Showing 9 changed files with 153 additions and 3 deletions.
33 changes: 33 additions & 0 deletions filebeat/docs/fields.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1138,6 +1138,17 @@ The REST endpoint URI
--
*`elasticsearch.audit.uri_params`*::
+
--
type: text
example: {username=jacknich2}
REST URI parameters
--
*`elasticsearch.audit.indices`*::
+
--
Expand All @@ -1149,6 +1160,28 @@ Indices accessed by action
--
*`elasticsearch.audit.request_id`*::
+
--
type: keyword
example: WzL_kb6VSvOhAq0twPvHOQ
Unique ID of request
--
*`elasticsearch.audit.request_method`*::
+
--
type: keyword
example: GET
Method of HTTP request
--
*`elasticsearch.audit.request`*::
+
--
Expand Down
2 changes: 1 addition & 1 deletion filebeat/include/fields.go
View file

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions filebeat/module/elasticsearch/audit/_meta/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,22 @@
description: "The REST endpoint URI"
example: /_xpack/security/_authenticate
type: keyword
- name: uri_params
description: "REST URI parameters"
example: "{username=jacknich2}"
type: text
- name: indices
description: "Indices accessed by action"
example: [ "foo-2019.01.04", "foo-2019.01.03", "foo-2019.01.06" ]
type: keyword
- name: request_id
description: "Unique ID of request"
example: "WzL_kb6VSvOhAq0twPvHOQ"
type: keyword
- name: request_method
description: "Method of HTTP request"
example: "GET"
type: keyword
- name: request
description: "The type of request that was executed"
example: "ClearScrollRequest"
Expand Down
46 changes: 46 additions & 0 deletions filebeat/module/elasticsearch/audit/ingest/pipeline-json.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,45 @@
"target_field": "elasticsearch.audit.principal"
}
},
{
"dot_expander": {
"field": "request.method",
"path": "elasticsearch.audit"
}
},
{
"rename": {
"if": "ctx.elasticsearch.audit?.request?.method != null",
"field": "elasticsearch.audit.request.method",
"target_field": "elasticsearch.audit.request_method"
}
},
{
"dot_expander": {
"field": "request.id",
"path": "elasticsearch.audit"
}
},
{
"rename": {
"if": "ctx.elasticsearch.audit?.request?.id != null",
"field": "elasticsearch.audit.request.id",
"target_field": "elasticsearch.audit.request_id"
}
},
{
"dot_expander": {
"field": "request.body",
"path": "elasticsearch.audit"
}
},
{
"rename": {
"if": "ctx.elasticsearch.audit?.request?.body != null",
"field": "elasticsearch.audit.request.body",
"target_field": "elasticsearch.audit.request_body"
}
},
{
"dot_expander": {
"field": "request.name",
Expand Down Expand Up @@ -173,6 +212,13 @@
"path": "elasticsearch.audit"
}
},
{
"rename": {
"if": "ctx.elasticsearch.audit?.user?.roles != null",
"field": "elasticsearch.audit.user.roles",
"target_field": "elasticsearch.audit.roles"
}
},
{
"remove": {
"field": "elasticsearch.audit.user"
Expand Down
5 changes: 3 additions & 2 deletions filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,18 @@
"ES_AUDIT_EVENT_TYPE": "\\[%{WORD:elasticsearch.audit.event_type}\\]",
"ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?",
"ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?",
"ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?",
"ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{DATA:elasticsearch.audit.principal}\\])?",
"ES_AUDIT_REALM": "(realm\\=\\[%{WORD:elasticsearch.audit.realm}\\])?",
"ES_AUDIT_ROLES": "(roles\\=\\[%{DATA:elasticsearch.audit.roles}\\])?",
"ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}(\\[%{DATA:elasticsearch.audit.sub_action}\\])?\\])?",
"ES_AUDIT_URI": "(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?",
"ES_AUDIT_URI_PARAMS": "(params=\\[%{DATA:elasticsearch.audit.uri_params}\\])?",
"ES_AUDIT_INDICES": "(indices\\=\\[%{DATA:elasticsearch.audit.indices}\\])?",
"ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?",
"ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?"
},
"patterns": [
"%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
"%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_URI_PARAMS},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
]
}
},
Expand Down
1 change: 1 addition & 0 deletions filebeat/module/elasticsearch/audit/test/test-access.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@
[2018-06-19T05:55:26,898] [transport] [access_denied] origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]
[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed] origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]
[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]
[2019-01-27T20:04:27,244] [node-0] [rest] [authentication_success] origin_address=[::1], principal=[elastic-admin], realm=[default_file], uri=[/_xpack/security/user/jacknich2], params=[{username=jacknich2}], request_body=[{"metadata":{"intelligence":7},"full_name":"Jack Nicholson","roles":["admin","other_role1"],"email":"jacknich@example.com"}]
20 changes: 20 additions & 0 deletions filebeat/module/elasticsearch/audit/test/test-access.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,5 +150,25 @@
"offset": 1210,
"prospector.type": "log",
"service.name": "elasticsearch"
},
{
"@timestamp": "2019-01-27T20:04:27.244Z",
"elasticsearch.audit.event_type": "authentication_success",
"elasticsearch.audit.layer": "rest",
"elasticsearch.audit.origin_address": "::1",
"elasticsearch.audit.principal": "elastic-admin",
"elasticsearch.audit.realm": "default_file",
"elasticsearch.audit.request_body": "{\"metadata\":{\"intelligence\":7},\"full_name\":\"Jack Nicholson\",\"roles\":[\"admin\",\"other_role1\"",
"elasticsearch.audit.uri": "/_xpack/security/user/jacknich2",
"elasticsearch.audit.uri_params": "{username=jacknich2}",
"elasticsearch.node.name": "node-0",
"event.dataset": "elasticsearch.audit",
"fileset.module": "elasticsearch",
"fileset.name": "audit",
"input.type": "log",
"message": "[2019-01-27T20:04:27,244] [node-0] [rest] [authentication_success] origin_address=[::1], principal=[elastic-admin], realm=[default_file], uri=[/_xpack/security/user/jacknich2], params=[{username=jacknich2}], request_body=[{\"metadata\":{\"intelligence\":7},\"full_name\":\"Jack Nicholson\",\"roles\":[\"admin\",\"other_role1\"],\"email\":\"jacknich@example.com\"}]",
"offset": 1626,
"prospector.type": "log",
"service.name": "elasticsearch"
}
]
1 change: 1 addition & 0 deletions filebeat/module/elasticsearch/audit/test/test-audit.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@
{"@timestamp":"2018-10-31T09:35:11,430", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "action":"cluster:admin/xpack/security/realm/cache/clear[n]", "request.name":"Node"}
{"@timestamp":"2018-10-31T09:35:12,303", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest","origin.address":"[::1]:61711", "action":"cluster:admin/xpack/security/user/change_password", "request.name":"ChangePasswordRequest"}
{"@timestamp":"2018-10-31T09:35:12,314", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":[".security-6"]}
{"@timestamp":"2019-01-27T20:15:10,380", "node.name":"node-0", "node.id":"y8fa3M5zSSGo1M_KJRMUXw", "event.type":"rest", "event.action":"authentication_success", "user.name":"elastic-admin", "origin.type":"rest", "origin.address":"[::1]:58955", "realm":"default_file", "url.path":"/_search", "request.method":"GET", "request.body":"\n{\n \"query\" : {\n \"term\" : { \"user\" : \"kimchy\" }\n }\n}\n", "request.id":"WzL_kb6VSvOhAq0twPvHOQ"}
36 changes: 36 additions & 0 deletions filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@
"elasticsearch.audit.origin_type": "local_node",
"elasticsearch.audit.principal": "_xpack_security",
"elasticsearch.audit.request": "ClearRealmCacheRequest",
"elasticsearch.audit.roles": [
"superuser"
],
"elasticsearch.audit.user_realm": "__attach",
"elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
"event.dataset": "elasticsearch.audit",
Expand All @@ -68,6 +71,9 @@
"elasticsearch.audit.origin_type": "local_node",
"elasticsearch.audit.principal": "_xpack_security",
"elasticsearch.audit.request": "Node",
"elasticsearch.audit.roles": [
"superuser"
],
"elasticsearch.audit.user_realm": "__attach",
"elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
"event.dataset": "elasticsearch.audit",
Expand All @@ -89,6 +95,9 @@
"elasticsearch.audit.origin_type": "rest",
"elasticsearch.audit.principal": "elastic",
"elasticsearch.audit.request": "ChangePasswordRequest",
"elasticsearch.audit.roles": [
"superuser"
],
"elasticsearch.audit.user_realm": "reserved",
"elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
"event.dataset": "elasticsearch.audit",
Expand All @@ -113,6 +122,9 @@
"elasticsearch.audit.origin_type": "local_node",
"elasticsearch.audit.principal": "_xpack_security",
"elasticsearch.audit.request": "CreateIndexRequest",
"elasticsearch.audit.roles": [
"superuser"
],
"elasticsearch.audit.user_realm": "__attach",
"elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
"event.dataset": "elasticsearch.audit",
Expand All @@ -123,5 +135,29 @@
"offset": 1676,
"prospector.type": "log",
"service.name": "elasticsearch"
},
{
"@timestamp": "2019-01-27T20:15:10.380Z",
"elasticsearch.audit.event_type": "authentication_success",
"elasticsearch.audit.layer": "rest",
"elasticsearch.audit.origin_address": "::1",
"elasticsearch.audit.origin_port": 58955,
"elasticsearch.audit.origin_type": "rest",
"elasticsearch.audit.principal": "elastic-admin",
"elasticsearch.audit.realm": "default_file",
"elasticsearch.audit.request_body": "\n{\n \"query\" : {\n \"term\" : { \"user\" : \"kimchy\" }\n }\n}\n",
"elasticsearch.audit.request_id": "WzL_kb6VSvOhAq0twPvHOQ",
"elasticsearch.audit.request_method": "GET",
"elasticsearch.audit.uri": "/_search",
"elasticsearch.node.id": "y8fa3M5zSSGo1M_KJRMUXw",
"elasticsearch.node.name": "node-0",
"event.dataset": "elasticsearch.audit",
"fileset.module": "elasticsearch",
"fileset.name": "audit",
"input.type": "log",
"message": "{\"@timestamp\":\"2019-01-27T20:15:10,380\", \"node.name\":\"node-0\", \"node.id\":\"y8fa3M5zSSGo1M_KJRMUXw\", \"event.type\":\"rest\", \"event.action\":\"authentication_success\", \"user.name\":\"elastic-admin\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:58955\", \"realm\":\"default_file\", \"url.path\":\"/_search\", \"request.method\":\"GET\", \"request.body\":\"\\n{\\n \\\"query\\\" : {\\n \\\"term\\\" : { \\\"user\\\" : \\\"kimchy\\\" }\\n }\\n}\\n\", \"request.id\":\"WzL_kb6VSvOhAq0twPvHOQ\"}",
"offset": 2056,
"prospector.type": "log",
"service.name": "elasticsearch"
}
]

0 comments on commit 7207729

Please sign in to comment.