Skip to content

Commit

Permalink
Ingest ES structured audit logs
Browse files Browse the repository at this point in the history
  • Loading branch information
ycombinator committed Jan 27, 2019
1 parent 3f49f61 commit e9e01be
Show file tree
Hide file tree
Showing 12 changed files with 301 additions and 105 deletions.
53 changes: 14 additions & 39 deletions filebeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -3911,6 +3911,17 @@ elasticsearch Module
*`elasticsearch.node.id`*::
+
--
type: keyword
example: DSiWcTyeThWtUXLB9J0BMw
ID of the node
--
*`elasticsearch.node.name`*::
+
--
Expand Down Expand Up @@ -3972,7 +3983,7 @@ The layer from which this event originated: rest, transport or ip_filter
--
*`elasticsearch.audit.origin_type`*::
*`elasticsearch.audit.origin.type`*::
+
--
type: keyword
Expand All @@ -3983,7 +3994,7 @@ Where the request originated: rest (request originated from a REST API request),
--
*`elasticsearch.audit.realm`*::
*`elasticsearch.audit.user.realm`*::
+
--
type: keyword
Expand All @@ -3992,7 +4003,7 @@ The authentication realm
--
*`elasticsearch.audit.roles`*::
*`elasticsearch.audit.user.roles`*::
+
--
type: keyword
Expand Down Expand Up @@ -4036,33 +4047,6 @@ The type of request that was executed
--
*`elasticsearch.audit.event_type`*::
+
--
type: alias
alias to: event.type
--
*`elasticsearch.audit.origin_address`*::
+
--
type: alias
alias to: source.ip
--
*`elasticsearch.audit.uri`*::
+
--
type: alias
alias to: url.original
--
*`elasticsearch.audit.request_body`*::
+
--
Expand All @@ -4072,15 +4056,6 @@ alias to: http.request.body.content
--
*`elasticsearch.audit.principal`*::
+
--
type: alias
alias to: user.name
--
[float]
== deprecation fields
Expand Down
4 changes: 4 additions & 0 deletions filebeat/module/elasticsearch/_meta/fields.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
type: group
description: >
fields:
- name: node.id
description: "ID of the node"
example: "DSiWcTyeThWtUXLB9J0BMw"
type: keyword
- name: node.name
description: "Name of the node"
example: "vWNJsZ3"
Expand Down
22 changes: 3 additions & 19 deletions filebeat/module/elasticsearch/audit/_meta/fields.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@
description: "The layer from which this event originated: rest, transport or ip_filter"
example: "rest"
type: keyword
- name: origin_type
- name: origin.type
description: "Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)"
example: "local_node"
type: keyword
- name: realm
- name: user.realm
description: "The authentication realm"
example": "active_directory"
type: keyword
- name: roles
- name: user.roles
description: "Roles to which the principal belongs"
example: [ "kibana_user", "beats_admin" ]
type: keyword
Expand All @@ -30,23 +30,7 @@
description: "The type of request that was executed"
example: "ClearScrollRequest"
type: keyword
- name: event_type
type: alias
path: event.type
migration: true
- name: origin_address
type: alias
path: source.ip
migration: true
- name: uri
type: alias
path: url.original
migration: true
- name: request_body
type: alias
path: http.request.body.content
migration: true
- name: principal
type: alias
path: user.name
migration: true
137 changes: 137 additions & 0 deletions filebeat/module/elasticsearch/audit/ingest/pipeline-json.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
{
"description": "Pipeline for parsing elasticsearch audit logs in JSON format",
"processors": [
{
"json": {
"field": "message",
"target_field": "elasticsearch.audit"
}
},
{
"dot_expander": {
"field": "event.action",
"path": "elasticsearch.audit"
}
},
{
"rename": {
"field": "elasticsearch.audit.event.action",
"target_field": "event.action"
}
},
{
"dot_expander": {
"field": "event.type",
"path": "elasticsearch.audit"
}
},
{
"rename": {
"field": "elasticsearch.audit.event.type",
"target_field": "elasticsearch.audit.layer"
}
},
{
"dot_expander": {
"field": "origin.address",
"path": "elasticsearch.audit"
}
},
{
"grok": {
"field": "elasticsearch.audit.origin.address",
"patterns": [
"\\[%{IPORHOST:source.ip}\\]:%{INT:source.port:int}",
"%{IPORHOST:source.ip}:%{INT:source.port:int}"
]
}
},
{
"remove": {
"field": "elasticsearch.audit.origin.address"
}
},
{
"dot_expander": {
"field": "url.path",
"path": "elasticsearch.audit"
}
},
{
"dot_expander": {
"field": "url.query",
"path": "elasticsearch.audit"
}
},
{
"set": {
"if": "ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query == null",
"field": "url.original",
"value": "{{elasticsearch.audit.url.path}}"
}
},
{
"set": {
"if": "ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query != null",
"field": "url.original",
"value": "{{elasticsearch.audit.url.path}}?{{elasticsearch.audit.url.query}}"
}
},
{
"remove": {
"if": "ctx.elasticsearch.audit?.url?.path != null",
"field": "elasticsearch.audit.url.path"
}
},
{
"remove": {
"if": "ctx.elasticsearch.audit?.url?.query != null",
"field": "elasticsearch.audit.url.query"
}
},
{
"dot_expander": {
"field": "node.id",
"path": "elasticsearch.audit"
}
},
{
"dot_expander": {
"field": "node.name",
"path": "elasticsearch.audit"
}
},
{
"rename": {
"field": "elasticsearch.audit.node",
"target_field": "elasticsearch.node"
}
},
{
"dot_expander": {
"field": "user.name",
"path": "elasticsearch.audit"
}
},
{
"rename": {
"field": "elasticsearch.audit.user.name",
"target_field": "user.name"
}
},
{
"dot_expander": {
"field": "audit.@timestamp",
"path": "elasticsearch"
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
63 changes: 63 additions & 0 deletions filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
{
"description": "Pipeline for parsing elasticsearch audit logs in plaintext format",
"processors": [
{
"grok": {
"field": "message",
"pattern_definitions": {
"ES_TIMESTAMP": "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.@timestamp}\\]",
"ES_NODE_NAME": "(\\[%{DATA:elasticsearch.node.name}\\])?",
"ES_AUDIT_LAYER": "\\[%{WORD:elasticsearch.audit.layer}\\]",
"ES_AUDIT_EVENT_TYPE": "\\[%{WORD:event.type}\\]",
"ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin.type}\\])?",
"ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:source.ip}\\])?",
"ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:user.name}\\])?",
"ES_AUDIT_REALM": "(realm\\=\\[%{WORD:elasticsearch.audit.user.realm}\\])?",
"ES_AUDIT_ROLES": "(roles\\=\\[%{DATA:elasticsearch.audit.user.roles}\\])?",
"ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}(\\[%{DATA:elasticsearch.audit.sub_action}\\])?\\])?",
"ES_AUDIT_URI": "(uri=\\[%{DATA:url.original}\\])?",
"ES_AUDIT_INDICES": "(indices\\=\\[%{DATA:elasticsearch.audit.indices}\\])?",
"ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?",
"ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:http.request.body.content}\\])?"
},
"patterns": [
"%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
]
}
},
{
"split": {
"field": "elasticsearch.audit.user.roles",
"separator": ",",
"ignore_missing": true
}
},
{
"split": {
"field": "elasticsearch.audit.indices",
"separator": ",",
"ignore_missing": true
}
},
{
"script": {
"lang": "painless",
"source": "if (ctx.elasticsearch.audit.sub_action != null) { ctx.elasticsearch.audit.action += '[' + ctx.elasticsearch.audit.sub_action + ']' }"
}
},
{
"remove": {
"field": "elasticsearch.audit.sub_action",
"ignore_missing": true
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
Loading

0 comments on commit e9e01be

Please sign in to comment.