[Filebeat] Add module for Kibana audit logs (elastic#22696)

Co-authored-by: Thom Heymann <190132+thomheymann@users.noreply.github.com>

CHANGELOG.next.asciidoc

@@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Add fileset to ingest Kibana's ECS audit logs. {pull}22696[22696]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
 - Improve ECS categorization field mappings for nginx module. http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]

filebeat/docs/fields.asciidoc

@@ -89198,9 +89198,120 @@ kibana Module
 [float]
 === kibana
 
+Module for parsing Kibana logs.
 
 
 
+*`kibana.session_id`*::
++
+--
+The ID of the user session associated with this event. Each login attempt results in a unique session id.
+
+type: keyword
+
+example: 123e4567-e89b-12d3-a456-426614174000
+
+--
+
+*`kibana.space_id`*::
++
+--
+The id of the space associated with this event.
+
+type: keyword
+
+example: default
+
+--
+
+*`kibana.saved_object.type`*::
++
+--
+The type of the saved object associated with this event.
+
+type: keyword
+
+example: dashboard
+
+--
+
+*`kibana.saved_object.id`*::
++
+--
+The id of the saved object associated with this event.
+
+type: keyword
+
+example: 6295bdd0-0a0e-11e7-825f-6748cda7d858
+
+--
+
+*`kibana.add_to_spaces`*::
++
+--
+The set of space ids that a saved object was shared to.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.delete_from_spaces`*::
++
+--
+The set of space ids that a saved object was removed from.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.authentication_provider`*::
++
+--
+The authentication provider associated with a login event.
+
+type: keyword
+
+example: basic1
+
+--
+
+*`kibana.authentication_type`*::
++
+--
+The authentication provider type associated with a login event.
+
+type: keyword
+
+example: basic
+
+--
+
+*`kibana.authentication_realm`*::
++
+--
+The Elasticsearch authentication realm name which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
+*`kibana.lookup_realm`*::
++
+--
+The Elasticsearch lookup realm which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
 [float]
 === log
 

filebeat/docs/modules/kibana.asciidoc

@@ -38,6 +38,11 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
 
 
 [float]

filebeat/filebeat.reference.yml

@@ -203,14 +203,22 @@ filebeat.modules:
 
 #-------------------------------- Kibana Module --------------------------------
 - module: kibana
-  # All logs
+  # Server logs
   log:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+  # Audit logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #------------------------------- Logstash Module -------------------------------
 #- module: logstash
   # logs

filebeat/module/kibana/_meta/config.yml

@@ -1,8 +1,16 @@
 - module: kibana
-  # All logs
+  # Server logs
   log:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  # Audit logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/kibana/_meta/docs.asciidoc

@@ -33,3 +33,8 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+

filebeat/module/kibana/_meta/fields.yml

@@ -6,4 +6,45 @@
     - name: kibana
       type: group
       description: >
+        Module for parsing Kibana logs.
       fields:
+        - name: session_id
+          description: The ID of the user session associated with this event. Each login attempt results in a unique session id.
+          example: "123e4567-e89b-12d3-a456-426614174000"
+          type: keyword
+        - name: space_id
+          description: "The id of the space associated with this event."
+          example: "default"
+          type: keyword
+        - name: saved_object.type
+          description: "The type of the saved object associated with this event."
+          example: "dashboard"
+          type: keyword
+        - name: saved_object.id
+          description: "The id of the saved object associated with this event."
+          example: "6295bdd0-0a0e-11e7-825f-6748cda7d858"
+          type: keyword
+        - name: add_to_spaces
+          description: "The set of space ids that a saved object was shared to."
+          example: "['default', 'marketing']"
+          type: keyword
+        - name: delete_from_spaces
+          description: "The set of space ids that a saved object was removed from."
+          example: "['default', 'marketing']"
+          type: keyword
+        - name: authentication_provider
+          description: "The authentication provider associated with a login event."
+          example: "basic1"
+          type: keyword
+        - name: authentication_type
+          description: "The authentication provider type associated with a login event."
+          example: "basic"
+          type: keyword
+        - name: authentication_realm
+          description: "The Elasticsearch authentication realm name which fulfilled a login event."
+          example: "native"
+          type: keyword
+        - name: lookup_realm
+          description: "The Elasticsearch lookup realm which fulfilled a login event."
+          example: "native"
+          type: keyword

filebeat/module/kibana/audit/config/audit.yml

@@ -0,0 +1,16 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+processors:
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 0.0.0
+  - decode_json_fields:
+      fields: [message]
+      target: kibana._audit_temp

filebeat/module/kibana/audit/ingest/pipeline-json.yml

@@ -0,0 +1,106 @@
+description: Pipeline for parsing Kibana audit logs in JSON format
+processors:
+
+- set:
+    field: "@timestamp"
+    value: "{{kibana._audit_temp.@timestamp}}"
+
+- set:
+    field: message
+    value: "{{kibana._audit_temp.message}}"
+
+- set:
+    if: ctx.kibana._audit_temp.event.action != null
+    field: event.action
+    value: "{{kibana._audit_temp.event.action}}"
+- set:
+    if: ctx.kibana._audit_temp.event.category != null
+    field: event.category
+    value: "{{kibana._audit_temp.event.category}}"
+- set:
+    if: ctx.kibana._audit_temp.event.outcome != null
+    field: event.outcome
+    value: "{{kibana._audit_temp.event.outcome}}"
+- set:
+    if: ctx.kibana._audit_temp.event.type != null
+    field: event.type
+    value: "{{kibana._audit_temp.event.type}}"
+
+- remove:
+    field: 'ecs'
+- rename:
+    if: ctx.kibana._audit_temp.ecs != null
+    field: kibana._audit_temp.ecs
+    target_field: "ecs"
+
+- rename:
+    if: ctx.kibana._audit_temp.url != null
+    field: kibana._audit_temp.url
+    target_field: "url"
+
+- set:
+    if: ctx.url?.query == null
+    field: url.original
+    value: '{{url.path}}'
+    ignore_empty_value: true
+- set:
+    if: ctx.url?.path != null && ctx.url?.query != null
+    field: url.original
+    value: '{{url.path}}?{{url.query}}'
+
+- rename:
+    if: ctx.kibana._audit_temp.http != null
+    field: kibana._audit_temp.http
+    target_field: http
+
+- rename:
+    if: ctx.kibana._audit_temp.user != null
+    field: kibana._audit_temp.user
+    target_field: user
+
+- rename:
+    if: ctx.kibana._audit_temp.trace != null
+    field: kibana._audit_temp.trace
+    target_field: trace
+
+- rename:
+    if: ctx.kibana._audit_temp.process?.pid != null
+    target_field: process
+    field: kibana._audit_temp.process
+
+- rename:
+    if: ctx.kibana._audit_temp.error != null
+    target_field: error
+    field: kibana._audit_temp.error
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.session_id != null
+    target_field: kibana.session_id
+    field: kibana._audit_temp.kibana.session_id
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.space_id != null
+    target_field: kibana.space_id
+    field: kibana._audit_temp.kibana.space_id
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.saved_object != null
+    target_field: kibana.saved_object
+    field: kibana._audit_temp.kibana.saved_object
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.add_to_spaces != null
+    target_field: kibana.add_to_spaces
+    field: kibana._audit_temp.kibana.add_to_spaces
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.delete_from_spaces != null
+    target_field: kibana.delete_from_spaces
+    field: kibana._audit_temp.kibana.delete_from_spaces
+
+- remove:
+    field: 'kibana._audit_temp'
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/kibana/audit/ingest/pipeline.yml

@@ -0,0 +1,21 @@
+description: Pipeline for parsing Kibana audit logs
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- pipeline:
+    name: '{< IngestPipeline "pipeline-json" >}'
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: related.user
+    value: "{{user.name}}"
+    if: "ctx?.user?.name != null"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/kibana/audit/manifest.yml

@@ -0,0 +1,12 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/kibana/*_audit.json
+
+ingest_pipeline:
+  - ingest/pipeline.yml
+  - ingest/pipeline-json.yml
+
+input: config/audit.yml