Add module for Kibana audit logs

elastic · Dec 4, 2020 · d3cafc4 · d3cafc4
1 parent 3f73ee0
commit d3cafc4
Show file tree

Hide file tree

Showing 15 changed files with 525 additions and 6 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -89203,6 +89203,105 @@ kibana Module
 
 
 
+*`kibana.space_id`*::
++
+--
+The id of the space associated with this request.
+
+type: keyword
+
+example: default
+
+--
+
+*`kibana.saved_object.type`*::
++
+--
+The type of the saved object associated with this request.
+
+type: keyword
+
+example: dashboard
+
+--
+
+*`kibana.saved_object.id`*::
++
+--
+The id of the saved object associated with this request.
+
+type: keyword
+
+example: 6295bdd0-0a0e-11e7-825f-6748cda7d858
+
+--
+
+*`kibana.add_to_spaces`*::
++
+--
+The set of space ids that a saved object was shared to.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.delete_from_spaces`*::
++
+--
+The set of space ids that a saved object was removed from.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.authentication_provider`*::
++
+--
+The authentication provider associated with a login event.
+
+type: keyword
+
+example: basic1
+
+--
+
+*`kibana.authentication_type`*::
++
+--
+The authentication provider type associated with a login event.
+
+type: keyword
+
+example: basic
+
+--
+
+*`kibana.authentication_realm`*::
++
+--
+The Elasticsearch authentication realm name which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
+*`kibana.lookup_realm`*::
++
+--
+The Elasticsearch lookup realm which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
 [float]
 === log
 

diff --git a/filebeat/docs/modules/kibana.asciidoc b/filebeat/docs/modules/kibana.asciidoc
@@ -38,6 +38,11 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
 
 
 [float]

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -203,14 +203,22 @@ filebeat.modules:
 
 #-------------------------------- Kibana Module --------------------------------
 - module: kibana
-  # All logs
+  # Server logs
   log:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+  # Audit logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #------------------------------- Logstash Module -------------------------------
 #- module: logstash
   # logs

diff --git a/filebeat/module/kibana/_meta/config.yml b/filebeat/module/kibana/_meta/config.yml
@@ -1,8 +1,16 @@
 - module: kibana
-  # All logs
+  # Server logs
   log:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  # Audit logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
diff --git a/filebeat/module/kibana/_meta/docs.asciidoc b/filebeat/module/kibana/_meta/docs.asciidoc
@@ -33,3 +33,8 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
diff --git a/filebeat/module/kibana/_meta/fields.yml b/filebeat/module/kibana/_meta/fields.yml
@@ -5,5 +5,41 @@
   fields:
     - name: kibana
       type: group
-      description: >
+      description: ''
       fields:
+        - name: space_id
+          description: "The id of the space associated with this request."
+          example: "default"
+          type: keyword
+        - name: saved_object.type
+          description: "The type of the saved object associated with this request."
+          example: "dashboard"
+          type: keyword
+        - name: saved_object.id
+          description: "The id of the saved object associated with this request."
+          example: "6295bdd0-0a0e-11e7-825f-6748cda7d858"
+          type: keyword
+        - name: add_to_spaces
+          description: "The set of space ids that a saved object was shared to."
+          example: "['default', 'marketing']"
+          type: keyword
+        - name: delete_from_spaces
+          description: "The set of space ids that a saved object was removed from."
+          example: "['default', 'marketing']"
+          type: keyword
+        - name: authentication_provider
+          description: "The authentication provider associated with a login event."
+          example: "basic1"
+          type: keyword
+        - name: authentication_type
+          description: "The authentication provider type associated with a login event."
+          example: "basic"
+          type: keyword
+        - name: authentication_realm
+          description: "The Elasticsearch authentication realm name which fulfilled a login event."
+          example: "native"
+          type: keyword
+        - name: lookup_realm
+          description: "The Elasticsearch lookup realm which fulfilled a login event."
+          example: "native"
+          type: keyword
diff --git a/filebeat/module/kibana/audit/config/audit.yml b/filebeat/module/kibana/audit/config/audit.yml
@@ -0,0 +1,13 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+processors:
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.6.0
diff --git a/filebeat/module/kibana/audit/ingest/pipeline-json.yml b/filebeat/module/kibana/audit/ingest/pipeline-json.yml
@@ -0,0 +1,88 @@
+description: Pipeline for parsing Kibana audit logs in JSON format
+processors:
+- json:
+    field: message
+    target_field: kibana._audit_temp
+
+- set:
+    field: "@timestamp"
+    value: "{{kibana._audit_temp.@timestamp}}"
+
+- set:
+    field: message
+    value: "{{kibana._audit_temp.message}}"
+
+- set:
+    if: ctx.kibana._audit_temp.event.action != null
+    field: event.action
+    value: "{{kibana._audit_temp.event.action}}"
+- set:
+    if: ctx.kibana._audit_temp.event.category != null
+    field: event.category
+    value: "{{kibana._audit_temp.event.category}}"
+- set:
+    if: ctx.kibana._audit_temp.event.outcome != null
+    field: event.outcome
+    value: "{{kibana._audit_temp.event.outcome}}"
+
+- rename:
+    if: ctx.kibana._audit_temp.url != null
+    field: kibana._audit_temp.url
+    target_field: "url"
+
+- set:
+    if: ctx.url?.query == null
+    field: url.original
+    value: '{{url.path}}'
+    ignore_empty_value: true
+- set:
+    if: ctx.url?.path != null && ctx.url?.query != null
+    field: url.original
+    value: '{{url.path}}?{{url.query}}'
+
+- rename:
+    if: ctx.kibana._audit_temp.http != null
+    field: kibana._audit_temp.http
+    target_field: http
+
+- rename:
+    if: ctx.kibana._audit_temp.user != null
+    field: kibana._audit_temp.user
+    target_field: user
+
+- rename:
+    if: ctx.kibana._audit_temp.trace != null
+    field: kibana._audit_temp.trace
+    target_field: tracing.trace
+
+- rename:
+    if: ctx.kibana._audit_temp.process?.pid != null
+    target_field: process
+    field: kibana._audit_temp.process
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.space_id != null
+    target_field: kibana.space_id
+    field: kibana._audit_temp.kibana.space_id
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.saved_object != null
+    target_field: kibana.saved_object
+    field: kibana._audit_temp.kibana.saved_object
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.add_to_spaces != null
+    target_field: kibana.add_to_spaces
+    field: kibana._audit_temp.kibana.add_to_spaces
+
+- rename:
+    if: ctx.kibana._audit_temp.kibana.delete_from_spaces != null
+    target_field: kibana.delete_from_spaces
+    field: kibana._audit_temp.kibana.delete_from_spaces
+
+- remove:
+    field: 'kibana._audit_temp'
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/filebeat/module/kibana/audit/ingest/pipeline.yml b/filebeat/module/kibana/audit/ingest/pipeline.yml
@@ -0,0 +1,21 @@
+description: Pipeline for parsing Kibana audit logs
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- pipeline:
+    name: '{< IngestPipeline "pipeline-json" >}'
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: related.user
+    value: "{{user.name}}"
+    if: "ctx?.user?.name != null"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/filebeat/module/kibana/audit/manifest.yml b/filebeat/module/kibana/audit/manifest.yml
@@ -0,0 +1,12 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/kibana/*_audit.json
+
+ingest_pipeline:
+  - ingest/pipeline.yml
+  - ingest/pipeline-json.yml
+
+input: config/audit.yml
diff --git a/filebeat/module/kibana/audit/test/test-audit-711.log b/filebeat/module/kibana/audit/test/test-audit-711.log
@@ -0,0 +1,7 @@
+{"@timestamp":"2020-11-20T12:05:14.528-05:00","message":"User is updating config [id=8.0.0]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"saved_object_update","category":"database","type":"change","outcome":"unknown"},"kibana":{"space_id":"marketing","saved_object":{"type":"config","id":"8.0.0"}},"user":{"name":"elastic","roles":["superuser"]},"trace":{"id":"e0bd67a1-a1b0-424d-9652-a350f88188eb"}}
+{"@timestamp":"2020-11-20T12:05:14.849-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/_find] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/_find","port":5603,"query":"default_search_operator=AND&has_reference=%5B%5D&page=1&per_page=1000&search_fields=title%5E3&search_fields=description&type=dashboard","scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"ae67a156-3847-4d89-9c97-86769df5bc2e"}}
+{"@timestamp":"2020-11-20T12:05:15.841-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/_bulk_get] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/_bulk_get","port":5603,"scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"cef382d1-7442-4f9a-8bee-0512c2b1da5a"}}
+{"@timestamp":"2020-11-20T12:05:15.853-05:00","message":"User has accessed index-pattern [id=metrics-*]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"saved_object_get","category":"database","type":"access","outcome":"success"},"kibana":{"space_id":"marketing","saved_object":{"type":"index-pattern","id":"metrics-*"}},"user":{"name":"elastic","roles":["superuser"]},"trace":{"id":"cef382d1-7442-4f9a-8bee-0512c2b1da5a"}}
+{"@timestamp":"2020-11-20T12:05:24.103-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/_find] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/_find","port":5603,"query":"fields=title&per_page=10&search=%22My%20Dashboard%22&search_fields=title&type=dashboard","scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"b10ee3ab-8102-4122-b4b5-5727e9b3d6a3"}}
+{"@timestamp":"2020-11-20T12:05:24.143-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/dashboard] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/dashboard","port":5603,"query":"overwrite=true","scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"4995c6bd-903c-42c2-af28-5cf17cc1cb6b"}}
+{"@timestamp":"2020-11-20T12:05:24.150-05:00","message":"User is creating dashboard [id=undefined]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"saved_object_create","category":"database","type":"creation","outcome":"unknown"},"kibana":{"space_id":"marketing","saved_object":{"type":"dashboard"}},"user":{"name":"elastic","roles":["superuser"]},"trace":{"id":"4995c6bd-903c-42c2-af28-5cf17cc1cb6b"}}