[Filebeat] Improve ECS field mappings in santa module (#17982)

* Improve ECS field mappings in santa module

- move certificate.common_name to
  santa.certificate.common_name (breaking change)
- move certificate.sha256 to
  santa.certificate.sha256 (breaking change)
- move hash.sha256 to process.hash.sha256 (breaking change)
- event.action
- event.category
- event.kind
- event.type
- event.outcome
- log.level
- add full path to executable to process.args
- related.hash
- related.user
- Add new default file path

Closes #16180

(cherry picked from commit 81dfe61)

CHANGELOG.next.asciidoc

@@ -33,6 +33,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - CEF extensions are now mapped to the data types defined in the CEF guide. {pull}14342[14342]
 - Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
+- Improve ECS field mappings in santa module. move hash.sha256 to process.hash.sha256 & move certificate fields to santa.certificate . {issue}16180[16180] {pull}17982[17982]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -28708,7 +28708,7 @@ The disk volume path.
 
 --
 
-*`certificate.common_name`*::
+*`santa.certificate.common_name`*::
 +
 --
 Common name from code signing certificate.
@@ -28717,7 +28717,7 @@ type: keyword
 
 --
 
-*`certificate.sha256`*::
+*`santa.certificate.sha256`*::
 +
 --
 SHA256 hash of code signing certificate.

filebeat/module/santa/_meta/fields.yml

@@ -56,10 +56,10 @@
             - name: mount
               description: The disk volume path.
 
-    - name: certificate.common_name
-      type: keyword
-      description: Common name from code signing certificate.
+        - name: certificate.common_name
+          type: keyword
+          description: Common name from code signing certificate.
 
-    - name: certificate.sha256
-      type: keyword
-      description: SHA256 hash of code signing certificate.
+        - name: certificate.sha256
+          type: keyword
+          description: SHA256 hash of code signing certificate.

filebeat/module/santa/log/ingest/pipeline.yml

@@ -0,0 +1,91 @@
+description: Pipeline for parsing Google Santa logs.
+processors:
+- grok:
+    field: message
+    patterns:
+    - '\[%{TIMESTAMP_ISO8601:process.start}\] %{NOT_SEPARATOR:log.level} santad: action=%{NOT_SEPARATOR:santa.action}\|decision=%{NOT_SEPARATOR:santa.decision}\|reason=%{NOT_SEPARATOR:santa.reason}\|sha256=%{NOT_SEPARATOR:process.hash.sha256}\|path=%{NOT_SEPARATOR:process.executable}(\|args=%{NOT_SEPARATOR:santa.args})?(\|cert_sha256=%{NOT_SEPARATOR:santa.certificate.sha256})?(\|cert_cn=%{NOT_SEPARATOR:santa.certificate.common_name})?\|pid=%{NUMBER:process.pid:long}\|ppid=%{NUMBER:process.ppid:long}\|uid=%{NUMBER:user.id}\|user=%{NOT_SEPARATOR:user.name}\|gid=%{NUMBER:group.id}\|group=%{NOT_SEPARATOR:group.name}\|mode=%{WORD:santa.mode}'
+    - '\[%{TIMESTAMP_ISO8601:timestamp}\] %{NOT_SEPARATOR:log.level} santad: action=%{NOT_SEPARATOR:santa.action}\|mount=%{NOT_SEPARATOR:santa.disk.mount}\|volume=%{NOT_SEPARATOR:santa.disk.volume}\|bsdname=%{NOT_SEPARATOR:santa.disk.bsdname}\|fs=%{NOT_SEPARATOR:santa.disk.fs}\|model=%{NOT_SEPARATOR:santa.disk.model}\|serial=%{NOT_SEPARATOR:santa.disk.serial}\|bus=%{NOT_SEPARATOR:santa.disk.bus}\|dmgpath=%{NOT_SEPARATOR:santa.disk.dmgpath}?'
+    pattern_definitions:
+      NOT_SEPARATOR: '[^\|]+'
+- rename:
+    field: message
+    target_field: log.original
+- date:
+    field: process.start
+    target_field: process.start
+    formats:
+    - ISO8601
+    ignore_failure: true
+- set:
+    field: '@timestamp'
+    value: '{{ process.start }}'
+    ignore_failure: true
+- split:
+    field: santa.args
+    separator: ' '
+    ignore_failure: true
+- date:
+    field: timestamp
+    target_field: '@timestamp'
+    formats:
+    - ISO8601
+    ignore_failure: true
+- remove:
+    field: timestamp
+    ignore_missing: true
+- append:
+    field: process.args
+    value: "{{process.executable}}"
+    if: "ctx?.process?.executable != null"
+- foreach:
+    field: santa.args
+    processor:
+      append:
+        field: process.args
+        value: "{{_ingest._value}}"
+    ignore_missing: true
+- remove:
+    field: santa.args
+    ignore_missing: true
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: event.category
+    value: process
+    if: "ctx?.santa?.action == 'EXEC'"
+- append:
+    field: event.type
+    value: start
+    if: "ctx?.santa?.action == 'EXEC'"
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.santa?.decision == 'ALLOW'"
+- set:
+    field: event.outcome
+    value: failure
+    if: "ctx?.santa?.decision == 'DENY'"
+- set:
+    field: event.action
+    value: "{{santa.action}}"
+    if: "ctx?.santa?.action != null"
+- lowercase:
+    field: event.action
+    ignore_missing: true
+- append:
+    field: related.user
+    value: "{{user.name}}"
+    if: "ctx?.user?.name != null"
+- append:
+    field: related.hash
+    value: "{{santa.certificate.sha256}}"
+    if: "ctx?.santa?.certificate?.sha256 != null"
+- append:
+    field: related.hash
+    value: "{{process.hash.sha256}}"
+    if: "ctx?.process?.hash != null"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/santa/log/manifest.yml

@@ -4,8 +4,9 @@ var:
   - name: paths
     default:
       - /var/log/santa.log
+      - /var/db/santa/santa.log
   - name: input
     default: file
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml