diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 791a258e98a..0ef0442a3f9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -96,6 +96,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Improve handling of different path variants on Windows. {pull}3781[3781] - Restructure input.Event to be inline with outputs.Data {pull}3823[3823] - Add base for supporting prospector level processors {pull}3853[3853] +- Add auditd module for reading audit logs on Linux. {pull}3750[3750] {pull}3941[3941] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index eb5550bec47..c0e5ca69bf9 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -13,6 +13,7 @@ This document describes the fields that are exported by Filebeat. They are grouped in the following categories: * <> +* <> * <> * <> * <> @@ -285,6 +286,144 @@ type: keyword The module producing the logged message. +[[exported-fields-auditd]] +== Auditd Fields + +Module for parsing auditd logs. + + + +[float] +== auditd Fields + +Fields from the auditd logs. + + + +[float] +== log Fields + +Fields from the Linux audit log. Not all fields are documented here because they are dynamic and vary by audit event type. + + + +[float] +=== auditd.log.record_type + +The audit event type. + + +[float] +=== auditd.log.old_auid + +For login events this is the old audit ID used for the user prior to this login. + + +[float] +=== auditd.log.new_auid + +For login events this is the new audit ID. The audit ID can be used to trace future events to the user even if their identity changes (like becoming root). + + +[float] +=== auditd.log.old_ses + +For login events this is the old session ID used for the user prior to this login. + + +[float] +=== auditd.log.new_ses + +For login events this is the new session ID. It can be used to tie a user to future events by session ID. + + +[float] +=== auditd.log.sequence + +type: long + +The audit event sequence number. + + +[float] +=== auditd.log.pid + +type: long + +The ID of the process. + + +[float] +=== auditd.log.ppid + +type: long + +The ID of the process. + + +[float] +=== auditd.log.items + +type: long + +The number of items in an event. + + +[float] +=== auditd.log.item + +type: long + +The item field indicates which item out of the total number of items. This number is zero-based; a value of 0 means it is the first item. + + +[float] +== geoip Fields + +Contains GeoIP information gathered based on the `auditd.log.addr` field. Only present if the GeoIP Elasticsearch plugin is available and used. + + + +[float] +=== auditd.log.geoip.continent_name + +type: keyword + +The name of the continent. + + +[float] +=== auditd.log.geoip.city_name + +type: keyword + +The name of the city. + + +[float] +=== auditd.log.geoip.region_name + +type: keyword + +The name of the region. + + +[float] +=== auditd.log.geoip.country_iso_code + +type: keyword + +Country ISO code. + + +[float] +=== auditd.log.geoip.location + +type: geo_point + +The longitude and latitude. + + [[exported-fields-beat]] == Beat Fields @@ -865,130 +1004,6 @@ Fields from the system log files. -[float] -== audit Fields - -Fields from the Linux audit logs. Not all fields are documented here because they are dynamic and vary by audit event type. - - - -[float] -=== system.audit.record_type - -The audit event type. - - -[float] -=== system.audit.old_auid - -For login events this is the old audit ID used for the user prior to this login. - - -[float] -=== system.audit.new_auid - -For login events this is the new audit ID. The audit ID can be used to trace future events to the user even if their identity changes (like becoming root). - - -[float] -=== system.audit.old_ses - -For login events this is the old session ID used for the user prior to this login. - - -[float] -=== system.audit.new_ses - -For login events this is the new session ID. It can be used to tie a user to future events by session ID. - - -[float] -=== system.audit.sequence - -type: long - -The audit event sequence number. - - -[float] -=== system.audit.pid - -type: long - -The ID of the process. - - -[float] -=== system.audit.ppid - -type: long - -The ID of the process. - - -[float] -=== system.audit.items - -type: long - -The number of items in an event. - - -[float] -=== system.audit.item - -type: long - -The item field indicates which item out of the total number of items. This number is zero-based; a value of 0 means it is the first item. - - -[float] -== geoip Fields - -Contains GeoIP information gathered based on the `system.audit.addr` field. Only present if the GeoIP Elasticsearch plugin is available and used. - - - -[float] -=== system.audit.geoip.continent_name - -type: keyword - -The name of the continent. - - -[float] -=== system.audit.geoip.city_name - -type: keyword - -The name of the city. - - -[float] -=== system.audit.geoip.region_name - -type: keyword - -The name of the region. - - -[float] -=== system.audit.geoip.country_iso_code - -type: keyword - -Country ISO code. - - -[float] -=== system.audit.geoip.location - -type: geo_point - -The longitude and latitude. - - [float] == auth Fields diff --git a/filebeat/docs/images/kibana-system-audit.png b/filebeat/docs/images/kibana-audit-auditd.png similarity index 100% rename from filebeat/docs/images/kibana-system-audit.png rename to filebeat/docs/images/kibana-audit-auditd.png diff --git a/filebeat/docs/modules/auditd.asciidoc b/filebeat/docs/modules/auditd.asciidoc new file mode 100644 index 00000000000..beeca663729 --- /dev/null +++ b/filebeat/docs/modules/auditd.asciidoc @@ -0,0 +1,41 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-auditd]] +== Auditd module + +This module collects and parses logs from the audit daemon (`auditd`). + +[float] +=== Compatibility + +This module was tested with logs from `auditd` on OSes like CentOS 6 and +CentOS 7. + +This module is not available for Windows. + +[float] +=== Dashboard + +This module comes with a sample dashboard showing an overview of the audit log +data. You can build more specific dashboards that are tailored to the audit +rules that you use on your systems. + +image::./images/kibana-audit-auditd.png[] + +[float] +=== Syslog fileset settings + +[float] +==== var.paths + +An array of paths where to look for the log files. If left empty, Filebeat +will choose the paths depending on your operating systems. + + +=== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc index 756989d1e12..944bca571de 100644 --- a/filebeat/docs/modules/system.asciidoc +++ b/filebeat/docs/modules/system.asciidoc @@ -19,15 +19,10 @@ This module is not available for Windows. [float] === Dashboard -This module comes with sample dashboards. The first is a more generic syslog -dashboard that shows syslog data. +This module comes with a sample dashboard showing syslog data. image::./images/kibana-system.png[] -And second is the audit dashboard that shows audit log data. - -image::./images/kibana-system-audit.png[] - [float] === Syslog fileset settings diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index 30d29bd6b51..48dfa054713 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -4,6 +4,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> + * <> * <> * <> * <> @@ -14,6 +15,7 @@ This file is generated! See scripts/docs_collector.py include::modules-overview.asciidoc[] include::modules/apache2.asciidoc[] +include::modules/auditd.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nginx.asciidoc[] include::modules/system.asciidoc[] diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index f3caf49222a..25a143998eb 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -56,6 +56,19 @@ filebeat.modules: # can be added under this section. #prospector: +#------------------------------- Auditd Module ------------------------------- +#- module: auditd + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: + #-------------------------------- MySQL Module ------------------------------- #- module: mysql # Error logs diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index a493dd77eb5..545eac7b54a 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -14,6 +14,9 @@ filebeat.modules: #------------------------------- System Module ------------------------------- #- module: system +#------------------------------- Auditd Module ------------------------------- +#- module: auditd + #-------------------------------- MySQL Module ------------------------------- #- module: mysql diff --git a/filebeat/fileset/modules_test.go b/filebeat/fileset/modules_test.go index 60ac12d3031..5a3097302b5 100644 --- a/filebeat/fileset/modules_test.go +++ b/filebeat/fileset/modules_test.go @@ -29,6 +29,7 @@ func TestNewModuleRegistry(t *testing.T) { {Module: "nginx"}, {Module: "mysql"}, {Module: "system"}, + {Module: "auditd"}, } reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0") @@ -36,9 +37,10 @@ func TestNewModuleRegistry(t *testing.T) { assert.NotNil(t, reg) expectedModules := map[string][]string{ + "auditd": {"log"}, "nginx": {"access", "error"}, "mysql": {"slowlog", "error"}, - "system": {"syslog", "audit", "auth"}, + "system": {"syslog", "auth"}, } assert.Equal(t, len(expectedModules), len(reg.registry)) diff --git a/filebeat/module/auditd/_meta/config.full.yml b/filebeat/module/auditd/_meta/config.full.yml new file mode 100644 index 00000000000..5b0d02baf41 --- /dev/null +++ b/filebeat/module/auditd/_meta/config.full.yml @@ -0,0 +1,11 @@ +#- module: auditd + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: diff --git a/filebeat/module/auditd/_meta/config.yml b/filebeat/module/auditd/_meta/config.yml new file mode 100644 index 00000000000..844a9b2ee1a --- /dev/null +++ b/filebeat/module/auditd/_meta/config.yml @@ -0,0 +1 @@ +#- module: auditd diff --git a/filebeat/module/auditd/_meta/docs.asciidoc b/filebeat/module/auditd/_meta/docs.asciidoc new file mode 100644 index 00000000000..484c75da4de --- /dev/null +++ b/filebeat/module/auditd/_meta/docs.asciidoc @@ -0,0 +1,29 @@ +== Auditd module + +This module collects and parses logs from the audit daemon (`auditd`). + +[float] +=== Compatibility + +This module was tested with logs from `auditd` on OSes like CentOS 6 and +CentOS 7. + +This module is not available for Windows. + +[float] +=== Dashboard + +This module comes with a sample dashboard showing an overview of the audit log +data. You can build more specific dashboards that are tailored to the audit +rules that you use on your systems. + +image::./images/kibana-audit-auditd.png[] + +[float] +=== Syslog fileset settings + +[float] +==== var.paths + +An array of paths where to look for the log files. If left empty, Filebeat +will choose the paths depending on your operating systems. diff --git a/filebeat/module/auditd/_meta/fields.yml b/filebeat/module/auditd/_meta/fields.yml new file mode 100644 index 00000000000..360fb732671 --- /dev/null +++ b/filebeat/module/auditd/_meta/fields.yml @@ -0,0 +1,10 @@ +- key: auditd + title: "Auditd" + description: > + Module for parsing auditd logs. + fields: + - name: auditd + type: group + description: > + Fields from the auditd logs. + fields: diff --git a/filebeat/module/system/audit/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json b/filebeat/module/auditd/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json similarity index 86% rename from filebeat/module/system/audit/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json rename to filebeat/module/auditd/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json index 04dc54b9251..8910a16a501 100644 --- a/filebeat/module/system/audit/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json +++ b/filebeat/module/auditd/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json @@ -2,12 +2,12 @@ "hits": 0, "timeRestore": false, "description": "", - "title": "Filebeat System Audit", + "title": "Filebeat Auditd", "uiStateJSON": "{\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"6295bdd0-0a0e-11e7-825f-6748cda7d858\",\"panelIndex\":1,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"5ebdbe50-0a0f-11e7-825f-6748cda7d858\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2bb0fa70-0a11-11e7-9e84-43da493ad0c7\",\"panelIndex\":3,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"d1726930-0a7f-11e7-8b04-eb22a5669f27\",\"panelIndex\":5,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"c5411910-0a87-11e7-8b04-eb22a5669f27\",\"panelIndex\":6,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":7,\"type\":\"search\",\"id\":\"4ac0a370-0a11-11e7-8b04-eb22a5669f27\",\"col\":1,\"row\":8,\"columns\":[\"system.audit.record_type\",\"system.audit.sequence\",\"system.audit.acct\"],\"sort\":[\"@timestamp\",\"desc\"]}]", + "panelsJSON": "[{\"col\":1,\"id\":\"6295bdd0-0a0e-11e7-825f-6748cda7d858\",\"panelIndex\":1,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"5ebdbe50-0a0f-11e7-825f-6748cda7d858\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2bb0fa70-0a11-11e7-9e84-43da493ad0c7\",\"panelIndex\":3,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"d1726930-0a7f-11e7-8b04-eb22a5669f27\",\"panelIndex\":5,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"c5411910-0a87-11e7-8b04-eb22a5669f27\",\"panelIndex\":6,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":7,\"type\":\"search\",\"id\":\"4ac0a370-0a11-11e7-8b04-eb22a5669f27\",\"col\":1,\"row\":8,\"columns\":[\"auditd.log.record_type\",\"auditd.log.sequence\",\"auditd.log.acct\"],\"sort\":[\"@timestamp\",\"desc\"]}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } -} \ No newline at end of file +} diff --git a/filebeat/module/system/audit/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json b/filebeat/module/auditd/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json similarity index 66% rename from filebeat/module/system/audit/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json rename to filebeat/module/auditd/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json index 66e6e02edef..f342d27da69 100644 --- a/filebeat/module/system/audit/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json +++ b/filebeat/module/auditd/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json @@ -8,11 +8,11 @@ "title": "Audit Events", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:system.audit\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:auditd.log\",\"analyze_wildcard\":true}},\"filter\":[]}" }, "columns": [ - "system.audit.record_type", - "system.audit.sequence", - "system.audit.acct" + "auditd.log.record_type", + "auditd.log.sequence", + "auditd.log.acct" ] -} \ No newline at end of file +} diff --git a/filebeat/module/auditd/_meta/kibana/visualization/2bb0fa70-0a11-11e7-9e84-43da493ad0c7.json b/filebeat/module/auditd/_meta/kibana/visualization/2bb0fa70-0a11-11e7-9e84-43da493ad0c7.json new file mode 100644 index 00000000000..422da1d66f7 --- /dev/null +++ b/filebeat/module/auditd/_meta/kibana/visualization/2bb0fa70-0a11-11e7-9e84-43da493ad0c7.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"type\":\"timelion\",\"title\":\"Audit Event Results\",\"params\":{\"expression\":\".es(q=\\\"_exists_:auditd.log NOT auditd.log.res:failure\\\").label(\\\"Success\\\") .es(q=\\\"auditd.log.res:failed\\\").label(\\\"Failure\\\").title(\\\"Audit Event Results\\\")\",\"interval\":\"auto\"}}", + "description": "", + "title": "Audit Event Results", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} diff --git a/filebeat/module/system/audit/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json b/filebeat/module/auditd/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json similarity index 69% rename from filebeat/module/system/audit/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json rename to filebeat/module/auditd/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json index 57104c8c177..c12a7b3daf5 100644 --- a/filebeat/module/system/audit/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json +++ b/filebeat/module/auditd/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json @@ -1,10 +1,10 @@ { - "visState": "{\"title\":\"Audit Top Exec Commands\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"system.audit.a0\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Command (arg 0)\"}}],\"listeners\":{}}", + "visState": "{\"title\":\"Audit Top Exec Commands\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"auditd.log.a0\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Command (arg 0)\"}}],\"listeners\":{}}", "description": "", "title": "Audit Top Exec Commands", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"system.audit.record_type:EXECVE\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"auditd.log.record_type:EXECVE\",\"analyze_wildcard\":true}},\"filter\":[]}" } -} \ No newline at end of file +} diff --git a/filebeat/module/system/audit/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json b/filebeat/module/auditd/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json similarity index 82% rename from filebeat/module/system/audit/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json rename to filebeat/module/auditd/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json index a46013440c4..73de4815ee2 100644 --- a/filebeat/module/system/audit/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json +++ b/filebeat/module/auditd/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json @@ -1,5 +1,5 @@ { - "visState": "{\n \"title\": \"Audit Event Types\",\n \"type\": \"pie\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"system.audit.record_type\",\n \"size\": 50,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", + "visState": "{\n \"title\": \"Audit Event Types\",\n \"type\": \"pie\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"auditd.log.record_type\",\n \"size\": 50,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "Audit Event Types", "uiStateJSON": "{}", @@ -7,4 +7,4 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": "{\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" } -} \ No newline at end of file +} diff --git a/filebeat/module/system/audit/_meta/kibana/visualization/c5411910-0a87-11e7-8b04-eb22a5669f27.json b/filebeat/module/auditd/_meta/kibana/visualization/c5411910-0a87-11e7-8b04-eb22a5669f27.json similarity index 86% rename from filebeat/module/system/audit/_meta/kibana/visualization/c5411910-0a87-11e7-8b04-eb22a5669f27.json rename to filebeat/module/auditd/_meta/kibana/visualization/c5411910-0a87-11e7-8b04-eb22a5669f27.json index 56acd439134..22a4543f66a 100644 --- a/filebeat/module/system/audit/_meta/kibana/visualization/c5411910-0a87-11e7-8b04-eb22a5669f27.json +++ b/filebeat/module/auditd/_meta/kibana/visualization/c5411910-0a87-11e7-8b04-eb22a5669f27.json @@ -1,5 +1,5 @@ { - "visState": "{\"title\":\"Audit Event Account Tag Cloud\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"linear\",\"orientation\":\"single\",\"minFontSize\":15,\"maxFontSize\":42,\"hideLabel\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"system.audit.acct\",\"size\":15,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "visState": "{\"title\":\"Audit Event Account Tag Cloud\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"linear\",\"orientation\":\"single\",\"minFontSize\":15,\"maxFontSize\":42,\"hideLabel\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"auditd.log.acct\",\"size\":15,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", "description": "", "title": "Audit Event Account Tag Cloud", "uiStateJSON": "{}", @@ -7,4 +7,4 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" } -} \ No newline at end of file +} diff --git a/filebeat/module/system/audit/_meta/kibana/visualization/d1726930-0a7f-11e7-8b04-eb22a5669f27.json b/filebeat/module/auditd/_meta/kibana/visualization/d1726930-0a7f-11e7-8b04-eb22a5669f27.json similarity index 87% rename from filebeat/module/system/audit/_meta/kibana/visualization/d1726930-0a7f-11e7-8b04-eb22a5669f27.json rename to filebeat/module/auditd/_meta/kibana/visualization/d1726930-0a7f-11e7-8b04-eb22a5669f27.json index 2b81cc06eb6..23850e2d62d 100644 --- a/filebeat/module/system/audit/_meta/kibana/visualization/d1726930-0a7f-11e7-8b04-eb22a5669f27.json +++ b/filebeat/module/auditd/_meta/kibana/visualization/d1726930-0a7f-11e7-8b04-eb22a5669f27.json @@ -1,5 +1,5 @@ { - "visState": "{\"title\":\"Audit Event Address Geo Location\",\"type\":\"tile_map\",\"params\":{\"mapType\":\"Scaled Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatRadius\":25,\"heatBlur\":15,\"heatNormalizeData\":true,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[15,5],\"wms\":{\"enabled\":false,\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\"options\":{\"version\":\"1.3.0\",\"layers\":\"0\",\"format\":\"image/png\",\"transparent\":true,\"attribution\":\"Maps provided by USGS\",\"styles\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"system.audit.geoip.location\",\"autoPrecision\":true,\"precision\":2}}],\"listeners\":{}}", + "visState": "{\"title\":\"Audit Event Address Geo Location\",\"type\":\"tile_map\",\"params\":{\"mapType\":\"Scaled Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatRadius\":25,\"heatBlur\":15,\"heatNormalizeData\":true,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[15,5],\"wms\":{\"enabled\":false,\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\"options\":{\"version\":\"1.3.0\",\"layers\":\"0\",\"format\":\"image/png\",\"transparent\":true,\"attribution\":\"Maps provided by USGS\",\"styles\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"auditd.log.geoip.location\",\"autoPrecision\":true,\"precision\":2}}],\"listeners\":{}}", "description": "", "title": "Audit Event Address Geo Location", "uiStateJSON": "{}", @@ -7,4 +7,4 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" } -} \ No newline at end of file +} diff --git a/filebeat/module/system/audit/_meta/fields.yml b/filebeat/module/auditd/log/_meta/fields.yml similarity index 92% rename from filebeat/module/system/audit/_meta/fields.yml rename to filebeat/module/auditd/log/_meta/fields.yml index 05b16091eed..fab2ee82ac7 100644 --- a/filebeat/module/system/audit/_meta/fields.yml +++ b/filebeat/module/auditd/log/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: audit +- name: log type: group description: > - Fields from the Linux audit logs. Not all fields are documented here because + Fields from the Linux audit log. Not all fields are documented here because they are dynamic and vary by audit event type. fields: - name: record_type @@ -48,7 +48,7 @@ - name: geoip type: group description: > - Contains GeoIP information gathered based on the `system.audit.addr` + Contains GeoIP information gathered based on the `auditd.log.addr` field. Only present if the GeoIP Elasticsearch plugin is available and used. fields: diff --git a/filebeat/module/system/audit/config/audit.yml b/filebeat/module/auditd/log/config/log.yml similarity index 100% rename from filebeat/module/system/audit/config/audit.yml rename to filebeat/module/auditd/log/config/log.yml diff --git a/filebeat/module/system/audit/ingest/pipeline.json b/filebeat/module/auditd/log/ingest/pipeline.json similarity index 59% rename from filebeat/module/system/audit/ingest/pipeline.json rename to filebeat/module/auditd/log/ingest/pipeline.json index 2b176e90fce..77a37f82d72 100644 --- a/filebeat/module/system/audit/ingest/pipeline.json +++ b/filebeat/module/auditd/log/ingest/pipeline.json @@ -1,49 +1,49 @@ { - "description": "Pipeline for parsing Linux audit logs", + "description": "Pipeline for parsing Linux auditd logs", "processors": [ { "grok": { "field": "message", "pattern_definitions": { - "AUDIT_TYPE": "^type=%{NOTSPACE:system.audit.record_type}", - "AUDIT_PREFIX": "%{AUDIT_TYPE} msg=audit\\(%{NUMBER:system.audit.epoch}:%{NUMBER:system.audit.sequence}\\):(%{DATA})?", + "AUDIT_TYPE": "^type=%{NOTSPACE:auditd.log.record_type}", + "AUDIT_PREFIX": "%{AUDIT_TYPE} msg=audit\\(%{NUMBER:auditd.log.epoch}:%{NUMBER:auditd.log.sequence}\\):(%{DATA})?", "AUDIT_KEY_VALUES": "%{WORD}=%{GREEDYDATA}" }, "patterns": [ - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:system.audit.kv} old auid=%{NUMBER:system.audit.old_auid} new auid=%{NUMBER:system.audit.new_auid} old ses=%{NUMBER:system.audit.old_ses} new ses=%{NUMBER:system.audit.new_ses}", - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:system.audit.kv} msg=['\"](%{DATA:system.audit.msg}\\s+)?%{AUDIT_KEY_VALUES:system.audit.sub_kv}['\"]", - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:system.audit.kv}", + "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} old auid=%{NUMBER:auditd.log.old_auid} new auid=%{NUMBER:auditd.log.new_auid} old ses=%{NUMBER:auditd.log.old_ses} new ses=%{NUMBER:auditd.log.new_ses}", + "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg=['\"](%{DATA:auditd.log.msg}\\s+)?%{AUDIT_KEY_VALUES:auditd.log.sub_kv}['\"]", + "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv}", "%{AUDIT_PREFIX}", - "%{AUDIT_TYPE} %{AUDIT_KEY_VALUES:system.audit.kv}" + "%{AUDIT_TYPE} %{AUDIT_KEY_VALUES:auditd.log.kv}" ] } }, { "kv": { - "field": "system.audit.kv", + "field": "auditd.log.kv", "field_split": "\\s+", "value_split": "=", - "target_field": "system.audit" + "target_field": "auditd.log" } }, { "kv": { - "field": "system.audit.sub_kv", + "field": "auditd.log.sub_kv", "field_split": "\\s+", "value_split": "=", - "target_field": "system.audit", + "target_field": "auditd.log", "ignore_missing": true } }, { "remove": { - "field": "system.audit.kv", + "field": "auditd.log.kv", "ignore_failure": true } }, { "remove": { - "field": "system.audit.sub_kv", + "field": "auditd.log.sub_kv", "ignore_failure": true } }, @@ -55,7 +55,7 @@ }, { "date": { - "field": "system.audit.epoch", + "field": "auditd.log.epoch", "target_field": "@timestamp", "formats": [ "UNIX" @@ -65,41 +65,41 @@ }, { "remove": { - "field": "system.audit.epoch", + "field": "auditd.log.epoch", "ignore_failure": true } }, { "convert": { - "field" : "system.audit.sequence", + "field" : "auditd.log.sequence", "type": "integer", "ignore_missing": true } }, { "convert": { - "field" : "system.audit.pid", + "field" : "auditd.log.pid", "type": "integer", "ignore_missing": true } }, { "convert": { - "field" : "system.audit.ppid", + "field" : "auditd.log.ppid", "type": "integer", "ignore_missing": true } }, { "convert": { - "field" : "system.audit.item", + "field" : "auditd.log.item", "type": "integer", "ignore_missing": true } }, { "convert": { - "field" : "system.audit.items", + "field" : "auditd.log.items", "type": "integer", "ignore_missing": true } @@ -107,13 +107,13 @@ { "script": { "lang": "painless", - "inline": " String trimQuotes(def v) {\n if (v.startsWith(\"'\") || v.startsWith('\"')) {\n v = v.substring(1, v.length());\n }\n if (v.endsWith(\"'\") || v.endsWith('\"')) {\n v = v.substring(0, v.length()-1);\n } \n return v;\n }\n \n boolean isHexAscii(String v) {\n def len = v.length();\n if (len == 0 || len % 2 != 0) {\n return false; \n }\n \n for (int i = 0 ; i < len ; i++) {\n if (Character.digit(v.charAt(i), 16) == -1) {\n return false;\n }\n }\n\n return true;\n }\n \n String convertHexToString(String hex) {\n\t StringBuilder sb = new StringBuilder();\n\n for (int i=0; i < hex.length() - 1; i+=2) {\n String output = hex.substring(i, (i + 2));\n int decimal = Integer.parseInt(output, 16);\n sb.append((char)decimal);\n }\n\n return sb.toString();\n }\n \n def possibleHexKeys = ['exe', 'cmd'];\n \n def audit = ctx.system.get(\"audit\");\n Iterator entries = audit.entrySet().iterator();\n while (entries.hasNext()) {\n def e = entries.next();\n def k = e.getKey();\n def v = e.getValue(); \n\n // Remove entries whose value is ?\n if (v == \"?\" || v == \"(null)\" || v == \"\") {\n entries.remove();\n continue;\n }\n \n // Convert hex values to ASCII.\n if (possibleHexKeys.contains(k) && isHexAscii(v)) {\n v = convertHexToString(v);\n audit.put(k, v);\n }\n \n // Trim quotes.\n if (v instanceof String) {\n v = trimQuotes(v);\n audit.put(k, v);\n }\n \n // Convert arch.\n if (k == \"arch\" && v == \"c000003e\") {\n audit.put(k, \"x86_64\");\n }\n }" + "inline": " String trimQuotes(def v) {\n if (v.startsWith(\"'\") || v.startsWith('\"')) {\n v = v.substring(1, v.length());\n }\n if (v.endsWith(\"'\") || v.endsWith('\"')) {\n v = v.substring(0, v.length()-1);\n } \n return v;\n }\n \n boolean isHexAscii(String v) {\n def len = v.length();\n if (len == 0 || len % 2 != 0) {\n return false; \n }\n \n for (int i = 0 ; i < len ; i++) {\n if (Character.digit(v.charAt(i), 16) == -1) {\n return false;\n }\n }\n\n return true;\n }\n \n String convertHexToString(String hex) {\n\t StringBuilder sb = new StringBuilder();\n\n for (int i=0; i < hex.length() - 1; i+=2) {\n String output = hex.substring(i, (i + 2));\n int decimal = Integer.parseInt(output, 16);\n sb.append((char)decimal);\n }\n\n return sb.toString();\n }\n \n def possibleHexKeys = ['exe', 'cmd'];\n \n def audit = ctx.auditd.get(\"log\");\n Iterator entries = audit.entrySet().iterator();\n while (entries.hasNext()) {\n def e = entries.next();\n def k = e.getKey();\n def v = e.getValue(); \n\n // Remove entries whose value is ?\n if (v == \"?\" || v == \"(null)\" || v == \"\") {\n entries.remove();\n continue;\n }\n \n // Convert hex values to ASCII.\n if (possibleHexKeys.contains(k) && isHexAscii(v)) {\n v = convertHexToString(v);\n audit.put(k, v);\n }\n \n // Trim quotes.\n if (v instanceof String) {\n v = trimQuotes(v);\n audit.put(k, v);\n }\n \n // Convert arch.\n if (k == \"arch\" && v == \"c000003e\") {\n audit.put(k, \"x86_64\");\n }\n }" } }, { "geoip": { - "field": "system.audit.addr", - "target_field": "system.audit.geoip", + "field": "auditd.log.addr", + "target_field": "auditd.log.geoip", "ignore_failure": true } } diff --git a/filebeat/module/system/audit/manifest.yml b/filebeat/module/auditd/log/manifest.yml similarity index 88% rename from filebeat/module/system/audit/manifest.yml rename to filebeat/module/auditd/log/manifest.yml index d5f35f5ac9b..5c8bf0a9eea 100644 --- a/filebeat/module/system/audit/manifest.yml +++ b/filebeat/module/auditd/log/manifest.yml @@ -8,7 +8,7 @@ var: os.windows: [] ingest_pipeline: ingest/pipeline.json -prospector: config/audit.yml +prospector: config/log.yml requires.processors: - name: geoip diff --git a/filebeat/module/system/audit/test/audit-rhel6.log b/filebeat/module/auditd/log/test/audit-rhel6.log similarity index 100% rename from filebeat/module/system/audit/test/audit-rhel6.log rename to filebeat/module/auditd/log/test/audit-rhel6.log diff --git a/filebeat/module/system/audit/test/audit-rhel7.log b/filebeat/module/auditd/log/test/audit-rhel7.log similarity index 100% rename from filebeat/module/system/audit/test/audit-rhel7.log rename to filebeat/module/auditd/log/test/audit-rhel7.log diff --git a/filebeat/module/system/audit/test/test.log b/filebeat/module/auditd/log/test/test.log similarity index 100% rename from filebeat/module/system/audit/test/test.log rename to filebeat/module/auditd/log/test/test.log diff --git a/filebeat/module/system/audit/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json similarity index 89% rename from filebeat/module/system/audit/test/test.log-expected.json rename to filebeat/module/auditd/log/test/test.log-expected.json index 0223a88afdb..7c58ee1cf48 100644 --- a/filebeat/module/system/audit/test/test.log-expected.json +++ b/filebeat/module/auditd/log/test/test.log-expected.json @@ -8,16 +8,16 @@ "_source": { "offset": 172, "input_type": "log", - "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/system/audit/test/test.log", + "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", "fileset": { - "module": "system", - "name": "audit" + "module": "auditd", + "name": "log" }, "type": "log", "error": "", "@timestamp": "2017-01-31T20:18:46.912Z", - "system": { - "audit": { + "auditd": { + "log": { "ses": "4294967295", "op": "SPD-delete", "res": "1", @@ -46,16 +46,16 @@ "_source": { "offset": 534, "input_type": "log", - "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/system/audit/test/test.log", + "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", "fileset": { - "module": "system", - "name": "audit" + "module": "auditd", + "name": "log" }, "type": "log", "error": "", "@timestamp": "2017-01-31T20:18:46.912Z", - "system": { - "audit": { + "auditd": { + "log": { "syscall": "44", "gid": "0", "fsgid": "0", diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc index 5c538fb3b16..a0d4d813e7a 100644 --- a/filebeat/module/system/_meta/docs.asciidoc +++ b/filebeat/module/system/_meta/docs.asciidoc @@ -14,15 +14,10 @@ This module is not available for Windows. [float] === Dashboard -This module comes with sample dashboards. The first is a more generic syslog -dashboard that shows syslog data. +This module comes with a sample dashboard showing syslog data. image::./images/kibana-system.png[] -And second is the audit dashboard that shows audit log data. - -image::./images/kibana-system-audit.png[] - [float] === Syslog fileset settings diff --git a/filebeat/module/system/audit/_meta/kibana/visualization/2bb0fa70-0a11-11e7-9e84-43da493ad0c7.json b/filebeat/module/system/audit/_meta/kibana/visualization/2bb0fa70-0a11-11e7-9e84-43da493ad0c7.json deleted file mode 100644 index a99239f1873..00000000000 --- a/filebeat/module/system/audit/_meta/kibana/visualization/2bb0fa70-0a11-11e7-9e84-43da493ad0c7.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "visState": "{\"type\":\"timelion\",\"title\":\"Audit Event Results\",\"params\":{\"expression\":\".es(q=\\\"_exists_:system.audit NOT system.audit.res:failure\\\").label(\\\"Success\\\") .es(q=\\\"system.audit.res:failed\\\").label(\\\"Failure\\\").title(\\\"Audit Event Results\\\")\",\"interval\":\"auto\"}}", - "description": "", - "title": "Audit Event Results", - "uiStateJSON": "{}", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{}" - } -} \ No newline at end of file diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 3dd4b35bb0a..0f95e2e152c 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -104,7 +104,7 @@ def run_on_file(self, module, fileset, test_file, cfgfile): # TODO: There are errors parsing the test logs from these modules. assert "error" not in obj - if module != "system" and fileset != "audit": + if module != "auditd" and fileset != "log": # There are dynamic fields in audit logs that are not documented. self.assert_fields_are_documented(obj)