Initial add of auditbeat (#4529)

Migrate audit module from Metricbeat to Auditbeat. This is a straight-forward migration
of the code from Metricbeat into its own Beat. This new Beat is fully based on the
Metricbeat framework (this may change in the future).

.travis.yml

@@ -43,6 +43,12 @@ jobs:
       go: $GO_VERSION
       stage: test
 
+    # Auditbeat
+    - os: linux
+      env: TARGETS="-C auditbeat testsuite"
+      go: $GO_VERSION
+      stage: test
+
     # Libbeat
     - os: linux
       env: TARGETS="-C libbeat testsuite"
@@ -127,6 +133,7 @@ notifications:
 
 after_success:
   # Copy full.cov to coverage.txt because codecov.io requires this file
+  - test -f auditbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f auditbeat/build/coverage/full.cov
   - test -f filebeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f filebeat/build/coverage/full.cov
   - test -f heartbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f heartbeat/build/coverage/full.cov
   - test -f libbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f libbeat/build/coverage/full.cov

Makefile

@@ -1,7 +1,7 @@
 
 BUILD_DIR=build
 COVERAGE_DIR=$(BUILD_DIR)/coverage
-BEATS=packetbeat filebeat winlogbeat metricbeat heartbeat
+BEATS=packetbeat filebeat winlogbeat metricbeat heartbeat auditbeat
 PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 SNAPSHOT?=yes

auditbeat/.gitignore

@@ -0,0 +1,9 @@
+build
+_meta/kibana
+_meta/beat.yml
+_meta/beat.full.yml
+
+/auditbeat
+/auditbeat.test
+/docs/html_docs
+

auditbeat/Makefile

@@ -0,0 +1,47 @@
+BEAT_NAME=auditbeat
+BEAT_DESCRIPTION=Audit the activities of users and processes on your system.
+SYSTEM_TESTS=false
+TEST_ENVIRONMENT=false
+
+# Path to the libbeat Makefile
+-include ../libbeat/scripts/Makefile
+
+# This is called by the beats packer before building starts
+.PHONY: before-build
+before-build:
+
+# Collects all dependencies and then calls update
+.PHONY: collect
+collect: fields collect-docs configs kibana
+
+# Collects all module and metricset fields
+.PHONY: fields
+fields: python-env
+	@mkdir -p _meta
+	@cp ${ES_BEATS}/metricbeat/_meta/fields.common.yml _meta/fields.generated.yml
+	@${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml
+
+# Collects all module configs
+.PHONY: configs
+configs: python-env
+	@cp ${ES_BEATS}/auditbeat/_meta/common.yml _meta/beat.yml
+	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml
+	@cp ${ES_BEATS}/auditbeat/_meta/common.full.yml _meta/beat.full.yml
+	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.full.yml
+
+# Collects all module docs
+.PHONY: collect-docs
+collect-docs: python-env
+	@-rm -rf docs/modules
+	@mkdir -p docs/modules
+	@${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/docs_collector.py --beat ${BEAT_NAME}
+
+# Collects all module dashboards
+.PHONY: kibana
+kibana:
+	# To not remove index-pattern as generated by update
+	@-rm -r _meta/kibana/dashboard _meta/kibana/search _meta/kibana/visualization
+	@mkdir -p _meta/kibana
+	@-cp -pr module/*/_meta/kibana _meta/
+
+

auditbeat/_meta/beat.yml

@@ -0,0 +1,24 @@
+###################### Auditbeat Configuration Example #######################
+
+# This file is an example configuration file highlighting only the most common
+# options. The auditbeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/auditbeat/index.html
+
+
+#==========================  Modules configuration ============================
+auditbeat.modules:
+
+#-------------------------------- Audit Module -------------------------------
+- module: audit
+  metricsets: ["kernel"]
+  kernel.audit_rules:
+    # Define audit rules here.
+    # Create file watches (-w) or syscall audits (-a or -A). For example:
+    #-w /etc/passwd -p wa -k identity
+    #-a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
+
+
+

auditbeat/_meta/common.full.yml

@@ -0,0 +1,27 @@
+########################## Auditbeat Configuration ###########################
+
+# This file is a full configuration example documenting all non-deprecated
+# options in comments. For a shorter configuration example, that contains only
+# the most common options, please see auditbeat.yml in the same directory.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/auditbeat/index.html
+
+#============================  Config Reloading ===============================
+
+# Config reloading allows to dynamically load modules. Each file which is
+# monitored must contain one or multiple modules as a list.
+auditbeat.config.modules:
+
+  # Glob pattern for configuration reloading
+  path: ${path.config}/conf.d/*.yml
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+# Maximum amount of time to randomly delay the start of a metricset. Use 0 to
+# disable startup delay.
+auditbeat.max_start_delay: 10s

auditbeat/_meta/common.yml

@@ -0,0 +1,9 @@
+###################### Auditbeat Configuration Example #######################
+
+# This file is an example configuration file highlighting only the most common
+# options. The auditbeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/auditbeat/index.html
+

auditbeat/_meta/fields.common.yml

@@ -0,0 +1,36 @@
+- key: common
+  title: Common
+  description: >
+    Contains common fields available in all event types.
+  fields:
+
+    - name: metricset.module
+      description: >
+        The name of the module that generated the event.
+
+    - name: metricset.name
+      description: >
+        The name of the metricset that generated the event.
+
+    - name: metricset.host
+      description: >
+        Hostname of the machine from which the metricset was collected. This
+        field may not be present when the data was collected locally.
+
+    - name: metricset.rtt
+      type: long
+      required: true
+      description: >
+        Event round trip time in microseconds.
+
+    - name: metricset.namespace
+      type: keyword
+      description: >
+        Namespace of dynamic metricsets.
+
+    - name: type
+      required: true
+      example: metricsets
+      description: >
+        The document type. Always set to "metricsets".
+