[Filebeat] Create ActiveMQ module (elastic#14840)

* Enable audit logs in ActiveMQ module

* Generate module stub

* Update configuration

* Add log files

* Add sample exception to ActiveMQ log

* Define ingest pipeline for ActiveMQ audit logs

* Define ingest pipeline for ActiveMQ logs

* Adjust ingest pipelines, bugfixing

* Do not check timestamp for audit logs

* Refactor fields

* Fix: mage fmt update

* Adjust fields after review

* Rename action to message to properly present logs in Kibana

* Refactor fields according to the review

filebeat/docs/fields.asciidoc

@@ -12,6 +12,7 @@ This file is generated! See _meta/fields.yml and scripts/generate_fields_docs.py
 This document describes the fields that are exported by Filebeat. They are
 grouped in the following categories:
 
+* <<exported-fields-activemq>>
 * <<exported-fields-apache>>
 * <<exported-fields-auditd>>
 * <<exported-fields-aws>>
@@ -61,6 +62,69 @@ grouped in the following categories:
 * <<exported-fields-zeek>>
 
 --
+[[exported-fields-activemq]]
+== activemq fields
+
+Module for parsing ActiveMQ log files.
+
+
+
+[float]
+=== activemq
+
+
+
+
+*`activemq.caller`*::
++
+--
+Name of the caller issuing the logging request (class or resource).
+
+
+type: keyword
+
+--
+
+*`activemq.thread`*::
++
+--
+Thread that generated the logging event.
+
+
+type: keyword
+
+--
+
+*`activemq.user`*::
++
+--
+User that generated the logging event.
+
+
+type: keyword
+
+--
+
+[float]
+=== audit
+
+Fields from ActiveMQ audit logs.
+
+
+[float]
+=== log
+
+Fields from ActiveMQ application logs.
+
+
+
+*`activemq.log.stack_trace`*::
++
+--
+type: keyword
+
+--
+
 [[exported-fields-apache]]
 == Apache fields
 

filebeat/docs/modules/activemq.asciidoc

@@ -0,0 +1,69 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-activemq]]
+:modulename: activemq
+:has-dashboards: false
+
+== ActiveMQ module
+
+This module parses Apache ActiveMQ logs. It supports application and audit logs.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+The module has been tested with ActiveMQ 5.13.0 and 5.15.9. Other versions are expected to work.
+
+include::../include/running-modules.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for ActiveMQ logs:
+
+["source","yaml",subs="attributes"]
+-----
+- module: activemq
+  audit:
+    enabled: true
+    var.paths: ["/path/to/log/activemq/data/audit.log*"]
+  log:
+    enabled: true
+    var.paths: ["/path/to/log/activemq/data/activemq.log*"]
+-----
+
+To specify the same settings at the command line, you use:
+
+["source","sh",subs="attributes"]
+-----
+-M "activemq.audit.var.paths=[/path/to/log/activemq/data/audit.log*]"
+-M "activemq.log.var.paths=[/path/to/log/activemq/data/activemq.log*]"
+-----
+
+[float]
+==== `log` log fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+include::../include/timezone-support.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-activemq,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -3,6 +3,7 @@ This file is generated! See scripts/docs_collector.py
 ////
 
   * <<filebeat-modules-overview>>
+  * <<filebeat-module-activemq>>
   * <<filebeat-module-apache>>
   * <<filebeat-module-auditd>>
   * <<filebeat-module-aws>>
@@ -43,6 +44,7 @@ This file is generated! See scripts/docs_collector.py
 --
 
 include::modules-overview.asciidoc[]
+include::modules/activemq.asciidoc[]
 include::modules/apache.asciidoc[]
 include::modules/auditd.asciidoc[]
 include::modules/aws.asciidoc[]

filebeat/tests/system/test_modules.py

@@ -224,7 +224,8 @@ def clean_keys(obj):
     # ECS versions change for any ECS release, large or small
     ecs_key = ["ecs.version"]
     # datasets for which @timestamp is removed due to date missing
-    remove_timestamp = {"icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog", "cef.log"}
+    remove_timestamp = {"icinga.startup", "redis.log", "haproxy.log",
+                        "system.auth", "system.syslog", "cef.log", "activemq.audit"}
     # dataset + log file pairs for which @timestamp is kept as an exception from above
     remove_timestamp_exception = {
         ('system.syslog', 'tz-offset.log'),

x-pack/filebeat/filebeat.reference.yml

@@ -37,6 +37,24 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+#------------------------------- Activemq Module -------------------------------
+- module: activemq
+  # Audit logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Application logs
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #-------------------------------- Apache Module --------------------------------
 #- module: apache
   # Access logs

x-pack/filebeat/module/activemq/_meta/config.yml

@@ -0,0 +1,16 @@
+- module: activemq
+  # Audit logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Application logs
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

x-pack/filebeat/module/activemq/_meta/docs.asciidoc

@@ -0,0 +1,56 @@
+:modulename: activemq
+:has-dashboards: false
+
+== ActiveMQ module
+
+This module parses Apache ActiveMQ logs. It supports application and audit logs.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+The module has been tested with ActiveMQ 5.13.0 and 5.15.9. Other versions are expected to work.
+
+include::../include/running-modules.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for ActiveMQ logs:
+
+["source","yaml",subs="attributes"]
+-----
+- module: activemq
+  audit:
+    enabled: true
+    var.paths: ["/path/to/log/activemq/data/audit.log*"]
+  log:
+    enabled: true
+    var.paths: ["/path/to/log/activemq/data/activemq.log*"]
+-----
+
+To specify the same settings at the command line, you use:
+
+["source","sh",subs="attributes"]
+-----
+-M "activemq.audit.var.paths=[/path/to/log/activemq/data/audit.log*]"
+-M "activemq.log.var.paths=[/path/to/log/activemq/data/activemq.log*]"
+-----
+
+[float]
+==== `log` log fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+include::../include/timezone-support.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:

x-pack/filebeat/module/activemq/_meta/fields.yml

@@ -0,0 +1,21 @@
+- key: activemq
+  title: "activemq"
+  description: >
+    Module for parsing ActiveMQ log files.
+  fields:
+    - name: activemq
+      type: group
+      description: >
+      fields:
+        - name: caller
+          type: keyword
+          description: >
+            Name of the caller issuing the logging request (class or resource).
+        - name: thread
+          type: keyword
+          description: >
+            Thread that generated the logging event.
+        - name: user
+          type: keyword
+          description: >
+            User that generated the logging event.

x-pack/filebeat/module/activemq/audit/_meta/fields.yml

@@ -0,0 +1,5 @@
+- name: audit
+  type: group
+  description: >
+    Fields from ActiveMQ audit logs.
+  fields:

x-pack/filebeat/module/activemq/audit/config/audit.yml

@@ -0,0 +1,6 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]

x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml

@@ -0,0 +1,15 @@
+---
+description: Pipeline for parsing ActiveMQ audit logs.
+processors:
+  - grok:
+      field: message
+      pattern_definitions:
+        NOPIPEGREEDYDATA: "((?! \\|).)*"
+        THREAD_NAME: "((?! \n).)*"
+      patterns:
+        - "%{LOGLEVEL:log.level}%{SPACE}\\|%{SPACE}%{WORD:activemq.user}%{SPACE}%{NOPIPEGREEDYDATA:message}%{SPACE}\\|%{SPACE}%{THREAD_NAME:activemq.thread}"
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

x-pack/filebeat/module/activemq/audit/manifest.yml

@@ -0,0 +1,13 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /opt/apache-activemq-*/data/audit.log*
+    os.darwin:
+      - /usr/local/apache-activemq-*/data/audit.log*
+    os.windows:
+      - c:/apache-activemq-*/data/audit.log*
+
+ingest_pipeline: ingest/pipeline.yml
+input: config/audit.yml

x-pack/filebeat/module/activemq/audit/test/audit.log

@@ -0,0 +1,4 @@
+INFO  | anonymous called org.apache.activemq.broker.jmx.QueueView.retryMessages[] at 27-11-2019 08:45:57,213 | qtp443290224-47
+INFO  | admin called org.apache.activemq.broker.jmx.QueueView.retryMessages[] at 27-11-2019 08:45:57,229 | qtp443290224-45
+WARN  | admin requested /admin/createDestination.action [JMSDestination='test' JMSDestinationType='queue' secret='4eb0bc3e-9d7a-4256-844c-24f40fda98f1' ] from 127.0.0.1 | qtp12205619-39
+INFO  | guest requested /admin/purgeDestination.action [JMSDestination='test' JMSDestinationType='queue' secret='eff6a932-1b58-45da-a64a-1b30b246cfc9' ] from 127.0.0.1 | qtp12205619-36