diff --git a/filebeat/Makefile b/filebeat/Makefile index d771b206816b..384755b89c74 100644 --- a/filebeat/Makefile +++ b/filebeat/Makefile @@ -6,6 +6,8 @@ GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le" ES_BEATS?=.. FIELDS_FILE_PATH=module +DOCS_BRANCH=$(shell grep doc-branch ../libbeat/docs/version.asciidoc | cut -c 14-) + include ${ES_BEATS}/libbeat/scripts/Makefile # Collects all module dashboards @@ -23,8 +25,8 @@ configs: python-env @cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml @cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml - @rm -rf modules.d && mkdir -p modules.d - @for MODULE in `find module -type d -maxdepth 1 -mindepth 1 -exec basename {} \;`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done + @rm -rf modules.d + ${PYTHON_ENV}/bin/python ${ES_BEATS}/script/modules_collector.py --beat ${BEAT_NAME} --docs_branch=$(DOCS_BRANCH) @chmod go-w modules.d/* # Collects all module docs diff --git a/filebeat/modules.d/apache2.yml.disabled b/filebeat/modules.d/apache2.yml.disabled index 6a545ad3645b..9fa8f2146677 100644 --- a/filebeat/modules.d/apache2.yml.disabled +++ b/filebeat/modules.d/apache2.yml.disabled @@ -1,3 +1,6 @@ +# Module: apache2 +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-apache2.html + - module: apache2 # Access logs access: diff --git a/filebeat/modules.d/auditd.yml.disabled b/filebeat/modules.d/auditd.yml.disabled index bd952f49cc9d..4b0bd49c6f65 100644 --- a/filebeat/modules.d/auditd.yml.disabled +++ b/filebeat/modules.d/auditd.yml.disabled @@ -1,3 +1,6 @@ +# Module: auditd +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-auditd.html + - module: auditd log: enabled: true diff --git a/filebeat/modules.d/elasticsearch.yml.disabled b/filebeat/modules.d/elasticsearch.yml.disabled index 0c2562f27969..4db2df4eaea3 100644 --- a/filebeat/modules.d/elasticsearch.yml.disabled +++ b/filebeat/modules.d/elasticsearch.yml.disabled @@ -1,3 +1,6 @@ +# Module: elasticsearch +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-elasticsearch.html + - module: elasticsearch # Server log server: diff --git a/filebeat/modules.d/haproxy.yml.disabled b/filebeat/modules.d/haproxy.yml.disabled index 0e1431e503c0..7493d93d7633 100644 --- a/filebeat/modules.d/haproxy.yml.disabled +++ b/filebeat/modules.d/haproxy.yml.disabled @@ -1,3 +1,6 @@ +# Module: haproxy +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-haproxy.html + - module: haproxy # All logs log: diff --git a/filebeat/modules.d/icinga.yml.disabled b/filebeat/modules.d/icinga.yml.disabled index afcd57986a2d..2b136d520728 100644 --- a/filebeat/modules.d/icinga.yml.disabled +++ b/filebeat/modules.d/icinga.yml.disabled @@ -1,3 +1,6 @@ +# Module: icinga +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-icinga.html + - module: icinga # Main logs main: diff --git a/filebeat/modules.d/iis.yml.disabled b/filebeat/modules.d/iis.yml.disabled index 0ed84f14e52d..3fb8768b3911 100644 --- a/filebeat/modules.d/iis.yml.disabled +++ b/filebeat/modules.d/iis.yml.disabled @@ -1,3 +1,6 @@ +# Module: iis +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-iis.html + - module: iis # Access logs access: diff --git a/filebeat/modules.d/kafka.yml.disabled b/filebeat/modules.d/kafka.yml.disabled index 9193a87a5fc9..ec6b533399b1 100644 --- a/filebeat/modules.d/kafka.yml.disabled +++ b/filebeat/modules.d/kafka.yml.disabled @@ -1,3 +1,6 @@ +# Module: kafka +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-kafka.html + - module: kafka # All logs log: diff --git a/filebeat/modules.d/kibana.yml.disabled b/filebeat/modules.d/kibana.yml.disabled index 176a9b263bc4..beef9232b9f5 100644 --- a/filebeat/modules.d/kibana.yml.disabled +++ b/filebeat/modules.d/kibana.yml.disabled @@ -1,3 +1,6 @@ +# Module: kibana +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-kibana.html + - module: kibana # All logs log: diff --git a/filebeat/modules.d/logstash.yml.disabled b/filebeat/modules.d/logstash.yml.disabled index acacaaafa892..53091a8a411c 100644 --- a/filebeat/modules.d/logstash.yml.disabled +++ b/filebeat/modules.d/logstash.yml.disabled @@ -1,3 +1,6 @@ +# Module: logstash +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-logstash.html + - module: logstash # logs log: diff --git a/filebeat/modules.d/mongodb.yml.disabled b/filebeat/modules.d/mongodb.yml.disabled index be6ea989c1c2..36745bca4196 100644 --- a/filebeat/modules.d/mongodb.yml.disabled +++ b/filebeat/modules.d/mongodb.yml.disabled @@ -1,3 +1,6 @@ +# Module: mongodb +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-mongodb.html + - module: mongodb # All logs log: diff --git a/filebeat/modules.d/mysql.yml.disabled b/filebeat/modules.d/mysql.yml.disabled index 10afcb9e0ab6..a7904e69f1b0 100644 --- a/filebeat/modules.d/mysql.yml.disabled +++ b/filebeat/modules.d/mysql.yml.disabled @@ -1,3 +1,6 @@ +# Module: mysql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-mysql.html + - module: mysql # Error logs error: diff --git a/filebeat/modules.d/nginx.yml.disabled b/filebeat/modules.d/nginx.yml.disabled index 22fe90e486e4..cc77a6b34596 100644 --- a/filebeat/modules.d/nginx.yml.disabled +++ b/filebeat/modules.d/nginx.yml.disabled @@ -1,3 +1,6 @@ +# Module: nginx +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-nginx.html + - module: nginx # Access logs access: diff --git a/filebeat/modules.d/osquery.yml.disabled b/filebeat/modules.d/osquery.yml.disabled index b2a86b43c679..1c66965bfe94 100644 --- a/filebeat/modules.d/osquery.yml.disabled +++ b/filebeat/modules.d/osquery.yml.disabled @@ -1,3 +1,6 @@ +# Module: osquery +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-osquery.html + - module: osquery result: enabled: true diff --git a/filebeat/modules.d/postgresql.yml.disabled b/filebeat/modules.d/postgresql.yml.disabled index c82734a9570f..1e01709d02cf 100644 --- a/filebeat/modules.d/postgresql.yml.disabled +++ b/filebeat/modules.d/postgresql.yml.disabled @@ -1,3 +1,6 @@ +# Module: postgresql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-postgresql.html + - module: postgresql # All logs log: diff --git a/filebeat/modules.d/redis.yml.disabled b/filebeat/modules.d/redis.yml.disabled index 4aa2f1eacf0f..6a43828abfec 100644 --- a/filebeat/modules.d/redis.yml.disabled +++ b/filebeat/modules.d/redis.yml.disabled @@ -1,3 +1,6 @@ +# Module: redis +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-redis.html + - module: redis # Main logs log: diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled index 34d3bb35df23..32ddd6b6d9cc 100644 --- a/filebeat/modules.d/system.yml.disabled +++ b/filebeat/modules.d/system.yml.disabled @@ -1,3 +1,6 @@ +# Module: system +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-system.html + - module: system # Syslog syslog: diff --git a/filebeat/modules.d/traefik.yml.disabled b/filebeat/modules.d/traefik.yml.disabled index 16ec37f975e1..22e6cdf0dc84 100644 --- a/filebeat/modules.d/traefik.yml.disabled +++ b/filebeat/modules.d/traefik.yml.disabled @@ -1,3 +1,6 @@ +# Module: traefik +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-traefik.html + - module: traefik # Access logs access: diff --git a/metricbeat/Makefile b/metricbeat/Makefile index 0a4dc962a0e3..75e2e2967d0f 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -36,7 +36,7 @@ configs: python-env @cat ${ES_BEATS}/metricbeat/_meta/common.reference.yml > _meta/beat.reference.yml @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml @rm -rf modules.d - ${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/modules_collector.py --docs_branch=$(DOCS_BRANCH) + ${PYTHON_ENV}/bin/python ${ES_BEATS}/script/modules_collector.py --beat ${BEAT_NAME} --docs_branch=$(DOCS_BRANCH) @chmod go-w modules.d/* @# Enable system by default: @if [ -f modules.d/system.yml.disabled ]; then mv modules.d/system.yml.disabled modules.d/system.yml; fi diff --git a/metricbeat/scripts/modules_collector.py b/metricbeat/scripts/modules_collector.py deleted file mode 100644 index 0ee891a8bcc2..000000000000 --- a/metricbeat/scripts/modules_collector.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import argparse -import yaml -import six - -# Collects module configs to modules.d - - -def collect(docs_branch): - - base_dir = "module" - path = os.path.abspath("module") - - # TODO add module release status if beta or experimental - header = """# Module: {module} -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/{docs_branch}/metricbeat-module-{module}.html - -""" - - # Create directory for module confs - os.mkdir(os.path.abspath('modules.d')) - - # Iterate over all modules - for module in sorted(os.listdir(base_dir)): - - module_conf = path + '/' + module + '/_meta/config.yml' - if os.path.isfile(module_conf) == False: - continue - - module_file = header.format(module=module, docs_branch=docs_branch) - - with open(module_conf) as f: - module_file += f.read() - - # Write disabled module conf - with open(os.path.abspath('modules.d/') + '/' + module + '.yml.disabled', 'w') as f: - f.write(module_file) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Collects modules confs") - parser.add_argument("--docs_branch", help="Docs branch") - - args = parser.parse_args() - docs_branch = args.docs_branch - - collect(docs_branch) diff --git a/script/modules_collector.py b/script/modules_collector.py new file mode 100644 index 000000000000..51b38980666a --- /dev/null +++ b/script/modules_collector.py @@ -0,0 +1,61 @@ +import os +import argparse +import yaml +import six +import glob +import re + +# Collects module configs to modules.d + +REFERENCE_CONFIG_RE = re.compile('.+\.reference\.yml') + + +def collect(beat_name, docs_branch): + + base_dir = "module" + path = os.path.abspath("module") + + # TODO add module release status if beta or experimental + header = """# Module: {module} +# Docs: https://www.elastic.co/guide/en/beats/{beat_name}/{docs_branch}/{beat_name}-module-{module}.html + +""" + + # Create directory for module confs + os.mkdir(os.path.abspath('modules.d')) + + # Iterate over all modules + for module in sorted(os.listdir(base_dir)): + + module_confs = path + '/' + module + '/_meta/config*.yml' + for module_conf in glob.glob(module_confs): + + # Ignore reference confs + if REFERENCE_CONFIG_RE.match(module_conf): + continue + + if os.path.isfile(module_conf) == False: + continue + + module_file = header.format(module=module, beat_name=beat_name, docs_branch=docs_branch) + disabled_config_filename = os.path.basename(module_conf).replace('config', module) + '.disabled' + + with open(module_conf) as f: + module_file += f.read() + + # Write disabled module conf + with open(os.path.abspath('modules.d/') + '/' + disabled_config_filename, 'w') as f: + f.write(module_file) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Collects modules confs") + parser.add_argument("--beat", help="Beat name") + parser.add_argument("--docs_branch", help="Docs branch") + + args = parser.parse_args() + beat_name = args.beat + docs_branch = args.docs_branch + + collect(beat_name, docs_branch)