Auto generate modules list (#3131)

This is to ensure no modules are forgotten in the future

metricbeat/docs/modules.asciidoc

@@ -7,28 +7,7 @@ This section contains detailed information about the metric collecting modules
 contained in {beatname_uc}. Each module contains one or multiple metricsets. More details
 about each module can be found under the links below.
 
-  * <<metricbeat-module-apache,Apache>>
-  * <<metricbeat-module-docker,Docker>>
-  * <<metricbeat-module-haproxy,HAProxy>>
-  * <<metricbeat-module-mongodb,MongoDB>>
-  * <<metricbeat-module-mysql,MySQL>>
-  * <<metricbeat-module-nginx,Nginx>>
-  * <<metricbeat-module-redis,Redis>>
-  * <<metricbeat-module-postgresql,Postgresql>>
-  * <<metricbeat-module-system,System>>
-  * <<metricbeat-module-zookeeper,ZooKeeper>>
+include::modules_list.asciidoc[]
 
---
-
-include::modules/apache.asciidoc[]
-include::modules/docker.asciidoc[]
-include::modules/haproxy.asciidoc[]
-include::modules/mongodb.asciidoc[]
-include::modules/mysql.asciidoc[]
-include::modules/nginx.asciidoc[]
-include::modules/postgresql.asciidoc[]
-include::modules/redis.asciidoc[]
-include::modules/system.asciidoc[]
-include::modules/zookeeper.asciidoc[]
 
 

metricbeat/docs/modules_list.asciidoc

@@ -0,0 +1,30 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+  * <<metricbeat-module-apache,Apache>>
+  * <<metricbeat-module-docker,docker>>
+  * <<metricbeat-module-haproxy,haproxy>>
+  * <<metricbeat-module-kafka,kafka>>
+  * <<metricbeat-module-mongodb,MongoDB>>
+  * <<metricbeat-module-mysql,MySQL>>
+  * <<metricbeat-module-nginx,Nginx>>
+  * <<metricbeat-module-postgresql,PostgreSQL>>
+  * <<metricbeat-module-redis,Redis>>
+  * <<metricbeat-module-system,System>>
+  * <<metricbeat-module-zookeeper,ZooKeeper>>
+
+
+--
+
+include::modules/apache.asciidoc[]
+include::modules/docker.asciidoc[]
+include::modules/haproxy.asciidoc[]
+include::modules/kafka.asciidoc[]
+include::modules/mongodb.asciidoc[]
+include::modules/mysql.asciidoc[]
+include::modules/nginx.asciidoc[]
+include::modules/postgresql.asciidoc[]
+include::modules/redis.asciidoc[]
+include::modules/system.asciidoc[]
+include::modules/zookeeper.asciidoc[]

metricbeat/scripts/docs_collector.py

@@ -16,6 +16,8 @@ def collect(beat_name):
 
 """
 
+    modules_list = {}
+
     # Iterate over all modules
     for module in sorted(os.listdir(base_dir)):
 
@@ -41,6 +43,8 @@ def collect(beat_name):
             fields = yaml.load(f.read())
             title = fields[0]["title"]
 
+        modules_list[module] = title
+
         config_file = beat_path + "/config.yml"
 
         # Add example config file
@@ -131,6 +135,19 @@ def collect(beat_name):
         with open(os.path.abspath("docs") + "/modules/" + module + ".asciidoc", 'w') as f:
             f.write(module_file)
 
+    module_list_output = generated_note
+    for m, title in sorted(modules_list.iteritems()):
+        module_list_output += "  * <<metricbeat-module-" + m + "," + title + ">>\n"
+
+    module_list_output += "\n\n--\n\n"
+    for m, title in sorted(modules_list.iteritems()):
+        module_list_output += "include::modules/"+ m + ".asciidoc[]\n"
+
+    # Write module link list
+    with open(os.path.abspath("docs") + "/modules_list.asciidoc", 'w') as f:
+        f.write(module_list_output)
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Collects modules docs")