Update docs and test files regarding Kafka 2.0 support in filebeat (#…

…8853) (#8871)

(cherry picked from commit d81bd7c)

CHANGELOG.asciidoc

@@ -134,6 +134,7 @@ https://github.com/elastic/beats/compare/v6.4.0...6.x[Check the HEAD diff]
 - Added default and TCP parsing formats to HAproxy module {issue}8311[8311] {pull}8637[8637]
 - Allow to force CRI format parsing for better performance {pull}8424[8424]
 - Add Suricata IDS/IDP/NSM module. {issue}8153[8153] {pull}8693[8693]
+- Support for Kafka 2.0.0 {pull}8853[8853]
 
 *Heartbeat*
 

filebeat/docs/modules/kafka.asciidoc

@@ -16,7 +16,7 @@ include::../include/what-happens.asciidoc[]
 [float]
 === Compatibility
 
-The +{modulename}+ module was tested with logs from versions 0.9 and 1.11.
+The +{modulename}+ module was tested with logs from versions 0.9, 1.1.0 and 2.0.0.
 
 include::../include/running-modules.asciidoc[]
 

filebeat/module/kafka/_meta/docs.asciidoc

@@ -11,7 +11,7 @@ include::../include/what-happens.asciidoc[]
 [float]
 === Compatibility
 
-The +{modulename}+ module was tested with logs from versions 0.9 and 1.11.
+The +{modulename}+ module was tested with logs from versions 0.9, 1.1.0 and 2.0.0.
 
 include::../include/running-modules.asciidoc[]
 

filebeat/module/kafka/log/test/controller-2.0.0.log

@@ -0,0 +1,23 @@
+[2018-10-31 15:03:32,474] TRACE [Controller id=10] Checking need to trigger auto leader balancing (kafka.controller.KafkaController)
+[2018-10-31 15:03:32,474] DEBUG [Controller id=10] Preferred replicas by broker Map(20 -> Map(__consumer_offsets-22 -> Vector(20), __consumer_offsets-4 -> Vector(20), __consumer_offsets-7 -> Vector(20), __consumer_offsets-46 -> Vector(20), __consumer_offsets-25 -> Vector(20), __consumer_offsets-49 -> Vector(20), __consumer_offsets-16 -> Vector(20), test-0 -> Vector(20, 30, 10), __consumer_offsets-28 -> Vector(20), __consumer_offsets-31 -> Vector(20), test-2-2 -> Vector(20, 30), __consumer_offsets-37 -> Vector(20), filebeat-system-0 -> Vector(20), test-3-3 -> Vector(20, 30), __consumer_offsets-19 -> Vector(20), __consumer_offsets-13 -> Vector(20), __consumer_offsets-43 -> Vector(20), __consumer_offsets-1 -> Vector(20), __consumer_offsets-34 -> Vector(20), __consumer_offsets-10 -> Vector(20), test-3 -> Vector(20, 10, 30), __consumer_offsets-40 -> Vector(20)), 10 -> Map(__consumer_offsets-30 -> Vector(10), __consumer_offsets-21 -> Vector(10), __consumer_offsets-27 -> Vector(10), __consumer_offsets-9 -> Vector(10), __consumer_offsets-33 -> Vector(10), __consumer_offsets-36 -> Vector(10), __consumer_offsets-42 -> Vector(10), __consumer_offsets-3 -> Vector(10), __consumer_offsets-18 -> Vector(10), test-5 -> Vector(10, 30, 20), __consumer_offsets-15 -> Vector(10), __consumer_offsets-24 -> Vector(10), test-3-1 -> Vector(10, 20), __consumer_offsets-48 -> Vector(10), filebeat-kafka-0 -> Vector(10), __consumer_offsets-6 -> Vector(10), test-2-1 -> Vector(10, 20), test-3-2 -> Vector(10, 30), __consumer_offsets-0 -> Vector(10), __consumer_offsets-39 -> Vector(10), __consumer_offsets-12 -> Vector(10), __consumer_offsets-45 -> Vector(10), test-2 -> Vector(10, 20, 30)), 30 -> Map(__consumer_offsets-8 -> Vector(30), __consumer_offsets-35 -> Vector(30), __consumer_offsets-41 -> Vector(30), __consumer_offsets-23 -> Vector(30), __consumer_offsets-47 -> Vector(30), metricbeat-0 -> Vector(30), test-3-0 -> Vector(30, 10), metricbeat-kafka-0 -> Vector(30), filebeat-0 -> Vector(30), test-2-0 -> Vector(30, 10), __consumer_offsets-38 -> Vector(30), __consumer_offsets-17 -> Vector(30), test-1 -> Vector(30, 10, 20), test-2-3 -> Vector(30, 20), __consumer_offsets-11 -> Vector(30), __consumer_offsets-2 -> Vector(30), __consumer_offsets-14 -> Vector(30), test-4 -> Vector(30, 20, 10), metricbeat-system-0 -> Vector(30), __consumer_offsets-20 -> Vector(30), __consumer_offsets-44 -> Vector(30), __consumer_offsets-5 -> Vector(30), __consumer_offsets-26 -> Vector(30), __consumer_offsets-29 -> Vector(30), __consumer_offsets-32 -> Vector(30))) (kafka.controller.KafkaController)
+[2018-10-31 15:03:32,474] DEBUG [Controller id=10] Topics not in preferred replica for broker 20 Map() (kafka.controller.KafkaController)
+[2018-10-31 15:03:32,475] TRACE [Controller id=10] Leader imbalance ratio for broker 20 is 0.0 (kafka.controller.KafkaController)
+[2018-10-31 15:03:32,475] DEBUG [Controller id=10] Topics not in preferred replica for broker 10 Map(test-3-1 -> Vector(10, 20)) (kafka.controller.KafkaController)
+[2018-10-31 15:03:32,475] TRACE [Controller id=10] Leader imbalance ratio for broker 10 is 0.043478260869565216 (kafka.controller.KafkaController)
+[2018-10-31 15:03:32,475] DEBUG [Controller id=10] Topics not in preferred replica for broker 30 Map() (kafka.controller.KafkaController)
+[2018-10-31 15:03:32,475] TRACE [Controller id=10] Leader imbalance ratio for broker 30 is 0.0 (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] TRACE [Controller id=10] Checking need to trigger auto leader balancing (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] DEBUG [Controller id=10] Preferred replicas by broker Map(20 -> Map(__consumer_offsets-22 -> Vector(20), __consumer_offsets-4 -> Vector(20), __consumer_offsets-7 -> Vector(20), __consumer_offsets-46 -> Vector(20), __consumer_offsets-25 -> Vector(20), __consumer_offsets-49 -> Vector(20), __consumer_offsets-16 -> Vector(20), test-0 -> Vector(20, 30, 10), __consumer_offsets-28 -> Vector(20), __consumer_offsets-31 -> Vector(20), test-2-2 -> Vector(20, 30), __consumer_offsets-37 -> Vector(20), filebeat-system-0 -> Vector(20), test-3-3 -> Vector(20, 30), __consumer_offsets-19 -> Vector(20), __consumer_offsets-13 -> Vector(20), __consumer_offsets-43 -> Vector(20), __consumer_offsets-1 -> Vector(20), __consumer_offsets-34 -> Vector(20), __consumer_offsets-10 -> Vector(20), test-3 -> Vector(20, 10, 30), __consumer_offsets-40 -> Vector(20)), 10 -> Map(__consumer_offsets-30 -> Vector(10), __consumer_offsets-21 -> Vector(10), __consumer_offsets-27 -> Vector(10), __consumer_offsets-9 -> Vector(10), __consumer_offsets-33 -> Vector(10), __consumer_offsets-36 -> Vector(10), __consumer_offsets-42 -> Vector(10), __consumer_offsets-3 -> Vector(10), __consumer_offsets-18 -> Vector(10), test-5 -> Vector(10, 30, 20), __consumer_offsets-15 -> Vector(10), __consumer_offsets-24 -> Vector(10), test-3-1 -> Vector(10, 20), __consumer_offsets-48 -> Vector(10), filebeat-kafka-0 -> Vector(10), __consumer_offsets-6 -> Vector(10), test-2-1 -> Vector(10, 20), test-3-2 -> Vector(10, 30), __consumer_offsets-0 -> Vector(10), __consumer_offsets-39 -> Vector(10), __consumer_offsets-12 -> Vector(10), __consumer_offsets-45 -> Vector(10), test-2 -> Vector(10, 20, 30)), 30 -> Map(__consumer_offsets-8 -> Vector(30), __consumer_offsets-35 -> Vector(30), __consumer_offsets-41 -> Vector(30), __consumer_offsets-23 -> Vector(30), __consumer_offsets-47 -> Vector(30), metricbeat-0 -> Vector(30), test-3-0 -> Vector(30, 10), metricbeat-kafka-0 -> Vector(30), filebeat-0 -> Vector(30), test-2-0 -> Vector(30, 10), __consumer_offsets-38 -> Vector(30), __consumer_offsets-17 -> Vector(30), test-1 -> Vector(30, 10, 20), test-2-3 -> Vector(30, 20), __consumer_offsets-11 -> Vector(30), __consumer_offsets-2 -> Vector(30), __consumer_offsets-14 -> Vector(30), test-4 -> Vector(30, 20, 10), metricbeat-system-0 -> Vector(30), __consumer_offsets-20 -> Vector(30), __consumer_offsets-44 -> Vector(30), __consumer_offsets-5 -> Vector(30), __consumer_offsets-26 -> Vector(30), __consumer_offsets-29 -> Vector(30), __consumer_offsets-32 -> Vector(30))) (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] DEBUG [Controller id=10] Topics not in preferred replica for broker 20 Map() (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] TRACE [Controller id=10] Leader imbalance ratio for broker 20 is 0.0 (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] DEBUG [Controller id=10] Topics not in preferred replica for broker 10 Map(test-3-1 -> Vector(10, 20)) (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] TRACE [Controller id=10] Leader imbalance ratio for broker 10 is 0.043478260869565216 (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] DEBUG [Controller id=10] Topics not in preferred replica for broker 30 Map() (kafka.controller.KafkaController)
+[2018-10-31 15:08:32,475] TRACE [Controller id=10] Leader imbalance ratio for broker 30 is 0.0 (kafka.controller.KafkaController)
+[2018-10-31 15:09:30,306] INFO [Controller id=10] New topics: [Set(foo)], deleted topics: [Set()], new partition replica assignment [Map(foo-0 -> Vector(20))] (kafka.controller.KafkaController)
+[2018-10-31 15:09:30,307] INFO [Controller id=10] New partition creation callback for foo-0 (kafka.controller.KafkaController)
+[2018-10-31 15:09:30,396] INFO [RequestSendThread controllerId=10] Controller 10 connected to 10.122.220.20:9094 (id: 20 rack: null) for sending state change requests (kafka.controller.RequestSendThread)
+[2018-10-31 15:09:30,397] INFO [RequestSendThread controllerId=10] Controller 10 connected to 10.122.220.20:9093 (id: 10 rack: null) for sending state change requests (kafka.controller.RequestSendThread)
+[2018-10-31 15:09:30,396] INFO [RequestSendThread controllerId=10] Controller 10 connected to 10.122.220.20:9095 (id: 30 rack: null) for sending state change requests (kafka.controller.RequestSendThread)
+[2018-10-31 15:13:32,475] TRACE [Controller id=10] Checking need to trigger auto leader balancing (kafka.controller.KafkaController)
+