diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a6630e2b494..30ef8b0474c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -13,6 +13,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update to Golang 1.12.1. {pull}11330[11330] - Update to Golang 1.12.4. {pull}11782[11782] - Update to ECS 1.0.1. {pull}12284[12284] {pull}12317[12317] +- Default of output.kafka.metadata.full is set to false by now. This reduced the amount of metadata to be queried from a kafka cluster. {pull}12738[12738] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 95c31e50b7e..a0f2799142b 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -668,8 +668,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 823b172c4a0..38d123b40a6 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1372,8 +1372,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 008eb35636d..9525e30cff8 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -812,8 +812,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 42989dfd619..d29f6d5752f 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -608,8 +608,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl index e48ff516b80..5a297e39881 100644 --- a/libbeat/_meta/config.reference.yml.tmpl +++ b/libbeat/_meta/config.reference.yml.tmpl @@ -556,8 +556,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 4edb606ed2c..b039351bdc2 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -1142,7 +1142,7 @@ brokers, topics, partition, and active leaders to use for publishing. *`full`*:: Strategy to use when fetching metadata, when this option is `true`, the client will maintain a full set of metadata for all the available topics, if the this option is set to `false` it will only refresh the -metadata for the configured topics. The default is true. +metadata for the configured topics. The default is false. *`retry.max`*:: Total number of metadata update retries when cluster is in middle of leader election. The default is 3. diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 7a3d7208b08..e96f96ef7a5 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -93,7 +93,7 @@ func defaultConfig() kafkaConfig { Backoff: 250 * time.Millisecond, }, RefreshFreq: 10 * time.Minute, - Full: true, + Full: false, }, KeepAlive: 0, MaxMessageBytes: nil, // use library default diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4b1c9e50e47..92e69b9d08a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1326,8 +1326,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index e95ef3fd435..a35116ec654 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1045,8 +1045,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index cba009eed0f..67700af19ff 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -589,8 +589,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 4c1961fb2f4..253ab0f1d8e 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -719,8 +719,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index ac938746b1d..f2436f5d630 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1520,8 +1520,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ffac94653ad..a9222ac900d 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1421,8 +1421,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index ae1594907c2..3ba4221449a 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -601,8 +601,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1