From 12b3c5862b91793ddbd9ceaebc375e6c9df0ed93 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 30 Oct 2019 16:26:05 +0200 Subject: [PATCH 01/29] Add kafka borker light module Signed-off-by: chrismark --- metricbeat/module/kafka/broker/manifest.yml | 11 +++ metricbeat/module/kafka/module.yml | 4 ++ .../module/kafka/_meta/config.reference.yml | 26 +++++++ .../metricbeat/module/kafka/_meta/config.yml | 28 ++++++++ .../metricbeat/module/kafka/broker/data.json | 0 .../module/kafka/broker/manifest.yml | 69 +++++++++++++++++++ x-pack/metricbeat/module/kafka/module.yml | 3 + .../metricbeat/modules.d/kafka.yml.disabled | 30 ++++++++ 8 files changed, 171 insertions(+) create mode 100644 metricbeat/module/kafka/broker/manifest.yml create mode 100644 metricbeat/module/kafka/module.yml create mode 100644 x-pack/metricbeat/module/kafka/_meta/config.reference.yml create mode 100644 x-pack/metricbeat/module/kafka/_meta/config.yml create mode 100644 x-pack/metricbeat/module/kafka/broker/data.json create mode 100644 x-pack/metricbeat/module/kafka/broker/manifest.yml create mode 100644 x-pack/metricbeat/module/kafka/module.yml create mode 100644 x-pack/metricbeat/modules.d/kafka.yml.disabled diff --git a/metricbeat/module/kafka/broker/manifest.yml b/metricbeat/module/kafka/broker/manifest.yml new file mode 100644 index 000000000000..82a50a4c64cf --- /dev/null +++ b/metricbeat/module/kafka/broker/manifest.yml @@ -0,0 +1,11 @@ +default: true +input: + module: jolokia + metricset: jmx + defaults: + namespace: "broker" + jmx.mappings: + - mbean: 'kafka.network:type=RequestChannel,name=RequestQueueSize' + attributes: + - attr: Value + field: request.channel.queue.size diff --git a/metricbeat/module/kafka/module.yml b/metricbeat/module/kafka/module.yml new file mode 100644 index 000000000000..5fe49e5bdf08 --- /dev/null +++ b/metricbeat/module/kafka/module.yml @@ -0,0 +1,4 @@ +name: aws +metricsets: + - elb + - ebs diff --git a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml new file mode 100644 index 000000000000..4fbf077d9f9a --- /dev/null +++ b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml @@ -0,0 +1,26 @@ +- module: kafka + metricsets: ["broker", "consumergroup", "partition"] + period: 10s + hosts: ["localhost:9092"] + enabled: true + + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" diff --git a/x-pack/metricbeat/module/kafka/_meta/config.yml b/x-pack/metricbeat/module/kafka/_meta/config.yml new file mode 100644 index 000000000000..acef2c0a8256 --- /dev/null +++ b/x-pack/metricbeat/module/kafka/_meta/config.yml @@ -0,0 +1,28 @@ +- module: kafka + #metricsets: + # - broker + # - partition + # - consumergroup + period: 10s + hosts: ["localhost:9092"] + + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" diff --git a/x-pack/metricbeat/module/kafka/broker/data.json b/x-pack/metricbeat/module/kafka/broker/data.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/x-pack/metricbeat/module/kafka/broker/manifest.yml b/x-pack/metricbeat/module/kafka/broker/manifest.yml new file mode 100644 index 000000000000..caefc5e089b3 --- /dev/null +++ b/x-pack/metricbeat/module/kafka/broker/manifest.yml @@ -0,0 +1,69 @@ +default: true +input: + module: jolokia + metricset: jmx + defaults: + namespace: "broker" + hosts: ["localhost:8779"] + path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" + jmx.mappings: + - mbean: 'kafka.network:type=RequestChannel,name=RequestQueueSize' + attributes: + - attr: Value + field: request.channel.queue.size + - mbean: 'kafka.server:name=FailedProduceRequestsPerSec,type=BrokerTopicMetrics' + attributes: + - attr: MeanRate + field: request.produce.failed_per_second + - mbean: 'kafka.server:name=FailedFetchRequestsPerSec,type=BrokerTopicMetrics' + attributes: + - attr: MeanRate + field: request.fetch.failed_per_second + - mbean: 'kafka.controller:name=LeaderElectionRateAndTimeMs,type=ControllerStats' + attributes: + - attr: MeanRate + field: replication.leader_elections + - mbean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec' + attributes: + - attr: MeanRate + field: replication.unclean_leader_elections + - mbean: 'kafka.server:name=ZooKeeperDisconnectsPerSec,type=SessionExpireListener' + attributes: + - attr: MeanRate + field: session.zookeeper.disconnect + - mbean: 'kafka.server:name=ZooKeeperExpiresPerSec,type=SessionExpireListener' + attributes: + - attr: MeanRate + field: session.zookeeper.expire + - mbean: 'kafka.server:name=ZooKeeperReadOnlyConnectsPerSec,type=SessionExpireListener' + attributes: + - attr: MeanRate + field: session.zookeeper.readonly + - mbean: 'kafka.server:name=ZooKeeperSyncConnectsPerSec,type=SessionExpireListener' + attributes: + - attr: MeanRate + field: session.zookeeper.sync + - mbean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs' + attributes: + - attr: MeanRate + field: log.flush_rate + - mbean: 'kafka.server:name=BytesRejectedPerSec,type=BrokerTopicMetrics' + attributes: + - attr: MeanRate + field: topic.net.bytes_rejected + - mbean: 'kafka.server:name=BytesInPerSec,type=BrokerTopicMetrics' + attributes: + - attr: MeanRate + field: topic.net.bytes_in + - mbean: 'kafka.server:name=BytesOutPerSec,type=BrokerTopicMetrics' + attributes: + - attr: MeanRate + field: topic.net.bytes_out + - mbean: 'kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics' + attributes: + - attr: MeanRate + field: topic.net.bytes_rejected + - mbean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec' + attributes: + - attr: MeanRate + field: topic.messages_in diff --git a/x-pack/metricbeat/module/kafka/module.yml b/x-pack/metricbeat/module/kafka/module.yml new file mode 100644 index 000000000000..0e8fbe76e34f --- /dev/null +++ b/x-pack/metricbeat/module/kafka/module.yml @@ -0,0 +1,3 @@ +name: kafka +metricsets: +- broker diff --git a/x-pack/metricbeat/modules.d/kafka.yml.disabled b/x-pack/metricbeat/modules.d/kafka.yml.disabled new file mode 100644 index 000000000000..e27d86ff240f --- /dev/null +++ b/x-pack/metricbeat/modules.d/kafka.yml.disabled @@ -0,0 +1,30 @@ +# Module: kafka +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-kafka.html + +- module: kafka + #metricsets: + # - broker + # - partition + # - consumergroup + period: 10s + hosts: ["localhost:8779"] + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" From cc25ce91e618c8b1a14679fc110709a1497c5ad5 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 30 Oct 2019 16:35:20 +0200 Subject: [PATCH 02/29] Add docker files Signed-off-by: chrismark --- metricbeat/docker-compose.yml | 7 +- metricbeat/module/kafka/_meta/Dockerfile | 6 +- metricbeat/module/kafka/_meta/healthcheck.sh | 2 +- metricbeat/module/kafka/_meta/run.sh | 5 +- metricbeat/module/kafka/broker/manifest.yml | 11 --- metricbeat/module/kafka/module.yml | 4 - x-pack/metricbeat/metricbeat.reference.yml | 38 ++++++++- .../metricbeat/module/kafka/_meta/config.yml | 3 +- .../metricbeat/module/kafka/broker/data.json | 82 +++++++++++++++++++ 9 files changed, 132 insertions(+), 26 deletions(-) delete mode 100644 metricbeat/module/kafka/broker/manifest.yml delete mode 100644 metricbeat/module/kafka/module.yml diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index ef8970d649c1..5aadf6d51d3c 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -129,7 +129,7 @@ services: haproxy: image: docker.elastic.co/observability-ci/beats-integration-haproxy:${HAPROXY_VERSION:-1.8}-1 - build: + build: context: ./module/haproxy/_meta args: HAPROXY_VERSION: ${HAPROXY_VERSION:-1.8} @@ -162,6 +162,7 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-2.1.1} ports: - 9092 + - 8779 kibana: image: docker.elastic.co/observability-ci/beats-integration-kibana:${KIBANA_VERSION:-7.4.0}-1 @@ -262,7 +263,7 @@ services: nginx: image: docker.elastic.co/observability-ci/beats-integration-nginx:${NGINX_VERSION:-1.9}-1 - build: + build: context: ./module/nginx/_meta args: NGINX_VERSION: ${NGINX_VERSION:-1.9} @@ -271,7 +272,7 @@ services: phpfpm: image: docker.elastic.co/observability-ci/beats-integration-phpfpm:${PHPFPM_VERSION:-7.1}-1 - build: + build: context: ./module/php_fpm/_meta args: PHPFPM_VERSION: ${PHPFPM_VERSION:-7.1} diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index ebfdbf295caf..73d596fc1cca 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -8,7 +8,7 @@ ENV KAFKA_LOGS_DIR="/kafka-logs" ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true" ENV TERM=linux -RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils +RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils wget RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \ curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \ @@ -16,12 +16,16 @@ RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \ "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \ tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1 +RUN wget -O /opt/jolokia-jvm-1.5.0-agent.jar http://search.maven.org/remotecontent\?filepath\=org/jolokia/jolokia-jvm/1.5.0/jolokia-jvm-1.5.0-agent.jar +# RUN mv jolokia-jvm-1.5.0-agent.jar /opt/jolokia-jvm-1.5.0-agent.jar + ADD kafka_server_jaas.conf /etc/kafka/server_jaas.conf ADD run.sh /run.sh ADD healthcheck.sh /healthcheck.sh EXPOSE 9092 EXPOSE 2181 +EXPOSE 8779 # Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh diff --git a/metricbeat/module/kafka/_meta/healthcheck.sh b/metricbeat/module/kafka/_meta/healthcheck.sh index 97d70b812edc..1012b9c1ba5d 100755 --- a/metricbeat/module/kafka/_meta/healthcheck.sh +++ b/metricbeat/module/kafka/_meta/healthcheck.sh @@ -4,7 +4,7 @@ TOPIC="foo-`date '+%s-%N'`" -${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1 +${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1 --config flush.ms=2 rc=$? if [[ $rc != 0 ]]; then exit $rc diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index 68df3e8a4c6a..7a79081752af 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +KAFKA_ADVERTISED_HOST_AUTO=True + if [ -n "$KAFKA_ADVERTISED_HOST_AUTO" ]; then KAFKA_ADVERTISED_HOST=$(dig +short $HOSTNAME):9092 fi @@ -41,7 +43,7 @@ wait_for_port 2181 echo "Starting Kafka broker" mkdir -p ${KAFKA_LOGS_DIR} -export KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/server_jaas.conf +export KAFKA_OPTS="-Djava.security.auth.login.config=/etc/kafka/server_jaas.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8779,host=0.0.0.0" ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \ --override authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer \ --override super.users=User:admin \ @@ -55,6 +57,7 @@ ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \ --override logs.dir=${KAFKA_LOGS_DIR} & wait_for_port 9092 +wait_for_port 8779 echo "Kafka load status code $?" diff --git a/metricbeat/module/kafka/broker/manifest.yml b/metricbeat/module/kafka/broker/manifest.yml deleted file mode 100644 index 82a50a4c64cf..000000000000 --- a/metricbeat/module/kafka/broker/manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ -default: true -input: - module: jolokia - metricset: jmx - defaults: - namespace: "broker" - jmx.mappings: - - mbean: 'kafka.network:type=RequestChannel,name=RequestQueueSize' - attributes: - - attr: Value - field: request.channel.queue.size diff --git a/metricbeat/module/kafka/module.yml b/metricbeat/module/kafka/module.yml deleted file mode 100644 index 5fe49e5bdf08..000000000000 --- a/metricbeat/module/kafka/module.yml +++ /dev/null @@ -1,4 +0,0 @@ -name: aws -metricsets: - - elb - - ebs diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index b0670dda2ed9..6db0fd4ebdb4 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -446,14 +446,18 @@ metricbeat.modules: #------------------------------- Jolokia Module ------------------------------- - module: jolokia - #metricsets: ["jmx"] + metricsets: ["jmx"] period: 10s - hosts: ["localhost"] + hosts: ["localhost:8779"] namespace: "metrics" #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" #username: "user" #password: "secret" jmx.mappings: + - mbean: 'kafka.network:type=RequestChannel,name=RequestQueueSize' + attributes: + - attr: Value + field: request.channel.queue.size #- mbean: 'java.lang:type=Runtime' # attributes: # - attr: Uptime @@ -477,7 +481,35 @@ metricbeat.modules: #-------------------------------- Kafka Module -------------------------------- - module: kafka - metricsets: ["consumergroup", "partition"] + metricsets: ["broker", "consumergroup", "partition"] + period: 10s + hosts: ["localhost:9092"] + enabled: true + + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" + +#-------------------------------- Kafka Module -------------------------------- +- module: kafka + metricsets: ["broker", "consumergroup", "partition"] period: 10s hosts: ["localhost:9092"] enabled: true diff --git a/x-pack/metricbeat/module/kafka/_meta/config.yml b/x-pack/metricbeat/module/kafka/_meta/config.yml index acef2c0a8256..6d12b13b7c28 100644 --- a/x-pack/metricbeat/module/kafka/_meta/config.yml +++ b/x-pack/metricbeat/module/kafka/_meta/config.yml @@ -4,8 +4,7 @@ # - partition # - consumergroup period: 10s - hosts: ["localhost:9092"] - + hosts: ["localhost:8779"] #client_id: metricbeat #retries: 3 #backoff: 250ms diff --git a/x-pack/metricbeat/module/kafka/broker/data.json b/x-pack/metricbeat/module/kafka/broker/data.json index e69de29bb2d1..15ac30d2f18d 100644 --- a/x-pack/metricbeat/module/kafka/broker/data.json +++ b/x-pack/metricbeat/module/kafka/broker/data.json @@ -0,0 +1,82 @@ +{ + "@timestamp": "2019-10-30T14:22:37.475Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "agent": { + "ephemeral_id": "f08b3bea-3631-4aaf-b35d-4cef29aeeb06", + "hostname": "MBP.lan", + "id": "79dd1677-1bea-4efd-9131-e8ca464eddf0", + "version": "8.0.0", + "type": "metricbeat" + }, + "ecs": { + "version": "1.2.0" + }, + "metricset": { + "name": "broker", + "period": 10000 + }, + "service": { + "address": "localhost:8779", + "type": "kafka" + }, + "kafka": { + "broker": { + "request": { + "fetch": { + "failed_per_second": 0 + }, + "channel": { + "queue": { + "size": 0 + } + }, + "produce": { + "failed_per_second": 0 + } + }, + "replication": { + "leader_elections": 0, + "unclean_leader_elections": 0 + }, + "session": { + "zookeeper": { + "expire": 0, + "readonly": 0, + "sync": 0.00017675970397749868, + "disconnect": 0 + } + }, + "topic": { + "net": { + "bytes_out": 0, + "bytes_rejected": 0, + "bytes_in": 0 + }, + "messages_in": 0 + } + } + }, + "event": { + "dataset": "kafka.broker", + "module": "kafka", + "duration": 7870293 + }, + "host": { + "name": "MBP.lan", + "hostname": "MBP.lan", + "architecture": "x86_64", + "os": { + "version": "10.14.6", + "family": "darwin", + "name": "Mac OS X", + "kernel": "18.7.0", + "build": "18G95", + "platform": "darwin" + }, + "id": "883134FF-0EC4-5E1B-9F9E-FD06FB681D84" + } +} From 84ceb2ac3401a1e92243f8d7edbbbfe57081c433 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 30 Oct 2019 16:51:27 +0200 Subject: [PATCH 03/29] fix config files Signed-off-by: chrismark --- metricbeat/docs/modules/kafka/consumergroup.asciidoc | 1 - metricbeat/docs/modules/kafka/partition.asciidoc | 1 - metricbeat/module/kafka/_meta/healthcheck.sh | 2 +- metricbeat/module/kafka/_meta/run.sh | 2 -- x-pack/metricbeat/metricbeat.reference.yml | 12 ++++-------- .../module/kafka/_meta/config.reference.yml | 2 +- 6 files changed, 6 insertions(+), 14 deletions(-) diff --git a/metricbeat/docs/modules/kafka/consumergroup.asciidoc b/metricbeat/docs/modules/kafka/consumergroup.asciidoc index 10939b7743c2..00f2fde86a5d 100644 --- a/metricbeat/docs/modules/kafka/consumergroup.asciidoc +++ b/metricbeat/docs/modules/kafka/consumergroup.asciidoc @@ -7,7 +7,6 @@ This file is generated! See scripts/mage/docs_collector.go include::../../../module/kafka/consumergroup/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/kafka/partition.asciidoc b/metricbeat/docs/modules/kafka/partition.asciidoc index 7fd178edcdf2..99155eb45ef4 100644 --- a/metricbeat/docs/modules/kafka/partition.asciidoc +++ b/metricbeat/docs/modules/kafka/partition.asciidoc @@ -7,7 +7,6 @@ This file is generated! See scripts/mage/docs_collector.go include::../../../module/kafka/partition/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/module/kafka/_meta/healthcheck.sh b/metricbeat/module/kafka/_meta/healthcheck.sh index 1012b9c1ba5d..97d70b812edc 100755 --- a/metricbeat/module/kafka/_meta/healthcheck.sh +++ b/metricbeat/module/kafka/_meta/healthcheck.sh @@ -4,7 +4,7 @@ TOPIC="foo-`date '+%s-%N'`" -${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1 --config flush.ms=2 +${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1 rc=$? if [[ $rc != 0 ]]; then exit $rc diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index 7a79081752af..55441fa69bd3 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -1,7 +1,5 @@ #!/bin/bash -KAFKA_ADVERTISED_HOST_AUTO=True - if [ -n "$KAFKA_ADVERTISED_HOST_AUTO" ]; then KAFKA_ADVERTISED_HOST=$(dig +short $HOSTNAME):9092 fi diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 6db0fd4ebdb4..3355ac21de3e 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -446,18 +446,14 @@ metricbeat.modules: #------------------------------- Jolokia Module ------------------------------- - module: jolokia - metricsets: ["jmx"] + #metricsets: ["jmx"] period: 10s - hosts: ["localhost:8779"] + hosts: ["localhost"] namespace: "metrics" #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" #username: "user" #password: "secret" jmx.mappings: - - mbean: 'kafka.network:type=RequestChannel,name=RequestQueueSize' - attributes: - - attr: Value - field: request.channel.queue.size #- mbean: 'java.lang:type=Runtime' # attributes: # - attr: Uptime @@ -481,7 +477,7 @@ metricbeat.modules: #-------------------------------- Kafka Module -------------------------------- - module: kafka - metricsets: ["broker", "consumergroup", "partition"] + metricsets: ["consumergroup", "partition"] period: 10s hosts: ["localhost:9092"] enabled: true @@ -511,7 +507,7 @@ metricbeat.modules: - module: kafka metricsets: ["broker", "consumergroup", "partition"] period: 10s - hosts: ["localhost:9092"] + hosts: ["localhost:8779"] enabled: true #client_id: metricbeat diff --git a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml index 4fbf077d9f9a..391850bb1cf5 100644 --- a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml @@ -1,7 +1,7 @@ - module: kafka metricsets: ["broker", "consumergroup", "partition"] period: 10s - hosts: ["localhost:9092"] + hosts: ["localhost:8779"] enabled: true #client_id: metricbeat From e9a26b060e048e90ad34755f6f841762fde41a34 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 31 Oct 2019 11:17:19 +0200 Subject: [PATCH 04/29] Add producer tasks in docker env Signed-off-by: chrismark --- metricbeat/module/kafka/_meta/Dockerfile | 3 +++ .../kafka/_meta/jaas-kafka-client-producer.conf | 5 +++++ metricbeat/module/kafka/_meta/run.sh | 14 ++++++++++++++ .../module/kafka/_meta/sasl-producer.properties | 2 ++ 4 files changed, 24 insertions(+) create mode 100644 metricbeat/module/kafka/_meta/jaas-kafka-client-producer.conf create mode 100644 metricbeat/module/kafka/_meta/sasl-producer.properties diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index 73d596fc1cca..c7e069b1a5c6 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -20,12 +20,15 @@ RUN wget -O /opt/jolokia-jvm-1.5.0-agent.jar http://search.maven.org/remoteconte # RUN mv jolokia-jvm-1.5.0-agent.jar /opt/jolokia-jvm-1.5.0-agent.jar ADD kafka_server_jaas.conf /etc/kafka/server_jaas.conf +ADD jaas-kafka-client-producer.conf /kafka/bin/jaas-kafka-client-producer.conf +ADD sasl-producer.properties /kafka/bin/sasl-producer.properties ADD run.sh /run.sh ADD healthcheck.sh /healthcheck.sh EXPOSE 9092 EXPOSE 2181 EXPOSE 8779 +EXPOSE 8775 # Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh diff --git a/metricbeat/module/kafka/_meta/jaas-kafka-client-producer.conf b/metricbeat/module/kafka/_meta/jaas-kafka-client-producer.conf new file mode 100644 index 000000000000..fa9193ae05b4 --- /dev/null +++ b/metricbeat/module/kafka/_meta/jaas-kafka-client-producer.conf @@ -0,0 +1,5 @@ +KafkaClient { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="producer" + password="producer-secret"; +}; diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index 55441fa69bd3..71a086cd9868 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +KAFKA_ADVERTISED_HOST_AUTO=True + if [ -n "$KAFKA_ADVERTISED_HOST_AUTO" ]; then KAFKA_ADVERTISED_HOST=$(dig +short $HOSTNAME):9092 fi @@ -34,6 +36,7 @@ wait_for_port() { nc -z localhost $port } +${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic test --replication-factor 1 echo "Starting ZooKeeper" ${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties & @@ -57,6 +60,10 @@ ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \ wait_for_port 9092 wait_for_port 8779 + +KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-producer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8775,host=0.0.0.0" \ + + echo "Kafka load status code $?" # ACLS used to prepare tests @@ -69,5 +76,12 @@ ${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localh touch /tmp/.acls_loaded + +# Start a forever producer +cat /dev/urandom | KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-producer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8775,host=0.0.0.0" \ + ${KAFKA_HOME}/bin/kafka-console-producer.sh --topic test --broker-list localhost:9091 --producer.config ${KAFKA_HOME}/bin/sasl-producer.properties > /dev/null & + +wait_for_port 8775 + # Make sure the container keeps running tail -f /dev/null diff --git a/metricbeat/module/kafka/_meta/sasl-producer.properties b/metricbeat/module/kafka/_meta/sasl-producer.properties new file mode 100644 index 000000000000..74ec7eba0fde --- /dev/null +++ b/metricbeat/module/kafka/_meta/sasl-producer.properties @@ -0,0 +1,2 @@ +security.protocol=SASL_PLAINTEXT +sasl.mechanism=PLAIN From 10de02c80d38d60e5a6b51dcf8ca50e78fe05a6f Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 31 Oct 2019 12:24:14 +0200 Subject: [PATCH 05/29] Add producer metricset Signed-off-by: chrismark --- x-pack/metricbeat/metricbeat.reference.yml | 2 +- .../module/kafka/_meta/config.reference.yml | 2 +- .../metricbeat/module/kafka/_meta/config.yml | 1 + x-pack/metricbeat/module/kafka/module.yml | 1 + .../module/kafka/producer/data.json | 60 +++++++++++++++++++ .../module/kafka/producer/manifest.yml | 39 ++++++++++++ .../metricbeat/modules.d/kafka.yml.disabled | 1 + 7 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 x-pack/metricbeat/module/kafka/producer/data.json create mode 100644 x-pack/metricbeat/module/kafka/producer/manifest.yml diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 3355ac21de3e..831a354c88e9 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -505,7 +505,7 @@ metricbeat.modules: #-------------------------------- Kafka Module -------------------------------- - module: kafka - metricsets: ["broker", "consumergroup", "partition"] + metricsets: ["broker", "consumergroup", "partition", "producer"] period: 10s hosts: ["localhost:8779"] enabled: true diff --git a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml index 391850bb1cf5..f024e5f19130 100644 --- a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml @@ -1,5 +1,5 @@ - module: kafka - metricsets: ["broker", "consumergroup", "partition"] + metricsets: ["broker", "consumergroup", "partition", "producer"] period: 10s hosts: ["localhost:8779"] enabled: true diff --git a/x-pack/metricbeat/module/kafka/_meta/config.yml b/x-pack/metricbeat/module/kafka/_meta/config.yml index 6d12b13b7c28..fc805e74c769 100644 --- a/x-pack/metricbeat/module/kafka/_meta/config.yml +++ b/x-pack/metricbeat/module/kafka/_meta/config.yml @@ -3,6 +3,7 @@ # - broker # - partition # - consumergroup + # - producer period: 10s hosts: ["localhost:8779"] #client_id: metricbeat diff --git a/x-pack/metricbeat/module/kafka/module.yml b/x-pack/metricbeat/module/kafka/module.yml index 0e8fbe76e34f..26449afdafc8 100644 --- a/x-pack/metricbeat/module/kafka/module.yml +++ b/x-pack/metricbeat/module/kafka/module.yml @@ -1,3 +1,4 @@ name: kafka metricsets: - broker +- producer diff --git a/x-pack/metricbeat/module/kafka/producer/data.json b/x-pack/metricbeat/module/kafka/producer/data.json new file mode 100644 index 000000000000..77273c67d7f0 --- /dev/null +++ b/x-pack/metricbeat/module/kafka/producer/data.json @@ -0,0 +1,60 @@ +{ + "@timestamp": "2019-10-31T10:19:14.758Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "metricset": { + "name": "producer", + "period": 10000 + }, + "ecs": { + "version": "1.2.0" + }, + "host": { + "os": { + "name": "Mac OS X", + "kernel": "18.7.0", + "build": "18G95", + "platform": "darwin", + "version": "10.14.6", + "family": "darwin" + }, + "id": "883134FF-0EC4-5E1B-9F9E-FD06FB681D84", + "hostname": "abc.local", + "name": "abc.local", + "architecture": "x86_64" + }, + "agent": { + "type": "metricbeat", + "ephemeral_id": "b95327e7-2737-4262-a1a6-ab8547fc8c8d", + "hostname": "abc.local", + "id": "79dd1677-1bea-4efd-9131-e8ca464eddf0", + "version": "8.0.0" + }, + "service": { + "address": "localhost:8775", + "type": "kafka" + }, + "event": { + "dataset": "kafka.producer", + "module": "kafka", + "duration": 4485726 + }, + "kafka": { + "producer": { + "response_rate": 0, + "request_rate": 0, + "record_send_rate": 0, + "batch_size_avg": 0, + "record_size_avg": 0, + "record_retry_rate": 0, + "records_per_request": 0, + "io_wait": 1.2487715219630156e+07, + "mbean": "kafka.producer:client-id=console-producer,type=producer-metrics", + "available_buffer_bytes": 0, + "record_error_rate": 737.5234685412391 + } + } +} diff --git a/x-pack/metricbeat/module/kafka/producer/manifest.yml b/x-pack/metricbeat/module/kafka/producer/manifest.yml new file mode 100644 index 000000000000..9f8af3452f9b --- /dev/null +++ b/x-pack/metricbeat/module/kafka/producer/manifest.yml @@ -0,0 +1,39 @@ +default: true +input: + module: jolokia + metricset: jmx + defaults: + namespace: "producer" + hosts: ["localhost:8775"] + path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" + jmx.mappings: + - mbean: 'kafka.producer:type=producer-metrics,client-id=*' + attributes: + - attr: buffer-available-bytes + field: available_buffer_bytes + - attr: batch-size-avg + field: batch_size_avg + - attr: batch-size-max + field: batch_size_max + - attr: record-send-rate + field: record_send_rate + - attr: record-retry-rate + field: record_retry_rate + - attr: record-error-rate + field: record_error_rate + - attr: records-per-request-avg + field: records_per_request + - attr: record-size-avg + field: record_size_avg + - attr: record-size-max + field: record_size_max + - attr: request-rate + field: request_rate + - attr: response-rate + field: response_rate + - attr: io-wait-time-ns-avg + field: io_wait + - mbean: 'kafka.producer:client-id=console-producer,node-id=*,type=producer-node-metrics' + attributes: + - attr: outgoing-byte-total + field: bytes_out diff --git a/x-pack/metricbeat/modules.d/kafka.yml.disabled b/x-pack/metricbeat/modules.d/kafka.yml.disabled index e27d86ff240f..02d0d6d29c48 100644 --- a/x-pack/metricbeat/modules.d/kafka.yml.disabled +++ b/x-pack/metricbeat/modules.d/kafka.yml.disabled @@ -6,6 +6,7 @@ # - broker # - partition # - consumergroup + # - producer period: 10s hosts: ["localhost:8779"] #client_id: metricbeat From 0f17f51f6da7e893e271e8536ea34571444b64a3 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 31 Oct 2019 15:10:56 +0200 Subject: [PATCH 06/29] Add cosumer task in docker env Signed-off-by: chrismark --- metricbeat/docker-compose.yml | 2 ++ metricbeat/module/kafka/_meta/Dockerfile | 3 +++ .../module/kafka/_meta/jaas-kafka-client-consumer.conf | 5 +++++ metricbeat/module/kafka/_meta/run.sh | 5 +++++ metricbeat/module/kafka/_meta/sasl-consumer.properties | 2 ++ 5 files changed, 17 insertions(+) create mode 100644 metricbeat/module/kafka/_meta/jaas-kafka-client-consumer.conf create mode 100644 metricbeat/module/kafka/_meta/sasl-consumer.properties diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 5aadf6d51d3c..4b4186387d52 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -163,6 +163,8 @@ services: ports: - 9092 - 8779 + - 8775 + - 8774 kibana: image: docker.elastic.co/observability-ci/beats-integration-kibana:${KIBANA_VERSION:-7.4.0}-1 diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index c7e069b1a5c6..312bf9175f3f 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -22,6 +22,8 @@ RUN wget -O /opt/jolokia-jvm-1.5.0-agent.jar http://search.maven.org/remoteconte ADD kafka_server_jaas.conf /etc/kafka/server_jaas.conf ADD jaas-kafka-client-producer.conf /kafka/bin/jaas-kafka-client-producer.conf ADD sasl-producer.properties /kafka/bin/sasl-producer.properties +ADD jaas-kafka-client-consumer.conf /kafka/bin/jaas-kafka-client-consumer.conf +ADD sasl-consumer.properties /kafka/bin/sasl-consumer.properties ADD run.sh /run.sh ADD healthcheck.sh /healthcheck.sh @@ -29,6 +31,7 @@ EXPOSE 9092 EXPOSE 2181 EXPOSE 8779 EXPOSE 8775 +EXPOSE 8774 # Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh diff --git a/metricbeat/module/kafka/_meta/jaas-kafka-client-consumer.conf b/metricbeat/module/kafka/_meta/jaas-kafka-client-consumer.conf new file mode 100644 index 000000000000..80b39c1e16ca --- /dev/null +++ b/metricbeat/module/kafka/_meta/jaas-kafka-client-consumer.conf @@ -0,0 +1,5 @@ +KafkaClient { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="consumer" + password="consumer-secret"; +}; diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index 71a086cd9868..003b89185248 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -83,5 +83,10 @@ cat /dev/urandom | KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas wait_for_port 8775 +KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-consumer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8774,host=0.0.0.0" \ + ${KAFKA_HOME}/bin/kafka-console-consumer.sh --topic=test --bootstrap-server=localhost:9091 --consumer.config ${KAFKA_HOME}/bin/sasl-producer.properties > /dev/null & + +wait_for_port 8774 + # Make sure the container keeps running tail -f /dev/null diff --git a/metricbeat/module/kafka/_meta/sasl-consumer.properties b/metricbeat/module/kafka/_meta/sasl-consumer.properties new file mode 100644 index 000000000000..74ec7eba0fde --- /dev/null +++ b/metricbeat/module/kafka/_meta/sasl-consumer.properties @@ -0,0 +1,2 @@ +security.protocol=SASL_PLAINTEXT +sasl.mechanism=PLAIN From 6f24a772e5d3f1bf8db5027be9a6b7fb809b45eb Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 31 Oct 2019 15:24:28 +0200 Subject: [PATCH 07/29] Add consumer metricset Signed-off-by: chrismark --- x-pack/metricbeat/metricbeat.reference.yml | 2 +- .../module/kafka/_meta/config.reference.yml | 2 +- .../metricbeat/module/kafka/_meta/config.yml | 2 + .../module/kafka/consumer/data.json | 53 +++++++++++++++++++ .../module/kafka/consumer/manifest.yml | 21 ++++++++ x-pack/metricbeat/module/kafka/module.yml | 1 + .../metricbeat/modules.d/kafka.yml.disabled | 2 + 7 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 x-pack/metricbeat/module/kafka/consumer/data.json create mode 100644 x-pack/metricbeat/module/kafka/consumer/manifest.yml diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 831a354c88e9..0930250fa4cc 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -505,7 +505,7 @@ metricbeat.modules: #-------------------------------- Kafka Module -------------------------------- - module: kafka - metricsets: ["broker", "consumergroup", "partition", "producer"] + metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] period: 10s hosts: ["localhost:8779"] enabled: true diff --git a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml index f024e5f19130..efc34ff0bd80 100644 --- a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml @@ -1,5 +1,5 @@ - module: kafka - metricsets: ["broker", "consumergroup", "partition", "producer"] + metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] period: 10s hosts: ["localhost:8779"] enabled: true diff --git a/x-pack/metricbeat/module/kafka/_meta/config.yml b/x-pack/metricbeat/module/kafka/_meta/config.yml index fc805e74c769..06259150ed4e 100644 --- a/x-pack/metricbeat/module/kafka/_meta/config.yml +++ b/x-pack/metricbeat/module/kafka/_meta/config.yml @@ -2,8 +2,10 @@ #metricsets: # - broker # - partition + # - consumer # - consumergroup # - producer + period: 10s hosts: ["localhost:8779"] #client_id: metricbeat diff --git a/x-pack/metricbeat/module/kafka/consumer/data.json b/x-pack/metricbeat/module/kafka/consumer/data.json new file mode 100644 index 000000000000..e5cd7d1e031f --- /dev/null +++ b/x-pack/metricbeat/module/kafka/consumer/data.json @@ -0,0 +1,53 @@ +{ + "@timestamp": "2019-10-31T13:22:06.700Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "kafka": { + "consumer": { + "bytes_consumed": 2.9521300228e+10, + "fetch_rate": 0, + "records_consumed": 1.23075656e+08, + "mbean": "kafka.consumer:client-id=consumer-1,type=consumer-fetch-manager-metrics" + } + }, + "event": { + "dataset": "kafka.consumer", + "module": "kafka", + "duration": 7042831 + }, + "ecs": { + "version": "1.2.0" + }, + "host": { + "name": "pr.local", + "hostname": "pr.local", + "architecture": "x86_64", + "os": { + "kernel": "18.7.0", + "build": "18G95", + "platform": "darwin", + "version": "10.14.6", + "family": "darwin", + "name": "Mac OS X" + }, + "id": "883134FF-0EC4-5E1B-9F9E-FD06FB681D84" + }, + "agent": { + "hostname": "pr.local", + "id": "79dd1677-1bea-4efd-9131-e8ca464eddf0", + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "e40f5843-d3aa-4bdc-a100-64022b70851b" + }, + "metricset": { + "name": "consumer", + "period": 10000 + }, + "service": { + "address": "localhost:8774", + "type": "kafka" + } +} diff --git a/x-pack/metricbeat/module/kafka/consumer/manifest.yml b/x-pack/metricbeat/module/kafka/consumer/manifest.yml new file mode 100644 index 000000000000..93b8b29ad70a --- /dev/null +++ b/x-pack/metricbeat/module/kafka/consumer/manifest.yml @@ -0,0 +1,21 @@ +default: true +input: + module: jolokia + metricset: jmx + defaults: + namespace: "consumer" + hosts: ["localhost:8774"] + path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" + jmx.mappings: + - mbean: 'kafka.consumer:client-id=*,type=consumer-fetch-manager-metrics' + attributes: + - attr: fetch-rate + field: fetch_rate + - attr: bytes-consumed-total + field: bytes_consumed + - attr: records-consumed-total + field: records_consumed + - mbean: 'kafka.consumer:client-id=*,type=consumer-metrics' + attributes: + - attr: incoming-byte-total + field: bytes_in diff --git a/x-pack/metricbeat/module/kafka/module.yml b/x-pack/metricbeat/module/kafka/module.yml index 26449afdafc8..91621d292096 100644 --- a/x-pack/metricbeat/module/kafka/module.yml +++ b/x-pack/metricbeat/module/kafka/module.yml @@ -2,3 +2,4 @@ name: kafka metricsets: - broker - producer +- consumer diff --git a/x-pack/metricbeat/modules.d/kafka.yml.disabled b/x-pack/metricbeat/modules.d/kafka.yml.disabled index 02d0d6d29c48..312ab7d068e6 100644 --- a/x-pack/metricbeat/modules.d/kafka.yml.disabled +++ b/x-pack/metricbeat/modules.d/kafka.yml.disabled @@ -5,8 +5,10 @@ #metricsets: # - broker # - partition + # - consumer # - consumergroup # - producer + period: 10s hosts: ["localhost:8779"] #client_id: metricbeat From e6bc8dbfd1bb472a92e92421f2de1b052359f47f Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 31 Oct 2019 15:26:29 +0200 Subject: [PATCH 08/29] Fix back run script Signed-off-by: chrismark --- metricbeat/module/kafka/_meta/run.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index 003b89185248..fffc82aef92a 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -1,7 +1,5 @@ #!/bin/bash -KAFKA_ADVERTISED_HOST_AUTO=True - if [ -n "$KAFKA_ADVERTISED_HOST_AUTO" ]; then KAFKA_ADVERTISED_HOST=$(dig +short $HOSTNAME):9092 fi From 5ea1c6003c50a232814c0270616c5428d6380d58 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 1 Nov 2019 12:15:01 +0200 Subject: [PATCH 09/29] Add fields Signed-off-by: chrismark --- x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/metricbeat.reference.yml | 2 +- .../metricbeat/module/kafka/_meta/fields.yml | 9 ++++ .../module/kafka/broker/{ => _meta}/data.json | 0 .../module/kafka/broker/_meta/fields.yml | 50 +++++++++++++++++++ .../module/kafka/broker/manifest.yml | 4 -- .../kafka/consumer/{ => _meta}/data.json | 0 .../module/kafka/consumer/_meta/fields.yml | 20 ++++++++ x-pack/metricbeat/module/kafka/fields.go | 23 +++++++++ .../kafka/producer/{ => _meta}/data.json | 0 .../module/kafka/producer/_meta/fields.yml | 47 +++++++++++++++++ 11 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 x-pack/metricbeat/module/kafka/_meta/fields.yml rename x-pack/metricbeat/module/kafka/broker/{ => _meta}/data.json (100%) create mode 100644 x-pack/metricbeat/module/kafka/broker/_meta/fields.yml rename x-pack/metricbeat/module/kafka/consumer/{ => _meta}/data.json (100%) create mode 100644 x-pack/metricbeat/module/kafka/consumer/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/kafka/fields.go rename x-pack/metricbeat/module/kafka/producer/{ => _meta}/data.json (100%) create mode 100644 x-pack/metricbeat/module/kafka/producer/_meta/fields.yml diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index fd1d8f95ed05..306e384d569e 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -24,6 +24,7 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/cockroachdb" _ "github.com/elastic/beats/x-pack/metricbeat/module/coredns" _ "github.com/elastic/beats/x-pack/metricbeat/module/coredns/stats" + _ "github.com/elastic/beats/x-pack/metricbeat/module/kafka" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/transaction_log" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 0930250fa4cc..463421ecd2b0 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -503,7 +503,7 @@ metricbeat.modules: #username: "" #password: "" -#-------------------------------- Kafka Module -------------------------------- +#------------------------------- KafkaJmx Module ------------------------------- - module: kafka metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] period: 10s diff --git a/x-pack/metricbeat/module/kafka/_meta/fields.yml b/x-pack/metricbeat/module/kafka/_meta/fields.yml new file mode 100644 index 000000000000..417cf08c70de --- /dev/null +++ b/x-pack/metricbeat/module/kafka/_meta/fields.yml @@ -0,0 +1,9 @@ +- key: kafkajmx + title: 'KafkaJmx' + release: beta + description: > + KafkaJmx module + fields: + - name: KafkaJmx + type: group + fields: diff --git a/x-pack/metricbeat/module/kafka/broker/data.json b/x-pack/metricbeat/module/kafka/broker/_meta/data.json similarity index 100% rename from x-pack/metricbeat/module/kafka/broker/data.json rename to x-pack/metricbeat/module/kafka/broker/_meta/data.json diff --git a/x-pack/metricbeat/module/kafka/broker/_meta/fields.yml b/x-pack/metricbeat/module/kafka/broker/_meta/fields.yml new file mode 100644 index 000000000000..4231ee0cf295 --- /dev/null +++ b/x-pack/metricbeat/module/kafka/broker/_meta/fields.yml @@ -0,0 +1,50 @@ +- name: broker + type: group + description: Broker metrics from Kafka Broker JMX + release: beta + fields: + - name: mbean + description: Mbean that this event is related to + type: keyword + - name: request.channel.queue.size + description: The size of the request queue + type: long + - name: request.produce.failed_per_second + description: The rate of failed produce requests per second + type: long + - name: request.fetch.failed_per_second + description: The rate of client fetch request failures per second + type: long + - name: replication.leader_elections + description: The leader election rate + type: long + - name: replication.unclean_leader_elections + description: The unclean leader election rate + type: long + - name: session.zookeeper.disconnect + description: The ZooKeeper closed sessions per second + type: long + - name: session.zookeeper.expire + description: The ZooKeeper expired sessions per second + type: long + - name: session.zookeeper.readonly + description: The ZooKeeper readonly sessions per second + type: long + - name: session.zookeeper.sync + description: The ZooKeeper client connections per second + type: float + - name: log.flush_rate + description: The log flush rate + type: float + - name: topic.net.bytes_in + description: The incoming byte rate + type: long + - name: topic.net.bytes_out + description: The outgoing byte rate + type: long + - name: topic.net.bytes_rejected + description: The rejected byte rate + type: long + - name: topic.messages_in + description: The incoming message rate + type: long diff --git a/x-pack/metricbeat/module/kafka/broker/manifest.yml b/x-pack/metricbeat/module/kafka/broker/manifest.yml index caefc5e089b3..e8c68c4459ed 100644 --- a/x-pack/metricbeat/module/kafka/broker/manifest.yml +++ b/x-pack/metricbeat/module/kafka/broker/manifest.yml @@ -59,10 +59,6 @@ input: attributes: - attr: MeanRate field: topic.net.bytes_out - - mbean: 'kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics' - attributes: - - attr: MeanRate - field: topic.net.bytes_rejected - mbean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec' attributes: - attr: MeanRate diff --git a/x-pack/metricbeat/module/kafka/consumer/data.json b/x-pack/metricbeat/module/kafka/consumer/_meta/data.json similarity index 100% rename from x-pack/metricbeat/module/kafka/consumer/data.json rename to x-pack/metricbeat/module/kafka/consumer/_meta/data.json diff --git a/x-pack/metricbeat/module/kafka/consumer/_meta/fields.yml b/x-pack/metricbeat/module/kafka/consumer/_meta/fields.yml new file mode 100644 index 000000000000..cc39470b617f --- /dev/null +++ b/x-pack/metricbeat/module/kafka/consumer/_meta/fields.yml @@ -0,0 +1,20 @@ +- name: consumer + type: group + description: Consumer metrics from Kafka Consumer JMX + release: beta + fields: + - name: mbean + description: Mbean that this event is related to + type: keyword + - name: fetch_rate + description: The minimum rate at which the consumer sends fetch requests to a broker + type: float + - name: bytes_consumed + description: The average number of bytes consumed for a specific topic per second + type: float + - name: records_consumed + description: The average number of records consumed per second for a specific topic + type: float + - name: bytes_in + description: The rate of bytes coming in to the consumer + type: long diff --git a/x-pack/metricbeat/module/kafka/fields.go b/x-pack/metricbeat/module/kafka/fields.go new file mode 100644 index 000000000000..3fb8dc611432 --- /dev/null +++ b/x-pack/metricbeat/module/kafka/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package kafka + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "kafka", asset.ModuleFieldsPri, AssetKafka); err != nil { + panic(err) + } +} + +// AssetKafka returns asset data. +// This is the base64 encoded gzipped contents of module/kafka. +func AssetKafka() string { + return "eJzUlz+T2zYQxXt9iu1cmelVpEiq2ONJihSZNBwIfBRhAVgaf+4kf/oMQFLS6XjiH/lmYnUi8LA/YvEWy490wGlLB1EfxFdz3BAFFTS29OFzevTJHD9siBw0hMeWdghiQ1TBS6faoNhu6dcNEdEwnQxXUWNDVCvoym/z6EeywmB7npUfEoVTiy3tHce2f3ItuhbuHB/gzo/HpOn3Auy3rCGD4JT0VDs2HcAw8unLP1fa25ccfrdI11hmB2FfjNwwfEkTKDQiUGiUJzzBBlI+RRMBFQW+kXcvdsDpmV01GtThW4QPhWyEtdDFt4iIwqvvuEfydwNKc4hrCg2GVSirRxk02/1dgNZxFSWKWiiNqmzhSg/JtpricCJkjk5I/TrDup5aOBpdaSZYjSCb9VhSq5SlvMp5n9Jq0eERuFYrKVK4QkNUcCU0ZPrvp8i6+TTMz6Tro0crNYQtl1L0uh9A4+F9IvnOfABauKJSXrK1kGGK4l/mz1lDUrNHNSy2PjOvaXBslZu004Wkm/8uKA6iYqtP82EGxXvQ+JOVSxKUjdQndg5KrVmEURbN+6LW0TflyHF7bRjeU5799uF8O1TgVsnCIhS7U4Av1d0Sn8IpK9kou6ckWOGH24gcJ23AMez5B4Z0+AoZMF0k+3mPxTXwXuyX7W2vmRlziCfZ+miW9g6/96qx7uE89pP2D/lem+Uio6wy0XQXowj03CjZ5O5h2FXysJV/eVV6CkzidctGM6zXncV+9cmzKJ7g0omw0ezg0tWd9QNdRTU7EuRbSFUr2Z299SXIQbKrHsHrV7gAXlhGWVft3rSlhkZn2K1sL2VT3q6TO99kfQe30GR/9aoxk53HflKTiSehtNhplLtY13Bl3uupxAQOQpMwHG3ICcpaMjDsbjuAiTq7E8nlqeMvxdN+rZM87O1NNHkIL4GNOE6WGHHMJWZ24DsdbvJWmQrSrOL2tjtT7EerROkQ3Gk1SHAKVb9UX2MfBYJz7P5HQD5/mfV3xvsma/LMLHTJsA2vP7ln5mKpO6YCTn0Rz0r7ZW+Hcj77i/zeC/uWrcd6gk7/AILi8lmoySN2DvnHL39SElBQZml+Z7fvL6/grpPnGHIXEK5o3k73fwEAAP//6F+3kA==" +} diff --git a/x-pack/metricbeat/module/kafka/producer/data.json b/x-pack/metricbeat/module/kafka/producer/_meta/data.json similarity index 100% rename from x-pack/metricbeat/module/kafka/producer/data.json rename to x-pack/metricbeat/module/kafka/producer/_meta/data.json diff --git a/x-pack/metricbeat/module/kafka/producer/_meta/fields.yml b/x-pack/metricbeat/module/kafka/producer/_meta/fields.yml new file mode 100644 index 000000000000..e0c3eaf14a02 --- /dev/null +++ b/x-pack/metricbeat/module/kafka/producer/_meta/fields.yml @@ -0,0 +1,47 @@ +- name: producer + type: group + description: Producer metrics from Kafka Producer JMX + release: beta + fields: + - name: mbean + description: Mbean that this event is related to + type: keyword + - name: available_buffer_bytes + description: The total amount of buffer memory + type: long + - name: batch_size_avg + description: The average number of bytes sent + type: float + - name: batch_size_max + description: The maximum number of bytes sent + type: long + - name: record_send_rate + description: The average number of records sent per second + type: float + - name: record_retry_rate + description: The average number of retried record sends per second + type: float + - name: record_error_rate + description: The average number of retried record sends per second + type: float + - name: records_per_request + description: The average number of records sent per second + type: long + - name: record_size_avg + description: The average record size + type: float + - name: record_size_max + description: The maximum record size + type: long + - name: request_rate + description: The number of producer requests per second + type: float + - name: response_rate + description: The number of producer responses per second + type: float + - name: io_wait + description: The producer I/O wait time + type: float + - name: bytes_out + description: The rate of bytes going out for the producer + type: long From c2da3a3be10ac0cc89a1b44aee82899d4db4f6b2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 1 Nov 2019 13:19:31 +0200 Subject: [PATCH 10/29] Add docs Signed-off-by: chrismark --- .../metricbeat/module/kafka/broker/_meta/docs.asciidoc | 9 +++++++++ .../metricbeat/module/kafka/consumer/_meta/docs.asciidoc | 9 +++++++++ .../metricbeat/module/kafka/producer/_meta/docs.asciidoc | 9 +++++++++ 3 files changed, 27 insertions(+) create mode 100644 x-pack/metricbeat/module/kafka/broker/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/kafka/consumer/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/kafka/producer/_meta/docs.asciidoc diff --git a/x-pack/metricbeat/module/kafka/broker/_meta/docs.asciidoc b/x-pack/metricbeat/module/kafka/broker/_meta/docs.asciidoc new file mode 100644 index 000000000000..d6159805492a --- /dev/null +++ b/x-pack/metricbeat/module/kafka/broker/_meta/docs.asciidoc @@ -0,0 +1,9 @@ +This metricset periodically fetches JMX metrics from Kafka Broker JMX. + +[float] +=== Compatibility +The module has been tested with Kafka 2.1.1. Other versions are expected to work. + +[float] +=== Usage +The Broker metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. diff --git a/x-pack/metricbeat/module/kafka/consumer/_meta/docs.asciidoc b/x-pack/metricbeat/module/kafka/consumer/_meta/docs.asciidoc new file mode 100644 index 000000000000..9f3390c36fad --- /dev/null +++ b/x-pack/metricbeat/module/kafka/consumer/_meta/docs.asciidoc @@ -0,0 +1,9 @@ +This metricset periodically fetches JMX metrics from Kafka Producers JMX. + +[float] +=== Compatibility +The module has been tested with Kafka 2.1.1. Other versions are expected to work. + +[float] +=== Usage +The Consumer metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. diff --git a/x-pack/metricbeat/module/kafka/producer/_meta/docs.asciidoc b/x-pack/metricbeat/module/kafka/producer/_meta/docs.asciidoc new file mode 100644 index 000000000000..6bf7ddb8992f --- /dev/null +++ b/x-pack/metricbeat/module/kafka/producer/_meta/docs.asciidoc @@ -0,0 +1,9 @@ +This metricset periodically fetches JMX metrics from Kafka Producers JMX. + +[float] +=== Compatibility +The module has been tested with Kafka 2.1.1. Other versions are expected to work. + +[float] +=== Usage +The Producer metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. From e98ee2f5d487ec5efb63cbd8a55b80994ff560a4 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 12 Nov 2019 11:21:12 +0200 Subject: [PATCH 11/29] Move kafka jmx to oss Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 324 ++++++++++++++++++ metricbeat/docs/modules/kafka.asciidoc | 14 +- metricbeat/docs/modules/kafka/broker.asciidoc | 23 ++ .../docs/modules/kafka/consumer.asciidoc | 23 ++ .../docs/modules/kafka/consumergroup.asciidoc | 1 + .../docs/modules/kafka/partition.asciidoc | 1 + .../docs/modules/kafka/producer.asciidoc | 23 ++ metricbeat/docs/modules_list.asciidoc | 5 +- metricbeat/include/list_common.go | 3 + metricbeat/metricbeat.reference.yml | 2 +- .../module/kafka/_meta/config.reference.yml | 2 +- metricbeat/module/kafka/_meta/config.yml | 3 + .../module/kafka/broker/_meta/data.json | 0 .../module/kafka/broker/_meta/docs.asciidoc | 0 .../module/kafka/broker/_meta/fields.yml | 0 .../module/kafka/broker/manifest.yml | 0 .../module/kafka/consumer/_meta/data.json | 0 .../module/kafka/consumer/_meta/docs.asciidoc | 0 .../module/kafka/consumer/_meta/fields.yml | 0 .../module/kafka/consumer/manifest.yml | 0 metricbeat/module/kafka/fields.go | 2 +- .../module/kafka/module.yml | 0 .../module/kafka/producer/_meta/data.json | 0 .../module/kafka/producer/_meta/docs.asciidoc | 0 .../module/kafka/producer/_meta/fields.yml | 0 .../module/kafka/producer/manifest.yml | 0 metricbeat/modules.d/kafka.yml.disabled | 3 + x-pack/metricbeat/include/list.go | 1 - x-pack/metricbeat/metricbeat.reference.yml | 30 +- .../module/kafka/_meta/config.reference.yml | 26 -- .../metricbeat/module/kafka/_meta/config.yml | 30 -- .../metricbeat/module/kafka/_meta/fields.yml | 9 - x-pack/metricbeat/module/kafka/fields.go | 23 -- .../metricbeat/modules.d/kafka.yml.disabled | 33 -- 34 files changed, 425 insertions(+), 156 deletions(-) create mode 100644 metricbeat/docs/modules/kafka/broker.asciidoc create mode 100644 metricbeat/docs/modules/kafka/consumer.asciidoc create mode 100644 metricbeat/docs/modules/kafka/producer.asciidoc rename {x-pack/metricbeat => metricbeat}/module/kafka/broker/_meta/data.json (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/broker/_meta/docs.asciidoc (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/broker/_meta/fields.yml (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/broker/manifest.yml (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/consumer/_meta/data.json (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/consumer/_meta/docs.asciidoc (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/consumer/_meta/fields.yml (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/consumer/manifest.yml (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/module.yml (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/producer/_meta/data.json (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/producer/_meta/docs.asciidoc (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/producer/_meta/fields.yml (100%) rename {x-pack/metricbeat => metricbeat}/module/kafka/producer/manifest.yml (100%) delete mode 100644 x-pack/metricbeat/module/kafka/_meta/config.reference.yml delete mode 100644 x-pack/metricbeat/module/kafka/_meta/config.yml delete mode 100644 x-pack/metricbeat/module/kafka/_meta/fields.yml delete mode 100644 x-pack/metricbeat/module/kafka/fields.go delete mode 100644 x-pack/metricbeat/modules.d/kafka.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ad189ce4096c..1bb5f51670a8 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -14828,6 +14828,198 @@ type: keyword -- +[float] +=== broker + +Broker metrics from Kafka Broker JMX + + +*`kafka.broker.mbean`*:: ++ +-- +Mbean that this event is related to + +type: keyword + +-- + +*`kafka.broker.request.channel.queue.size`*:: ++ +-- +The size of the request queue + +type: long + +-- + +*`kafka.broker.request.produce.failed_per_second`*:: ++ +-- +The rate of failed produce requests per second + +type: long + +-- + +*`kafka.broker.request.fetch.failed_per_second`*:: ++ +-- +The rate of client fetch request failures per second + +type: long + +-- + +*`kafka.broker.replication.leader_elections`*:: ++ +-- +The leader election rate + +type: long + +-- + +*`kafka.broker.replication.unclean_leader_elections`*:: ++ +-- +The unclean leader election rate + +type: long + +-- + +*`kafka.broker.session.zookeeper.disconnect`*:: ++ +-- +The ZooKeeper closed sessions per second + +type: long + +-- + +*`kafka.broker.session.zookeeper.expire`*:: ++ +-- +The ZooKeeper expired sessions per second + +type: long + +-- + +*`kafka.broker.session.zookeeper.readonly`*:: ++ +-- +The ZooKeeper readonly sessions per second + +type: long + +-- + +*`kafka.broker.session.zookeeper.sync`*:: ++ +-- +The ZooKeeper client connections per second + +type: float + +-- + +*`kafka.broker.log.flush_rate`*:: ++ +-- +The log flush rate + +type: float + +-- + +*`kafka.broker.topic.net.bytes_in`*:: ++ +-- +The incoming byte rate + +type: long + +-- + +*`kafka.broker.topic.net.bytes_out`*:: ++ +-- +The outgoing byte rate + +type: long + +-- + +*`kafka.broker.topic.net.bytes_rejected`*:: ++ +-- +The rejected byte rate + +type: long + +-- + +*`kafka.broker.topic.messages_in`*:: ++ +-- +The incoming message rate + +type: long + +-- + +[float] +=== consumer + +Consumer metrics from Kafka Consumer JMX + + +*`kafka.consumer.mbean`*:: ++ +-- +Mbean that this event is related to + +type: keyword + +-- + +*`kafka.consumer.fetch_rate`*:: ++ +-- +The minimum rate at which the consumer sends fetch requests to a broker + +type: float + +-- + +*`kafka.consumer.bytes_consumed`*:: ++ +-- +The average number of bytes consumed for a specific topic per second + +type: float + +-- + +*`kafka.consumer.records_consumed`*:: ++ +-- +The average number of records consumed per second for a specific topic + +type: float + +-- + +*`kafka.consumer.bytes_in`*:: ++ +-- +The rate of bytes coming in to the consumer + +type: long + +-- + [float] === consumergroup @@ -15123,6 +15315,138 @@ type: keyword -- +[float] +=== producer + +Producer metrics from Kafka Producer JMX + + +*`kafka.producer.mbean`*:: ++ +-- +Mbean that this event is related to + +type: keyword + +-- + +*`kafka.producer.available_buffer_bytes`*:: ++ +-- +The total amount of buffer memory + +type: long + +-- + +*`kafka.producer.batch_size_avg`*:: ++ +-- +The average number of bytes sent + +type: float + +-- + +*`kafka.producer.batch_size_max`*:: ++ +-- +The maximum number of bytes sent + +type: long + +-- + +*`kafka.producer.record_send_rate`*:: ++ +-- +The average number of records sent per second + +type: float + +-- + +*`kafka.producer.record_retry_rate`*:: ++ +-- +The average number of retried record sends per second + +type: float + +-- + +*`kafka.producer.record_error_rate`*:: ++ +-- +The average number of retried record sends per second + +type: float + +-- + +*`kafka.producer.records_per_request`*:: ++ +-- +The average number of records sent per second + +type: long + +-- + +*`kafka.producer.record_size_avg`*:: ++ +-- +The average record size + +type: float + +-- + +*`kafka.producer.record_size_max`*:: ++ +-- +The maximum record size + +type: long + +-- + +*`kafka.producer.request_rate`*:: ++ +-- +The number of producer requests per second + +type: float + +-- + +*`kafka.producer.response_rate`*:: ++ +-- +The number of producer responses per second + +type: float + +-- + +*`kafka.producer.io_wait`*:: ++ +-- +The producer I/O wait time + +type: float + +-- + +*`kafka.producer.bytes_out`*:: ++ +-- +The rate of bytes going out for the producer + +type: long + +-- + [[exported-fields-kibana]] == Kibana fields diff --git a/metricbeat/docs/modules/kafka.asciidoc b/metricbeat/docs/modules/kafka.asciidoc index 4f97262b1f4f..8f23804cbf1f 100644 --- a/metricbeat/docs/modules/kafka.asciidoc +++ b/metricbeat/docs/modules/kafka.asciidoc @@ -40,7 +40,7 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: kafka - metricsets: ["consumergroup", "partition"] + metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] period: 10s hosts: ["localhost:9092"] enabled: true @@ -72,11 +72,23 @@ metricbeat.modules: The following metricsets are available: +* <> + +* <> + * <> * <> +* <> + +include::kafka/broker.asciidoc[] + +include::kafka/consumer.asciidoc[] + include::kafka/consumergroup.asciidoc[] include::kafka/partition.asciidoc[] +include::kafka/producer.asciidoc[] + diff --git a/metricbeat/docs/modules/kafka/broker.asciidoc b/metricbeat/docs/modules/kafka/broker.asciidoc new file mode 100644 index 000000000000..191351eb98f7 --- /dev/null +++ b/metricbeat/docs/modules/kafka/broker.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-broker]] +=== Kafka broker metricset + +beta[] + +include::../../../module/kafka/broker/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/broker/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kafka/consumer.asciidoc b/metricbeat/docs/modules/kafka/consumer.asciidoc new file mode 100644 index 000000000000..3a2acd51cda7 --- /dev/null +++ b/metricbeat/docs/modules/kafka/consumer.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-consumer]] +=== Kafka consumer metricset + +beta[] + +include::../../../module/kafka/consumer/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/consumer/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kafka/consumergroup.asciidoc b/metricbeat/docs/modules/kafka/consumergroup.asciidoc index 00f2fde86a5d..10939b7743c2 100644 --- a/metricbeat/docs/modules/kafka/consumergroup.asciidoc +++ b/metricbeat/docs/modules/kafka/consumergroup.asciidoc @@ -7,6 +7,7 @@ This file is generated! See scripts/mage/docs_collector.go include::../../../module/kafka/consumergroup/_meta/docs.asciidoc[] +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/kafka/partition.asciidoc b/metricbeat/docs/modules/kafka/partition.asciidoc index 99155eb45ef4..7fd178edcdf2 100644 --- a/metricbeat/docs/modules/kafka/partition.asciidoc +++ b/metricbeat/docs/modules/kafka/partition.asciidoc @@ -7,6 +7,7 @@ This file is generated! See scripts/mage/docs_collector.go include::../../../module/kafka/partition/_meta/docs.asciidoc[] +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/kafka/producer.asciidoc b/metricbeat/docs/modules/kafka/producer.asciidoc new file mode 100644 index 000000000000..2ce5ab0a3f43 --- /dev/null +++ b/metricbeat/docs/modules/kafka/producer.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-producer]] +=== Kafka producer metricset + +beta[] + +include::../../../module/kafka/producer/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/producer/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index e449cfa39a2a..e87fbb99a239 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -92,8 +92,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> +.5+| .5+| |<> beta[] +|<> beta[] +|<> |<> +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> |<> diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index b5158ec8380a..9ad89ca9d17e 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -84,8 +84,11 @@ import ( _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" + _ "github.com/elastic/beats/metricbeat/module/kafka/broker" + _ "github.com/elastic/beats/metricbeat/module/kafka/consumer" _ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/metricbeat/module/kafka/partition" + _ "github.com/elastic/beats/metricbeat/module/kafka/producer" _ "github.com/elastic/beats/metricbeat/module/kibana" _ "github.com/elastic/beats/metricbeat/module/kibana/stats" _ "github.com/elastic/beats/metricbeat/module/kibana/status" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index e7399cb6a18b..1e777842d36b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -375,7 +375,7 @@ metricbeat.modules: #-------------------------------- Kafka Module -------------------------------- - module: kafka - metricsets: ["consumergroup", "partition"] + metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] period: 10s hosts: ["localhost:9092"] enabled: true diff --git a/metricbeat/module/kafka/_meta/config.reference.yml b/metricbeat/module/kafka/_meta/config.reference.yml index 0a9d9fe88caf..2908f870b4c1 100644 --- a/metricbeat/module/kafka/_meta/config.reference.yml +++ b/metricbeat/module/kafka/_meta/config.reference.yml @@ -1,5 +1,5 @@ - module: kafka - metricsets: ["consumergroup", "partition"] + metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] period: 10s hosts: ["localhost:9092"] enabled: true diff --git a/metricbeat/module/kafka/_meta/config.yml b/metricbeat/module/kafka/_meta/config.yml index 9ed476ffe6ff..243bddd5e024 100644 --- a/metricbeat/module/kafka/_meta/config.yml +++ b/metricbeat/module/kafka/_meta/config.yml @@ -1,7 +1,10 @@ - module: kafka #metricsets: + # - broker # - partition + # - consumer # - consumergroup + # - producer period: 10s hosts: ["localhost:9092"] diff --git a/x-pack/metricbeat/module/kafka/broker/_meta/data.json b/metricbeat/module/kafka/broker/_meta/data.json similarity index 100% rename from x-pack/metricbeat/module/kafka/broker/_meta/data.json rename to metricbeat/module/kafka/broker/_meta/data.json diff --git a/x-pack/metricbeat/module/kafka/broker/_meta/docs.asciidoc b/metricbeat/module/kafka/broker/_meta/docs.asciidoc similarity index 100% rename from x-pack/metricbeat/module/kafka/broker/_meta/docs.asciidoc rename to metricbeat/module/kafka/broker/_meta/docs.asciidoc diff --git a/x-pack/metricbeat/module/kafka/broker/_meta/fields.yml b/metricbeat/module/kafka/broker/_meta/fields.yml similarity index 100% rename from x-pack/metricbeat/module/kafka/broker/_meta/fields.yml rename to metricbeat/module/kafka/broker/_meta/fields.yml diff --git a/x-pack/metricbeat/module/kafka/broker/manifest.yml b/metricbeat/module/kafka/broker/manifest.yml similarity index 100% rename from x-pack/metricbeat/module/kafka/broker/manifest.yml rename to metricbeat/module/kafka/broker/manifest.yml diff --git a/x-pack/metricbeat/module/kafka/consumer/_meta/data.json b/metricbeat/module/kafka/consumer/_meta/data.json similarity index 100% rename from x-pack/metricbeat/module/kafka/consumer/_meta/data.json rename to metricbeat/module/kafka/consumer/_meta/data.json diff --git a/x-pack/metricbeat/module/kafka/consumer/_meta/docs.asciidoc b/metricbeat/module/kafka/consumer/_meta/docs.asciidoc similarity index 100% rename from x-pack/metricbeat/module/kafka/consumer/_meta/docs.asciidoc rename to metricbeat/module/kafka/consumer/_meta/docs.asciidoc diff --git a/x-pack/metricbeat/module/kafka/consumer/_meta/fields.yml b/metricbeat/module/kafka/consumer/_meta/fields.yml similarity index 100% rename from x-pack/metricbeat/module/kafka/consumer/_meta/fields.yml rename to metricbeat/module/kafka/consumer/_meta/fields.yml diff --git a/x-pack/metricbeat/module/kafka/consumer/manifest.yml b/metricbeat/module/kafka/consumer/manifest.yml similarity index 100% rename from x-pack/metricbeat/module/kafka/consumer/manifest.yml rename to metricbeat/module/kafka/consumer/manifest.yml diff --git a/metricbeat/module/kafka/fields.go b/metricbeat/module/kafka/fields.go index 7a7d0fa0bb20..0aab1054bfd9 100644 --- a/metricbeat/module/kafka/fields.go +++ b/metricbeat/module/kafka/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKafka returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/kafka. func AssetKafka() string { - return "eJy8WM1u2zwQvPspFjk5h+g7fT34UKBNisJo0BZpezZocWUTpkiVpB347QtSli1RIvVjJzrFjDQzXC5nl3yAHR4XsCPZjswADDMcF3D3zf6+mwFQ1KlihWFSLODjDADA/Q9ySfccZwB6K5VZpVJkbLOAjHBtRxVyJBoXsLGwGUNO9cJ9/gCC5HihtI85FvZVJffFaaSDtwlTh1oruUN1Hu7CC2KWz2eHAI9S6H2OCr7aT2EpMqlyYj+ALTkgrBEFKCQUMiVzmJ8+2xJBORObBqTZIqQVnpNyn9Re8OdSnw+jjeFqPlx6FNEp1abF6KyTh1CqUOtOsh0eX6XyhQzjI/SAyjCN9EzRWjMjC5Ym9u/WurWpI7S/LY7DDHGgUlIlqaRtJi+ivTQOCixU0mYriDLMfps01m8s088KBhiNsrjZrTq44vFrkP0R7O8egVGQmcvY4sIu3EAZw34d5R58HzlABHW/StKkJa7ad74DjPaFEFDT3qontqdbHhXWU2oqFKbEIF3Ah+T/Sdvwlm4G/Y4WigDEnA1i7jZgqtBwuWokyN/teNDjeuN0+BxjLH6A6/qr+hQnchtmFNfgxKtZb1TCeRMPr2qDNVzcsi8QMss0mkmF9Zz0JQYwYWTNmtbIxMZtpLiCHA2ZuurpXhuZX5RYLKDEENBG1TdrJ3NnCRw+/3a6u+btrOa/SyzqFTIqKeUMRfdydPthVM8nrdlGID3BusWwi4IHFEaX/hZKwys8a6BdPJailk8wLwOn0Rgrr1SbMHrfb11bqf1wTRXSgAoS5piv/Yo+iZUJg0oQ7hWPE0F948YcY3Tt7gIZX7cjtjElTw+EcbLmeMLVVaOzYQcUtaZqZI4KfMVIekyvrd8dcGV8flfmy6yFjdO3EfTDAfcLmlCJJqznpfxYL36PbihWGQcIhtYJoxoNquRIaKtzHaC0V8ezA7bd/rzsjlvtZC1UOqzgqqbtmblssgTAqA4rUFhwlvoF/BZxeCmRuwMRjojQR5Gu+mStpeRI/IQfqGwpKLPZpoFlVQCAaWAi5XuKtDqVMfFgxVSvGLQVDubLXy+DZqJXPTn2JpOwwj3eoMRgAwU3WP8v556pbFQyNOnWtgdDbS1yzxHVd9Uh07RuRMLSvGseuN0ZpK1r6KnkdHcw5q7t+iO5x+ZpmXIfd72m4Hl19i8AAP//Xtd9qA==" + return "eJzUWk2P2zYQve+vGOS0OUQ5tYc9FGiTotgmaYI0BYpeBJocWexKpEJS3ji/viApybI+KEpeB82e1rI475HzweGjX8ADHu/ggWQP5AbAcFPgHTx7Yz8/uwFgqKnileFS3MFPNwAA7jsoJasLvAHQuVQmpVJkfH8HGSm0faqwQKLxDvbWbMaxYPrODX8BgpR4grR/5ljZV5Wsq+bJBO65mb6pnZIPqLrHU/Zmbfq/X5wFeCWFrktU8JsdCvcik6okdgDk5ICwQxSgkDDIlCzhthmWE8EKLvZnJk2OQFt7jsrzpPfCcC79+XB29ridTyEHEMEp9abF2c0kDmFModaTYA94fJRqSCQOj7ADKsM1sg5i5DMjK04T+//Ib2PoAOwna8fZnMNApaRKqGRjpMGKLsI4U2BNJWO0iijD7djkzH9rkT60ZoCzIIqbXTqBFV6/M7C/BP9cI3AGMnMRW53QhXvg13CZh8/Bb0MHiGDukwdNnqQgNLFbolGcap/gvtQ13/z+7u/e2K7A7dCQyLwud0hEKKPe2RfA5MSAybkGPKAwwLVFIwYZGBmdrC2ows81apPQnAiBRfK5xhoTzb9iiMmnHMG+0zqisQJudFx1GhKolGQ1xSQjvECWVqhSjVSKYI2xPBQxjocfCI2d1q6GChVMWooklqGh+XZatODWS85Kt07WWq3wEnJVwanbe5ICCUOVYoHUfh4W7BEz/z607zum29FrQQskIl3Lohn3BGw0am2ZfJXyAbFClTCuqRQCqVli8Y+Ub9wYoIW0O1JjbLtnxmzwS8XVYjqdmPj3r0LFNidSFMd4Mu2Ia7DRR0HXOMglUuPYGCpZIYmZ5FLIfZIVtc7TiXAbJ4zcg3t7PjjnoZpOBk2yOxrUKQ+WeAvHBZUlF3uwAzbkwxBR1otpIGuzl08IqfBfpAaXi2Tz3mW4JWpN9uvWthkTidnitR37ut6hOzdMdA/dd99p/+D2tagsKrngZV36jZEYeMw5zc/PQRoF0+dbpQYjgYxbNohIPR+LjfXFWCQHVDYiRF3uUNmt241v2THIpAICukLKM06bXnNzCVJIpWKX0GssnAieuExy3bR6yynVNjrtarn0sv24PHPu+iQb5tTqY/ucoXP1of0LpVYg/IbGPadKIbUpdQc/Jj9sOiU/pdgAy4LD3ApASHiAkPgQMVU4EyHaJ7P404IELIgS63gMMdYoMBGiyNCrr8NA83k7hxUdeD1lJEihO2PH7cyrOJzEjKWFkFmmcdjGxOleXdB7G8CFkT3lYIe2XtlECjMozzdiWON1WmsjyxMTawsYMQS0Uf1knUSeVKji5z8Od6etdmxentaiL2AFKflefEU9DPL5WWu+F8jaFt86wzrF9SdNtzQXhhfUrMhy8cqTun8Nt37hNBpj6Xm2CWfPl0tXLvVwubYSOTM1C1iibRPSy6fPhUElSDHYPBqAfuKGKsbqvXvKyPp9O1A2tsTpgfCC7Aps7OpW/trzA4qe5rkyRgU+YiA8tu+tfzjDbeEbiqZDmr1lK9h1CL13hpcJbdiJNvjztP3YWvwtuqHQzhhBGEYXAO3TWZZeXruCK9963Y4zuPXd8aid7C2VnmdwUdP2lrtosgDAmZ5n0CiWV1iHj97y9ELMr4jQR0HTJVo7KYvxoT6S2b1g3EabBp61C2BP+1zQombI2ksTLl5YMp2qi3aHg9v7Pz9GzUSnCzF2lUmYTshepjjbQMET+P/XrmfyjYpTL2x7EFvWAteQQX4XHTLN6MJyntrgFhae7gwy5hV7Kmmu9tZchV9+JOfTEljDZct1+eWcZs+r47bMX4yt1C4/NKOmtMvuu+9UuyRtP5fu6ixDlToJK3hodnfMhhRASlkLt/f4sbYflmp4sbIgX++IoXmq+VdMySF4iAsJlHruKBbQ9k7AJfmyBFySL065jQYOXBxSqViqUbAozXhe9LTYl4qvqUKjjpuJGMWRNaYa6fpSQq4g/48IaXfh3Ujx13XWYsyszJJ2Gca/ZIj0xdrsWAJc+qFBlNtPa9uW8+gfOoQmrCspNG5n4MdfQIHL9JHwxRDrIO9fvgc7AAyfaU6WLjYibkXPbzb8BamsjbtcMT028+7+LwAA///1XQeS" } diff --git a/x-pack/metricbeat/module/kafka/module.yml b/metricbeat/module/kafka/module.yml similarity index 100% rename from x-pack/metricbeat/module/kafka/module.yml rename to metricbeat/module/kafka/module.yml diff --git a/x-pack/metricbeat/module/kafka/producer/_meta/data.json b/metricbeat/module/kafka/producer/_meta/data.json similarity index 100% rename from x-pack/metricbeat/module/kafka/producer/_meta/data.json rename to metricbeat/module/kafka/producer/_meta/data.json diff --git a/x-pack/metricbeat/module/kafka/producer/_meta/docs.asciidoc b/metricbeat/module/kafka/producer/_meta/docs.asciidoc similarity index 100% rename from x-pack/metricbeat/module/kafka/producer/_meta/docs.asciidoc rename to metricbeat/module/kafka/producer/_meta/docs.asciidoc diff --git a/x-pack/metricbeat/module/kafka/producer/_meta/fields.yml b/metricbeat/module/kafka/producer/_meta/fields.yml similarity index 100% rename from x-pack/metricbeat/module/kafka/producer/_meta/fields.yml rename to metricbeat/module/kafka/producer/_meta/fields.yml diff --git a/x-pack/metricbeat/module/kafka/producer/manifest.yml b/metricbeat/module/kafka/producer/manifest.yml similarity index 100% rename from x-pack/metricbeat/module/kafka/producer/manifest.yml rename to metricbeat/module/kafka/producer/manifest.yml diff --git a/metricbeat/modules.d/kafka.yml.disabled b/metricbeat/modules.d/kafka.yml.disabled index 106e7fec6809..622c4c85a814 100644 --- a/metricbeat/modules.d/kafka.yml.disabled +++ b/metricbeat/modules.d/kafka.yml.disabled @@ -3,8 +3,11 @@ - module: kafka #metricsets: + # - broker # - partition + # - consumer # - consumergroup + # - producer period: 10s hosts: ["localhost:9092"] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 306e384d569e..fd1d8f95ed05 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -24,7 +24,6 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/cockroachdb" _ "github.com/elastic/beats/x-pack/metricbeat/module/coredns" _ "github.com/elastic/beats/x-pack/metricbeat/module/coredns/stats" - _ "github.com/elastic/beats/x-pack/metricbeat/module/kafka" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/transaction_log" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 463421ecd2b0..fcda6b75ebc1 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -476,38 +476,10 @@ metricbeat.modules: jmx.instance: #-------------------------------- Kafka Module -------------------------------- -- module: kafka - metricsets: ["consumergroup", "partition"] - period: 10s - hosts: ["localhost:9092"] - enabled: true - - #client_id: metricbeat - #retries: 3 - #backoff: 250ms - - # List of Topics to query metadata for. If empty, all topics will be queried. - #topics: [] - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # SASL authentication - #username: "" - #password: "" - -#------------------------------- KafkaJmx Module ------------------------------- - module: kafka metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] period: 10s - hosts: ["localhost:8779"] + hosts: ["localhost:9092"] enabled: true #client_id: metricbeat diff --git a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml b/x-pack/metricbeat/module/kafka/_meta/config.reference.yml deleted file mode 100644 index efc34ff0bd80..000000000000 --- a/x-pack/metricbeat/module/kafka/_meta/config.reference.yml +++ /dev/null @@ -1,26 +0,0 @@ -- module: kafka - metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] - period: 10s - hosts: ["localhost:8779"] - enabled: true - - #client_id: metricbeat - #retries: 3 - #backoff: 250ms - - # List of Topics to query metadata for. If empty, all topics will be queried. - #topics: [] - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # SASL authentication - #username: "" - #password: "" diff --git a/x-pack/metricbeat/module/kafka/_meta/config.yml b/x-pack/metricbeat/module/kafka/_meta/config.yml deleted file mode 100644 index 06259150ed4e..000000000000 --- a/x-pack/metricbeat/module/kafka/_meta/config.yml +++ /dev/null @@ -1,30 +0,0 @@ -- module: kafka - #metricsets: - # - broker - # - partition - # - consumer - # - consumergroup - # - producer - - period: 10s - hosts: ["localhost:8779"] - #client_id: metricbeat - #retries: 3 - #backoff: 250ms - - # List of Topics to query metadata for. If empty, all topics will be queried. - #topics: [] - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # SASL authentication - #username: "" - #password: "" diff --git a/x-pack/metricbeat/module/kafka/_meta/fields.yml b/x-pack/metricbeat/module/kafka/_meta/fields.yml deleted file mode 100644 index 417cf08c70de..000000000000 --- a/x-pack/metricbeat/module/kafka/_meta/fields.yml +++ /dev/null @@ -1,9 +0,0 @@ -- key: kafkajmx - title: 'KafkaJmx' - release: beta - description: > - KafkaJmx module - fields: - - name: KafkaJmx - type: group - fields: diff --git a/x-pack/metricbeat/module/kafka/fields.go b/x-pack/metricbeat/module/kafka/fields.go deleted file mode 100644 index 3fb8dc611432..000000000000 --- a/x-pack/metricbeat/module/kafka/fields.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package kafka - -import ( - "github.com/elastic/beats/libbeat/asset" -) - -func init() { - if err := asset.SetFields("metricbeat", "kafka", asset.ModuleFieldsPri, AssetKafka); err != nil { - panic(err) - } -} - -// AssetKafka returns asset data. -// This is the base64 encoded gzipped contents of module/kafka. -func AssetKafka() string { - return "eJzUlz+T2zYQxXt9iu1cmelVpEiq2ONJihSZNBwIfBRhAVgaf+4kf/oMQFLS6XjiH/lmYnUi8LA/YvEWy490wGlLB1EfxFdz3BAFFTS29OFzevTJHD9siBw0hMeWdghiQ1TBS6faoNhu6dcNEdEwnQxXUWNDVCvoym/z6EeywmB7npUfEoVTiy3tHce2f3ItuhbuHB/gzo/HpOn3Auy3rCGD4JT0VDs2HcAw8unLP1fa25ccfrdI11hmB2FfjNwwfEkTKDQiUGiUJzzBBlI+RRMBFQW+kXcvdsDpmV01GtThW4QPhWyEtdDFt4iIwqvvuEfydwNKc4hrCg2GVSirRxk02/1dgNZxFSWKWiiNqmzhSg/JtpricCJkjk5I/TrDup5aOBpdaSZYjSCb9VhSq5SlvMp5n9Jq0eERuFYrKVK4QkNUcCU0ZPrvp8i6+TTMz6Tro0crNYQtl1L0uh9A4+F9IvnOfABauKJSXrK1kGGK4l/mz1lDUrNHNSy2PjOvaXBslZu004Wkm/8uKA6iYqtP82EGxXvQ+JOVSxKUjdQndg5KrVmEURbN+6LW0TflyHF7bRjeU5799uF8O1TgVsnCIhS7U4Av1d0Sn8IpK9kou6ckWOGH24gcJ23AMez5B4Z0+AoZMF0k+3mPxTXwXuyX7W2vmRlziCfZ+miW9g6/96qx7uE89pP2D/lem+Uio6wy0XQXowj03CjZ5O5h2FXysJV/eVV6CkzidctGM6zXncV+9cmzKJ7g0omw0ezg0tWd9QNdRTU7EuRbSFUr2Z299SXIQbKrHsHrV7gAXlhGWVft3rSlhkZn2K1sL2VT3q6TO99kfQe30GR/9aoxk53HflKTiSehtNhplLtY13Bl3uupxAQOQpMwHG3ICcpaMjDsbjuAiTq7E8nlqeMvxdN+rZM87O1NNHkIL4GNOE6WGHHMJWZ24DsdbvJWmQrSrOL2tjtT7EerROkQ3Gk1SHAKVb9UX2MfBYJz7P5HQD5/mfV3xvsma/LMLHTJsA2vP7ln5mKpO6YCTn0Rz0r7ZW+Hcj77i/zeC/uWrcd6gk7/AILi8lmoySN2DvnHL39SElBQZml+Z7fvL6/grpPnGHIXEK5o3k73fwEAAP//6F+3kA==" -} diff --git a/x-pack/metricbeat/modules.d/kafka.yml.disabled b/x-pack/metricbeat/modules.d/kafka.yml.disabled deleted file mode 100644 index 312ab7d068e6..000000000000 --- a/x-pack/metricbeat/modules.d/kafka.yml.disabled +++ /dev/null @@ -1,33 +0,0 @@ -# Module: kafka -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-kafka.html - -- module: kafka - #metricsets: - # - broker - # - partition - # - consumer - # - consumergroup - # - producer - - period: 10s - hosts: ["localhost:8779"] - #client_id: metricbeat - #retries: 3 - #backoff: 250ms - - # List of Topics to query metadata for. If empty, all topics will be queried. - #topics: [] - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # SASL authentication - #username: "" - #password: "" From 9426d3f11f0d6f1cd26af3c2466d28ae9e0db1fe Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 12 Nov 2019 12:36:01 +0200 Subject: [PATCH 12/29] Add go files Signed-off-by: chrismark --- metricbeat/module/kafka/broker/broker.go | 18 ++++++++++++++++++ metricbeat/module/kafka/consumer/consumer.go | 18 ++++++++++++++++++ metricbeat/module/kafka/producer/producer.go | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 metricbeat/module/kafka/broker/broker.go create mode 100644 metricbeat/module/kafka/consumer/consumer.go create mode 100644 metricbeat/module/kafka/producer/producer.go diff --git a/metricbeat/module/kafka/broker/broker.go b/metricbeat/module/kafka/broker/broker.go new file mode 100644 index 000000000000..6dec23581b5f --- /dev/null +++ b/metricbeat/module/kafka/broker/broker.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package broker diff --git a/metricbeat/module/kafka/consumer/consumer.go b/metricbeat/module/kafka/consumer/consumer.go new file mode 100644 index 000000000000..6c4bce3ac878 --- /dev/null +++ b/metricbeat/module/kafka/consumer/consumer.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package consumer diff --git a/metricbeat/module/kafka/producer/producer.go b/metricbeat/module/kafka/producer/producer.go new file mode 100644 index 000000000000..76ffd9ba5f8e --- /dev/null +++ b/metricbeat/module/kafka/producer/producer.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package producer From 7006b1491390cfed24357dcc537f89c8aa48463a Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 15 Nov 2019 10:50:29 +0200 Subject: [PATCH 13/29] Remove go files and update docs and list.go Signed-off-by: chrismark --- metricbeat/docs/modules/kafka/broker.asciidoc | 1 + .../docs/modules/kafka/consumer.asciidoc | 1 + .../docs/modules/kafka/producer.asciidoc | 1 + metricbeat/include/list_common.go | 3 --- metricbeat/module/kafka/broker/broker.go | 18 ------------------ metricbeat/module/kafka/consumer/consumer.go | 18 ------------------ metricbeat/module/kafka/producer/producer.go | 18 ------------------ 7 files changed, 3 insertions(+), 57 deletions(-) delete mode 100644 metricbeat/module/kafka/broker/broker.go delete mode 100644 metricbeat/module/kafka/consumer/consumer.go delete mode 100644 metricbeat/module/kafka/producer/producer.go diff --git a/metricbeat/docs/modules/kafka/broker.asciidoc b/metricbeat/docs/modules/kafka/broker.asciidoc index 191351eb98f7..9ef1c702ce2a 100644 --- a/metricbeat/docs/modules/kafka/broker.asciidoc +++ b/metricbeat/docs/modules/kafka/broker.asciidoc @@ -9,6 +9,7 @@ beta[] include::../../../module/kafka/broker/_meta/docs.asciidoc[] +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/kafka/consumer.asciidoc b/metricbeat/docs/modules/kafka/consumer.asciidoc index 3a2acd51cda7..a30bd9001188 100644 --- a/metricbeat/docs/modules/kafka/consumer.asciidoc +++ b/metricbeat/docs/modules/kafka/consumer.asciidoc @@ -9,6 +9,7 @@ beta[] include::../../../module/kafka/consumer/_meta/docs.asciidoc[] +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/kafka/producer.asciidoc b/metricbeat/docs/modules/kafka/producer.asciidoc index 2ce5ab0a3f43..d7e9e6b9d536 100644 --- a/metricbeat/docs/modules/kafka/producer.asciidoc +++ b/metricbeat/docs/modules/kafka/producer.asciidoc @@ -9,6 +9,7 @@ beta[] include::../../../module/kafka/producer/_meta/docs.asciidoc[] +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index aa46bda73c6d..26db25d57441 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -79,11 +79,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" - _ "github.com/elastic/beats/metricbeat/module/kafka/broker" - _ "github.com/elastic/beats/metricbeat/module/kafka/consumer" _ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/metricbeat/module/kafka/partition" - _ "github.com/elastic/beats/metricbeat/module/kafka/producer" _ "github.com/elastic/beats/metricbeat/module/kibana" _ "github.com/elastic/beats/metricbeat/module/kibana/stats" _ "github.com/elastic/beats/metricbeat/module/kibana/status" diff --git a/metricbeat/module/kafka/broker/broker.go b/metricbeat/module/kafka/broker/broker.go deleted file mode 100644 index 6dec23581b5f..000000000000 --- a/metricbeat/module/kafka/broker/broker.go +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package broker diff --git a/metricbeat/module/kafka/consumer/consumer.go b/metricbeat/module/kafka/consumer/consumer.go deleted file mode 100644 index 6c4bce3ac878..000000000000 --- a/metricbeat/module/kafka/consumer/consumer.go +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package consumer diff --git a/metricbeat/module/kafka/producer/producer.go b/metricbeat/module/kafka/producer/producer.go deleted file mode 100644 index 76ffd9ba5f8e..000000000000 --- a/metricbeat/module/kafka/producer/producer.go +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package producer From 3543af074b85610893649784b1b5322c4293de23 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 15 Nov 2019 11:10:29 +0200 Subject: [PATCH 14/29] Add extra docs Signed-off-by: chrismark --- metricbeat/module/kafka/_meta/run.sh | 4 +--- metricbeat/module/kafka/broker/_meta/docs.asciidoc | 11 +++++++++++ metricbeat/module/kafka/consumer/_meta/docs.asciidoc | 11 +++++++++++ metricbeat/module/kafka/producer/_meta/docs.asciidoc | 11 +++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index fffc82aef92a..7575afee3d82 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -59,9 +59,6 @@ wait_for_port 9092 wait_for_port 8779 -KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-producer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8775,host=0.0.0.0" \ - - echo "Kafka load status code $?" # ACLS used to prepare tests @@ -81,6 +78,7 @@ cat /dev/urandom | KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas wait_for_port 8775 +# Start a forever consumer KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-consumer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8774,host=0.0.0.0" \ ${KAFKA_HOME}/bin/kafka-console-consumer.sh --topic=test --bootstrap-server=localhost:9091 --consumer.config ${KAFKA_HOME}/bin/sasl-producer.properties > /dev/null & diff --git a/metricbeat/module/kafka/broker/_meta/docs.asciidoc b/metricbeat/module/kafka/broker/_meta/docs.asciidoc index d6159805492a..8f4187ba8def 100644 --- a/metricbeat/module/kafka/broker/_meta/docs.asciidoc +++ b/metricbeat/module/kafka/broker/_meta/docs.asciidoc @@ -7,3 +7,14 @@ The module has been tested with Kafka 2.1.1. Other versions are expected to work [float] === Usage The Broker metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. + +Note that the Jolokia agent is required to be deployed along with the Kafka JVM application. This can be achieved by +using the `KAFKA_OPTS` environment variable when starting the Kafka broker application: + +[source,shell] +---- +export KAFKA_OPTS=-javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8779,host=localhost +./bin/kafka-server-start.sh ./config/server.properties +---- + +Then it will be possible to collect the JMX metrics from `localhost:8779`. diff --git a/metricbeat/module/kafka/consumer/_meta/docs.asciidoc b/metricbeat/module/kafka/consumer/_meta/docs.asciidoc index 9f3390c36fad..0fc1c0559933 100644 --- a/metricbeat/module/kafka/consumer/_meta/docs.asciidoc +++ b/metricbeat/module/kafka/consumer/_meta/docs.asciidoc @@ -7,3 +7,14 @@ The module has been tested with Kafka 2.1.1. Other versions are expected to work [float] === Usage The Consumer metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. + +Note that the Jolokia agent is required to be deployed along with the Kafka JVM application. This can be achieved by +using the `KAFKA_OPTS` environment variable when starting the Kafka consumer application: + +[source,shell] +---- +export KAFKA_OPTS=-javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8774,host=localhost +./bin/kafka-console-consumer.sh --topic=test --bootstrap-server=localhost:9091 +---- + +Then it will be possible to collect the JMX metrics from `localhost:8774`. diff --git a/metricbeat/module/kafka/producer/_meta/docs.asciidoc b/metricbeat/module/kafka/producer/_meta/docs.asciidoc index 6bf7ddb8992f..a542539a3289 100644 --- a/metricbeat/module/kafka/producer/_meta/docs.asciidoc +++ b/metricbeat/module/kafka/producer/_meta/docs.asciidoc @@ -7,3 +7,14 @@ The module has been tested with Kafka 2.1.1. Other versions are expected to work [float] === Usage The Producer metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. + +Note that the Jolokia agent is required to be deployed along with the Kafka JVM application. This can be achieved by +using the `KAFKA_OPTS` environment variable when starting the Kafka producer application: + +[source,shell] +---- +export KAFKA_OPTS=-javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8775,host=localhost +./bin/kafka-console-producer.sh --topic test --broker-list localhost:9091 +---- + +Then it will be possible to collect the JMX metrics from `localhost:8775`. From 100b68cc60eeb6ef7db7b1485dbae0269523670a Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 15 Nov 2019 15:59:58 +0200 Subject: [PATCH 15/29] Address review comments Signed-off-by: chrismark --- metricbeat/docs/modules/kafka.asciidoc | 34 +++++++++++++++++-- metricbeat/metricbeat.reference.yml | 28 +++++++++++++-- metricbeat/module/kafka/_meta/Dockerfile | 6 ++-- .../module/kafka/_meta/config.reference.yml | 28 +++++++++++++-- metricbeat/module/kafka/_meta/config.yml | 24 +++++++++++-- metricbeat/module/kafka/_meta/docs.asciidoc | 6 ++++ .../module/kafka/consumer/_meta/docs.asciidoc | 6 ++-- .../module/kafka/producer/_meta/docs.asciidoc | 6 ++-- metricbeat/modules.d/kafka.yml.disabled | 24 +++++++++++-- x-pack/metricbeat/metricbeat.reference.yml | 28 +++++++++++++-- 10 files changed, 167 insertions(+), 23 deletions(-) diff --git a/metricbeat/docs/modules/kafka.asciidoc b/metricbeat/docs/modules/kafka.asciidoc index 8f23804cbf1f..a0432c5bd7d4 100644 --- a/metricbeat/docs/modules/kafka.asciidoc +++ b/metricbeat/docs/modules/kafka.asciidoc @@ -29,6 +29,12 @@ kafka-acls --authorizer-properties zookeeper.connect=localhost:2181 --add --allo This module is tested with Kafka 0.10.2.1, 1.1.0 and 2.1.1. +The Broker, Producer, Consumer metricsets require <>to fetch JMX metrics. Refer to the link for Jolokia's compatibility notes. + +[float] +=== Usage +The Broker, Producer, Consumer metricsets require <>to fetch JMX metrics. Refer to those Metricsets' documentation about how to use Jolokia. + [float] === Example configuration @@ -39,11 +45,14 @@ in <>. Here is an example configuration: [source,yaml] ---- metricbeat.modules: +# Kafka metrics collected using the Kafka protocol - module: kafka - metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] + enabled: true + metricsets: + - consumergroup + - partition period: 10s hosts: ["localhost:9092"] - enabled: true #client_id: metricbeat #retries: 3 @@ -65,6 +74,27 @@ metricbeat.modules: # SASL authentication #username: "" #password: "" + +# Metrics collected from a Kafka broker using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['broker'] +# period: 10s +# hosts: ['localhost:8779'] + +# Metrics collected from a Java Kafka consumer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['consumer'] +# period: 10s +# hosts: ['localhost:8775'] + +# Metrics collected from a Java Kafka producer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['producer'] +# period: 10s +# hosts: ['localhost:8774'] ---- [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 1e777842d36b..5bd7606332e8 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -374,11 +374,14 @@ metricbeat.modules: jmx.instance: #-------------------------------- Kafka Module -------------------------------- +# Kafka metrics collected using the Kafka protocol - module: kafka - metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] + enabled: true + metricsets: + - consumergroup + - partition period: 10s hosts: ["localhost:9092"] - enabled: true #client_id: metricbeat #retries: 3 @@ -401,6 +404,27 @@ metricbeat.modules: #username: "" #password: "" +# Metrics collected from a Kafka broker using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['broker'] +# period: 10s +# hosts: ['localhost:8779'] + +# Metrics collected from a Java Kafka consumer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['consumer'] +# period: 10s +# hosts: ['localhost:8775'] + +# Metrics collected from a Java Kafka producer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['producer'] +# period: 10s +# hosts: ['localhost:8774'] + #-------------------------------- Kibana Module -------------------------------- - module: kibana metricsets: ["status"] diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index 312bf9175f3f..f8e2b71c8474 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -8,7 +8,7 @@ ENV KAFKA_LOGS_DIR="/kafka-logs" ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true" ENV TERM=linux -RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils wget +RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \ curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \ @@ -16,8 +16,8 @@ RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \ "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \ tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1 -RUN wget -O /opt/jolokia-jvm-1.5.0-agent.jar http://search.maven.org/remotecontent\?filepath\=org/jolokia/jolokia-jvm/1.5.0/jolokia-jvm-1.5.0-agent.jar -# RUN mv jolokia-jvm-1.5.0-agent.jar /opt/jolokia-jvm-1.5.0-agent.jar +RUN retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o /opt/jolokia-jvm-1.5.0-agent.jar \ + http://search.maven.org/remotecontent\?filepath\=org/jolokia/jolokia-jvm/1.5.0/jolokia-jvm-1.5.0-agent.jar ADD kafka_server_jaas.conf /etc/kafka/server_jaas.conf ADD jaas-kafka-client-producer.conf /kafka/bin/jaas-kafka-client-producer.conf diff --git a/metricbeat/module/kafka/_meta/config.reference.yml b/metricbeat/module/kafka/_meta/config.reference.yml index 2908f870b4c1..b5c95146a01e 100644 --- a/metricbeat/module/kafka/_meta/config.reference.yml +++ b/metricbeat/module/kafka/_meta/config.reference.yml @@ -1,8 +1,11 @@ +# Kafka metrics collected using the Kafka protocol - module: kafka - metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] + enabled: true + metricsets: + - consumergroup + - partition period: 10s hosts: ["localhost:9092"] - enabled: true #client_id: metricbeat #retries: 3 @@ -24,3 +27,24 @@ # SASL authentication #username: "" #password: "" + +# Metrics collected from a Kafka broker using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['broker'] +# period: 10s +# hosts: ['localhost:8779'] + +# Metrics collected from a Java Kafka consumer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['consumer'] +# period: 10s +# hosts: ['localhost:8775'] + +# Metrics collected from a Java Kafka producer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['producer'] +# period: 10s +# hosts: ['localhost:8774'] diff --git a/metricbeat/module/kafka/_meta/config.yml b/metricbeat/module/kafka/_meta/config.yml index 243bddd5e024..a3560d8dc653 100644 --- a/metricbeat/module/kafka/_meta/config.yml +++ b/metricbeat/module/kafka/_meta/config.yml @@ -1,10 +1,7 @@ - module: kafka #metricsets: - # - broker # - partition - # - consumer # - consumergroup - # - producer period: 10s hosts: ["localhost:9092"] @@ -28,3 +25,24 @@ # SASL authentication #username: "" #password: "" + +# Metrics collected from a Kafka broker using Jolokia +#- module: kafka +# metricsets: +# - broker +# period: 10s +# hosts: ["localhost:8779"] + +# Metrics collected from a Java Kafka consumer using Jolokia +#- module: kafka +# metricsets: +# - consumer +# period: 10s +# hosts: ["localhost:8775"] + +# Metrics collected from a Java Kafka producer using Jolokia +#- module: kafka +# metricsets: +# - producer +# period: 10s +# hosts: ["localhost:8774"] diff --git a/metricbeat/module/kafka/_meta/docs.asciidoc b/metricbeat/module/kafka/_meta/docs.asciidoc index 89f8e6a3f827..d6a59cbb8aef 100644 --- a/metricbeat/module/kafka/_meta/docs.asciidoc +++ b/metricbeat/module/kafka/_meta/docs.asciidoc @@ -21,3 +21,9 @@ kafka-acls --authorizer-properties zookeeper.connect=localhost:2181 --add --allo === Compatibility This module is tested with Kafka 0.10.2.1, 1.1.0 and 2.1.1. + +The Broker, Producer, Consumer metricsets require <>to fetch JMX metrics. Refer to the link for Jolokia's compatibility notes. + +[float] +=== Usage +The Broker, Producer, Consumer metricsets require <>to fetch JMX metrics. Refer to those Metricsets' documentation about how to use Jolokia. diff --git a/metricbeat/module/kafka/consumer/_meta/docs.asciidoc b/metricbeat/module/kafka/consumer/_meta/docs.asciidoc index 0fc1c0559933..0fd42e7d26e7 100644 --- a/metricbeat/module/kafka/consumer/_meta/docs.asciidoc +++ b/metricbeat/module/kafka/consumer/_meta/docs.asciidoc @@ -1,4 +1,4 @@ -This metricset periodically fetches JMX metrics from Kafka Producers JMX. +This metricset periodically fetches JMX metrics from Kafka Consumers implemented in java and expose JMX metrics through jolokia agent. [float] === Compatibility @@ -6,9 +6,9 @@ The module has been tested with Kafka 2.1.1. Other versions are expected to work [float] === Usage -The Consumer metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. +The Consumer metricset requires <>to fetch JMX metrics. Refer to the link for more information about Jolokia. -Note that the Jolokia agent is required to be deployed along with the Kafka JVM application. This can be achieved by +Note that the Jolokia agent is required to be deployed along with the JVM application. This can be achieved by using the `KAFKA_OPTS` environment variable when starting the Kafka consumer application: [source,shell] diff --git a/metricbeat/module/kafka/producer/_meta/docs.asciidoc b/metricbeat/module/kafka/producer/_meta/docs.asciidoc index a542539a3289..29bd793c708a 100644 --- a/metricbeat/module/kafka/producer/_meta/docs.asciidoc +++ b/metricbeat/module/kafka/producer/_meta/docs.asciidoc @@ -1,4 +1,4 @@ -This metricset periodically fetches JMX metrics from Kafka Producers JMX. +This metricset periodically fetches JMX metrics from Kafka Producers implemented in java and expose JMX metrics through jolokia agent. [float] === Compatibility @@ -6,9 +6,9 @@ The module has been tested with Kafka 2.1.1. Other versions are expected to work [float] === Usage -The Producer metricset requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. +The Producer metricset requires <>to fetch JMX metrics. Refer to the link for more information about Jolokia. -Note that the Jolokia agent is required to be deployed along with the Kafka JVM application. This can be achieved by +Note that the Jolokia agent is required to be deployed along with the JVM application. This can be achieved by using the `KAFKA_OPTS` environment variable when starting the Kafka producer application: [source,shell] diff --git a/metricbeat/modules.d/kafka.yml.disabled b/metricbeat/modules.d/kafka.yml.disabled index 622c4c85a814..491284e937fa 100644 --- a/metricbeat/modules.d/kafka.yml.disabled +++ b/metricbeat/modules.d/kafka.yml.disabled @@ -3,11 +3,8 @@ - module: kafka #metricsets: - # - broker # - partition - # - consumer # - consumergroup - # - producer period: 10s hosts: ["localhost:9092"] @@ -31,3 +28,24 @@ # SASL authentication #username: "" #password: "" + +# Metrics collected from a Kafka broker using Jolokia +#- module: kafka +# metricsets: +# - broker +# period: 10s +# hosts: ["localhost:8779"] + +# Metrics collected from a Java Kafka consumer using Jolokia +#- module: kafka +# metricsets: +# - consumer +# period: 10s +# hosts: ["localhost:8775"] + +# Metrics collected from a Java Kafka producer using Jolokia +#- module: kafka +# metricsets: +# - producer +# period: 10s +# hosts: ["localhost:8774"] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index fcda6b75ebc1..5750e36cc0fd 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -476,11 +476,14 @@ metricbeat.modules: jmx.instance: #-------------------------------- Kafka Module -------------------------------- +# Kafka metrics collected using the Kafka protocol - module: kafka - metricsets: ["broker", "consumer", "consumergroup", "partition", "producer"] + enabled: true + metricsets: + - consumergroup + - partition period: 10s hosts: ["localhost:9092"] - enabled: true #client_id: metricbeat #retries: 3 @@ -503,6 +506,27 @@ metricbeat.modules: #username: "" #password: "" +# Metrics collected from a Kafka broker using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['broker'] +# period: 10s +# hosts: ['localhost:8779'] + +# Metrics collected from a Java Kafka consumer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['consumer'] +# period: 10s +# hosts: ['localhost:8775'] + +# Metrics collected from a Java Kafka producer using Jolokia +#- module: kafka +# enabled: true +# metricsets: ['producer'] +# period: 10s +# hosts: ['localhost:8774'] + #-------------------------------- Kibana Module -------------------------------- - module: kibana metricsets: ["status"] From 923bb3df69f78f15c57793598663cdc676b1ccd0 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 15 Nov 2019 17:10:34 +0200 Subject: [PATCH 16/29] Add changelog entry Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7d3828b8ea99..5a9f257d324b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -481,6 +481,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221] - Add `connection.state` field for RabbitMQ module. {pull}13981[13981] - Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347] +- Add Kafka JMX metricsets. {pull}14330[14330] *Packetbeat* From 77d65fbfc54faca7512eef045de3ee37881429e6 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 18 Nov 2019 11:17:04 +0200 Subject: [PATCH 17/29] Remove congig.reference file Signed-off-by: chrismark --- metricbeat/docs/modules/kafka.asciidoc | 25 +++++----- metricbeat/metricbeat.reference.yml | 25 +++++----- .../module/kafka/_meta/config.reference.yml | 50 ------------------- metricbeat/module/kafka/_meta/config.yml | 1 + metricbeat/modules.d/kafka.yml.disabled | 1 + 5 files changed, 26 insertions(+), 76 deletions(-) delete mode 100644 metricbeat/module/kafka/_meta/config.reference.yml diff --git a/metricbeat/docs/modules/kafka.asciidoc b/metricbeat/docs/modules/kafka.asciidoc index a0432c5bd7d4..5d6d45c28ef0 100644 --- a/metricbeat/docs/modules/kafka.asciidoc +++ b/metricbeat/docs/modules/kafka.asciidoc @@ -47,10 +47,9 @@ in <>. Here is an example configuration: metricbeat.modules: # Kafka metrics collected using the Kafka protocol - module: kafka - enabled: true - metricsets: - - consumergroup - - partition + #metricsets: + # - partition + # - consumergroup period: 10s hosts: ["localhost:9092"] @@ -77,24 +76,24 @@ metricbeat.modules: # Metrics collected from a Kafka broker using Jolokia #- module: kafka -# enabled: true -# metricsets: ['broker'] +# metricsets: +# - broker # period: 10s -# hosts: ['localhost:8779'] +# hosts: ["localhost:8779"] # Metrics collected from a Java Kafka consumer using Jolokia #- module: kafka -# enabled: true -# metricsets: ['consumer'] +# metricsets: +# - consumer # period: 10s -# hosts: ['localhost:8775'] +# hosts: ["localhost:8775"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka -# enabled: true -# metricsets: ['producer'] +# metricsets: +# - producer # period: 10s -# hosts: ['localhost:8774'] +# hosts: ["localhost:8774"] ---- [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 5bd7606332e8..11f8b5db1323 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -376,10 +376,9 @@ metricbeat.modules: #-------------------------------- Kafka Module -------------------------------- # Kafka metrics collected using the Kafka protocol - module: kafka - enabled: true - metricsets: - - consumergroup - - partition + #metricsets: + # - partition + # - consumergroup period: 10s hosts: ["localhost:9092"] @@ -406,24 +405,24 @@ metricbeat.modules: # Metrics collected from a Kafka broker using Jolokia #- module: kafka -# enabled: true -# metricsets: ['broker'] +# metricsets: +# - broker # period: 10s -# hosts: ['localhost:8779'] +# hosts: ["localhost:8779"] # Metrics collected from a Java Kafka consumer using Jolokia #- module: kafka -# enabled: true -# metricsets: ['consumer'] +# metricsets: +# - consumer # period: 10s -# hosts: ['localhost:8775'] +# hosts: ["localhost:8775"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka -# enabled: true -# metricsets: ['producer'] +# metricsets: +# - producer # period: 10s -# hosts: ['localhost:8774'] +# hosts: ["localhost:8774"] #-------------------------------- Kibana Module -------------------------------- - module: kibana diff --git a/metricbeat/module/kafka/_meta/config.reference.yml b/metricbeat/module/kafka/_meta/config.reference.yml deleted file mode 100644 index b5c95146a01e..000000000000 --- a/metricbeat/module/kafka/_meta/config.reference.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Kafka metrics collected using the Kafka protocol -- module: kafka - enabled: true - metricsets: - - consumergroup - - partition - period: 10s - hosts: ["localhost:9092"] - - #client_id: metricbeat - #retries: 3 - #backoff: 250ms - - # List of Topics to query metadata for. If empty, all topics will be queried. - #topics: [] - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # SASL authentication - #username: "" - #password: "" - -# Metrics collected from a Kafka broker using Jolokia -#- module: kafka -# enabled: true -# metricsets: ['broker'] -# period: 10s -# hosts: ['localhost:8779'] - -# Metrics collected from a Java Kafka consumer using Jolokia -#- module: kafka -# enabled: true -# metricsets: ['consumer'] -# period: 10s -# hosts: ['localhost:8775'] - -# Metrics collected from a Java Kafka producer using Jolokia -#- module: kafka -# enabled: true -# metricsets: ['producer'] -# period: 10s -# hosts: ['localhost:8774'] diff --git a/metricbeat/module/kafka/_meta/config.yml b/metricbeat/module/kafka/_meta/config.yml index a3560d8dc653..b648192bda03 100644 --- a/metricbeat/module/kafka/_meta/config.yml +++ b/metricbeat/module/kafka/_meta/config.yml @@ -1,3 +1,4 @@ +# Kafka metrics collected using the Kafka protocol - module: kafka #metricsets: # - partition diff --git a/metricbeat/modules.d/kafka.yml.disabled b/metricbeat/modules.d/kafka.yml.disabled index 491284e937fa..5cf353232820 100644 --- a/metricbeat/modules.d/kafka.yml.disabled +++ b/metricbeat/modules.d/kafka.yml.disabled @@ -1,6 +1,7 @@ # Module: kafka # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-kafka.html +# Kafka metrics collected using the Kafka protocol - module: kafka #metricsets: # - partition From a427180f76d41c0b5995b2f1fcd37d5f6dc61a83 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 18 Nov 2019 11:49:05 +0200 Subject: [PATCH 18/29] Define port in Kafka integration tests Signed-off-by: chrismark --- .../consumergroup_integration_test.go | 4 +-- .../partition/partition_integration_test.go | 6 ++--- x-pack/metricbeat/metricbeat.reference.yml | 25 +++++++++---------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go index 281e822ee73a..1bf3a5bcae35 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go @@ -45,7 +45,7 @@ func TestData(t *testing.T) { compose.UpWithAdvertisedHostEnvFile, ) - c, err := startConsumer(t, service.Host(), "metricbeat-test") + c, err := startConsumer(t, service.HostForPort(9092), "metricbeat-test") if err != nil { t.Fatal(errors.Wrap(err, "starting kafka consumer")) } @@ -65,7 +65,7 @@ func TestData(t *testing.T) { func TestFetch(t *testing.T) { service := compose.EnsureUp(t, "kafka") - c, err := startConsumer(t, service.Host(), "metricbeat-test") + c, err := startConsumer(t, service.HostForPort(9092), "metricbeat-test") if err != nil { t.Fatal(errors.Wrap(err, "starting kafka consumer")) } diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go index 9045b3611048..6b7b5dd22db6 100644 --- a/metricbeat/module/kafka/partition/partition_integration_test.go +++ b/metricbeat/module/kafka/partition/partition_integration_test.go @@ -48,7 +48,7 @@ func TestData(t *testing.T) { compose.UpWithAdvertisedHostEnvFile, ) - generateKafkaData(t, service.Host(), "metricbeat-generate-data") + generateKafkaData(t, service.HostForPort(9092), "metricbeat-generate-data") ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host(), "")) err := mbtest.WriteEventsReporterV2Error(ms, t, "") @@ -66,7 +66,7 @@ func TestTopic(t *testing.T) { testTopic := fmt.Sprintf("test-metricbeat-%s", id) // Create initial topic - generateKafkaData(t, service.Host(), testTopic) + generateKafkaData(t, service.HostForPort(9092), testTopic) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host(), testTopic)) dataBefore, err := mbtest.ReportingFetchV2Error(f) @@ -82,7 +82,7 @@ func TestTopic(t *testing.T) { var i int64 = 0 // Create n messages for ; i < n; i++ { - generateKafkaData(t, service.Host(), testTopic) + generateKafkaData(t, service.HostForPort(9092), testTopic) } dataAfter, err := mbtest.ReportingFetchV2Error(f) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 5750e36cc0fd..8e1374a09eea 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -478,10 +478,9 @@ metricbeat.modules: #-------------------------------- Kafka Module -------------------------------- # Kafka metrics collected using the Kafka protocol - module: kafka - enabled: true - metricsets: - - consumergroup - - partition + #metricsets: + # - partition + # - consumergroup period: 10s hosts: ["localhost:9092"] @@ -508,24 +507,24 @@ metricbeat.modules: # Metrics collected from a Kafka broker using Jolokia #- module: kafka -# enabled: true -# metricsets: ['broker'] +# metricsets: +# - broker # period: 10s -# hosts: ['localhost:8779'] +# hosts: ["localhost:8779"] # Metrics collected from a Java Kafka consumer using Jolokia #- module: kafka -# enabled: true -# metricsets: ['consumer'] +# metricsets: +# - consumer # period: 10s -# hosts: ['localhost:8775'] +# hosts: ["localhost:8775"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka -# enabled: true -# metricsets: ['producer'] +# metricsets: +# - producer # period: 10s -# hosts: ['localhost:8774'] +# hosts: ["localhost:8774"] #-------------------------------- Kibana Module -------------------------------- - module: kibana From 045bac1a68eacc924bc58fa84ee5391e2ab5c458 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 18 Nov 2019 13:07:59 +0200 Subject: [PATCH 19/29] fix remaining ports Signed-off-by: chrismark --- .../kafka/consumergroup/consumergroup_integration_test.go | 4 ++-- .../module/kafka/partition/partition_integration_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go index 1bf3a5bcae35..f6f4f19a4fba 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go @@ -51,7 +51,7 @@ func TestData(t *testing.T) { } defer c.Close() - ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(9092))) for retries := 0; retries < 3; retries++ { err = mbtest.WriteEventsReporterV2Error(ms, t, "") if err == nil { @@ -71,7 +71,7 @@ func TestFetch(t *testing.T) { } defer c.Close() - f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(9092))) var data []mb.Event var errors []error diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go index 6b7b5dd22db6..4e38d362d2b7 100644 --- a/metricbeat/module/kafka/partition/partition_integration_test.go +++ b/metricbeat/module/kafka/partition/partition_integration_test.go @@ -50,7 +50,7 @@ func TestData(t *testing.T) { generateKafkaData(t, service.HostForPort(9092), "metricbeat-generate-data") - ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host(), "")) + ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(9092), "")) err := mbtest.WriteEventsReporterV2Error(ms, t, "") if err != nil { t.Fatal("write", err) @@ -68,7 +68,7 @@ func TestTopic(t *testing.T) { // Create initial topic generateKafkaData(t, service.HostForPort(9092), testTopic) - f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host(), testTopic)) + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(9092), testTopic)) dataBefore, err := mbtest.ReportingFetchV2Error(f) if err != nil { t.Fatal("write", err) From 1b5b1a59954a9f803ef6211a9ebf11fae02c813b Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 18 Nov 2019 17:07:51 +0200 Subject: [PATCH 20/29] Fix healthcheck script in Kafka container Signed-off-by: chrismark --- metricbeat/module/kafka/_meta/healthcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kafka/_meta/healthcheck.sh b/metricbeat/module/kafka/_meta/healthcheck.sh index 97d70b812edc..9314577c9523 100755 --- a/metricbeat/module/kafka/_meta/healthcheck.sh +++ b/metricbeat/module/kafka/_meta/healthcheck.sh @@ -10,5 +10,5 @@ if [[ $rc != 0 ]]; then exit $rc fi -${KAFKA_HOME}/bin/kafka-topic.sh --zookeeper=127.0.0.1:2181 --delete --topic "${TOPIC}" +${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --delete --topic "${TOPIC}" exit 0 From e251525d237a63264ddfbc0e6869ac6023256ec2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 19 Nov 2019 13:01:00 +0200 Subject: [PATCH 21/29] Comment out extra jolokia exposed ports Signed-off-by: chrismark --- metricbeat/docker-compose.yml | 3 --- metricbeat/module/kafka/_meta/Dockerfile | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 4b4186387d52..b0c3915b4d16 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -162,9 +162,6 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-2.1.1} ports: - 9092 - - 8779 - - 8775 - - 8774 kibana: image: docker.elastic.co/observability-ci/beats-integration-kibana:${KIBANA_VERSION:-7.4.0}-1 diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index f8e2b71c8474..87f9e0ee62c8 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -29,9 +29,9 @@ ADD healthcheck.sh /healthcheck.sh EXPOSE 9092 EXPOSE 2181 -EXPOSE 8779 -EXPOSE 8775 -EXPOSE 8774 +#EXPOSE 8779 +#EXPOSE 8775 +#EXPOSE 8774 # Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh From 6d4971b876c8f75806b474d53701a16b666cd1db Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 19 Nov 2019 14:23:04 +0200 Subject: [PATCH 22/29] Add less aggressive kafka producer Signed-off-by: chrismark --- metricbeat/module/kafka/_meta/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index 7575afee3d82..a598cec6012e 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -73,7 +73,7 @@ touch /tmp/.acls_loaded # Start a forever producer -cat /dev/urandom | KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-producer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8775,host=0.0.0.0" \ +{ while sleep 1; do echo message; done } | KAFKA_OPTS="-Djava.security.auth.login.config=/kafka/bin/jaas-kafka-client-producer.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8775,host=0.0.0.0" \ ${KAFKA_HOME}/bin/kafka-console-producer.sh --topic test --broker-list localhost:9091 --producer.config ${KAFKA_HOME}/bin/sasl-producer.properties > /dev/null & wait_for_port 8775 From ffcbaa6f84be073f66b9d75416c15d3a48f961b4 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 20 Nov 2019 10:37:36 +0200 Subject: [PATCH 23/29] Add back jolokia ports in docker Signed-off-by: chrismark --- metricbeat/docker-compose.yml | 3 +++ metricbeat/module/kafka/_meta/Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index db46838983bb..af06572a5210 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -162,6 +162,9 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-2.1.1} ports: - 9092 + - 8779 + - 8775 + - 8774 kibana: image: docker.elastic.co/observability-ci/beats-integration-kibana:${KIBANA_VERSION:-7.4.0}-1 diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index 87f9e0ee62c8..f8e2b71c8474 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -29,9 +29,9 @@ ADD healthcheck.sh /healthcheck.sh EXPOSE 9092 EXPOSE 2181 -#EXPOSE 8779 -#EXPOSE 8775 -#EXPOSE 8774 +EXPOSE 8779 +EXPOSE 8775 +EXPOSE 8774 # Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh From fbba3dd0b5a70ee40dad900cf8ecbe39641723c2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 20 Nov 2019 11:27:40 +0200 Subject: [PATCH 24/29] Fix config ports Signed-off-by: chrismark --- metricbeat/docs/modules/kafka.asciidoc | 4 ++-- metricbeat/metricbeat.reference.yml | 4 ++-- metricbeat/module/kafka/_meta/config.yml | 4 ++-- metricbeat/modules.d/kafka.yml.disabled | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/metricbeat/docs/modules/kafka.asciidoc b/metricbeat/docs/modules/kafka.asciidoc index 5d6d45c28ef0..6da37428fa96 100644 --- a/metricbeat/docs/modules/kafka.asciidoc +++ b/metricbeat/docs/modules/kafka.asciidoc @@ -86,14 +86,14 @@ metricbeat.modules: # metricsets: # - consumer # period: 10s -# hosts: ["localhost:8775"] +# hosts: ["localhost:8774"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka # metricsets: # - producer # period: 10s -# hosts: ["localhost:8774"] +# hosts: ["localhost:8775"] ---- [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 11f8b5db1323..ac3b96cf0c97 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -415,14 +415,14 @@ metricbeat.modules: # metricsets: # - consumer # period: 10s -# hosts: ["localhost:8775"] +# hosts: ["localhost:8774"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka # metricsets: # - producer # period: 10s -# hosts: ["localhost:8774"] +# hosts: ["localhost:8775"] #-------------------------------- Kibana Module -------------------------------- - module: kibana diff --git a/metricbeat/module/kafka/_meta/config.yml b/metricbeat/module/kafka/_meta/config.yml index b648192bda03..fa900f7b6666 100644 --- a/metricbeat/module/kafka/_meta/config.yml +++ b/metricbeat/module/kafka/_meta/config.yml @@ -39,11 +39,11 @@ # metricsets: # - consumer # period: 10s -# hosts: ["localhost:8775"] +# hosts: ["localhost:8774"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka # metricsets: # - producer # period: 10s -# hosts: ["localhost:8774"] +# hosts: ["localhost:8775"] diff --git a/metricbeat/modules.d/kafka.yml.disabled b/metricbeat/modules.d/kafka.yml.disabled index 5cf353232820..b4636d484864 100644 --- a/metricbeat/modules.d/kafka.yml.disabled +++ b/metricbeat/modules.d/kafka.yml.disabled @@ -42,11 +42,11 @@ # metricsets: # - consumer # period: 10s -# hosts: ["localhost:8775"] +# hosts: ["localhost:8774"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka # metricsets: # - producer # period: 10s -# hosts: ["localhost:8774"] +# hosts: ["localhost:8775"] From 2dfa5b89b99514ffa87e9e7aa6162ed70cb959a2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 20 Nov 2019 11:51:37 +0200 Subject: [PATCH 25/29] Increase number of retries in healthcheck Signed-off-by: chrismark --- metricbeat/module/kafka/_meta/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index f8e2b71c8474..956ec4248a03 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -33,7 +33,7 @@ EXPOSE 8779 EXPOSE 8775 EXPOSE 8774 -# Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available -HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh +# Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broker is available +HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh ENTRYPOINT ["/run.sh"] From 1209c6230e82090d904e784e04041a99205cd765 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 20 Nov 2019 11:54:08 +0200 Subject: [PATCH 26/29] Update docker image version Signed-off-by: chrismark --- metricbeat/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index af06572a5210..ebbcc352c769 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -155,7 +155,7 @@ services: - 8778 kafka: - image: docker.elastic.co/observability-ci/beats-integration-kafka:${KAFKA_VERSION:-2.1.1}-1 + image: docker.elastic.co/observability-ci/beats-integration-kafka:${KAFKA_VERSION:-2.1.1}-2 build: context: ./module/kafka/_meta args: From 10f982d92f3a90f4125f870fb5daedf8364d91dc Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 20 Nov 2019 12:16:26 +0200 Subject: [PATCH 27/29] Update x-pack configs Signed-off-by: chrismark --- x-pack/metricbeat/metricbeat.reference.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 8e1374a09eea..1c66d148ba70 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -517,14 +517,14 @@ metricbeat.modules: # metricsets: # - consumer # period: 10s -# hosts: ["localhost:8775"] +# hosts: ["localhost:8774"] # Metrics collected from a Java Kafka producer using Jolokia #- module: kafka # metricsets: # - producer # period: 10s -# hosts: ["localhost:8774"] +# hosts: ["localhost:8775"] #-------------------------------- Kibana Module -------------------------------- - module: kibana From 2e5437bb28755b0261552220e4b4e8ec8f433ffb Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 20 Nov 2019 13:43:52 +0200 Subject: [PATCH 28/29] Increase tests' timeout Signed-off-by: chrismark --- metricbeat/module/kafka/_meta/Dockerfile | 2 +- .../kafka/consumergroup/consumergroup_integration_test.go | 4 ++-- .../module/kafka/partition/partition_integration_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index 956ec4248a03..45f75c9016fc 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -34,6 +34,6 @@ EXPOSE 8775 EXPOSE 8774 # Healthcheck creates an empty topic foo. As soon as a topic is created, it assumes broker is available -HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh +HEALTHCHECK --interval=1s --retries=700 CMD /healthcheck.sh ENTRYPOINT ["/run.sh"] diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go index 1131b7a2d24d..4f1abe6092fd 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go @@ -41,7 +41,7 @@ const ( func TestData(t *testing.T) { service := compose.EnsureUp(t, "kafka", - compose.UpWithTimeout(120*time.Second), + compose.UpWithTimeout(600*time.Second), compose.UpWithAdvertisedHostEnvFileForPort(9092), ) @@ -64,7 +64,7 @@ func TestData(t *testing.T) { func TestFetch(t *testing.T) { service := compose.EnsureUp(t, "kafka", - compose.UpWithTimeout(120*time.Second), + compose.UpWithTimeout(600*time.Second), compose.UpWithAdvertisedHostEnvFileForPort(9092), ) diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go index 4ee53002884a..af5b0f72ea5b 100644 --- a/metricbeat/module/kafka/partition/partition_integration_test.go +++ b/metricbeat/module/kafka/partition/partition_integration_test.go @@ -44,7 +44,7 @@ const ( func TestData(t *testing.T) { service := compose.EnsureUp(t, "kafka", - compose.UpWithTimeout(120*time.Second), + compose.UpWithTimeout(600*time.Second), compose.UpWithAdvertisedHostEnvFileForPort(9092), ) @@ -60,7 +60,7 @@ func TestData(t *testing.T) { func TestTopic(t *testing.T) { service := compose.EnsureUp(t, "kafka", - compose.UpWithTimeout(120*time.Second), + compose.UpWithTimeout(600*time.Second), compose.UpWithAdvertisedHostEnvFileForPort(9092), ) From 54b58c2cc54f22676626551c6b0ff4fd9eee63c8 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 20 Nov 2019 15:02:13 +0200 Subject: [PATCH 29/29] Fix fields' type Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 30 +++++++------- .../module/kafka/broker/_meta/fields.yml | 22 +++++------ .../module/kafka/consumer/_meta/fields.yml | 2 +- metricbeat/module/kafka/fields.go | 2 +- .../module/kafka/producer/_meta/fields.yml | 6 +-- .../modules.d/googlecloud.yml.disabled | 39 +++++++++++++++++++ 6 files changed, 70 insertions(+), 31 deletions(-) create mode 100644 x-pack/filebeat/modules.d/googlecloud.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ff357094c36b..e5e617f8e928 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -15021,7 +15021,7 @@ type: long -- The rate of failed produce requests per second -type: long +type: float -- @@ -15030,7 +15030,7 @@ type: long -- The rate of client fetch request failures per second -type: long +type: float -- @@ -15039,7 +15039,7 @@ type: long -- The leader election rate -type: long +type: float -- @@ -15048,7 +15048,7 @@ type: long -- The unclean leader election rate -type: long +type: float -- @@ -15057,7 +15057,7 @@ type: long -- The ZooKeeper closed sessions per second -type: long +type: float -- @@ -15066,7 +15066,7 @@ type: long -- The ZooKeeper expired sessions per second -type: long +type: float -- @@ -15075,7 +15075,7 @@ type: long -- The ZooKeeper readonly sessions per second -type: long +type: float -- @@ -15102,7 +15102,7 @@ type: float -- The incoming byte rate -type: long +type: float -- @@ -15111,7 +15111,7 @@ type: long -- The outgoing byte rate -type: long +type: float -- @@ -15120,7 +15120,7 @@ type: long -- The rejected byte rate -type: long +type: float -- @@ -15129,7 +15129,7 @@ type: long -- The incoming message rate -type: long +type: float -- @@ -15180,7 +15180,7 @@ type: float -- The rate of bytes coming in to the consumer -type: long +type: float -- @@ -15499,7 +15499,7 @@ type: keyword -- The total amount of buffer memory -type: long +type: float -- @@ -15553,7 +15553,7 @@ type: float -- The average number of records sent per second -type: long +type: float -- @@ -15607,7 +15607,7 @@ type: float -- The rate of bytes going out for the producer -type: long +type: float -- diff --git a/metricbeat/module/kafka/broker/_meta/fields.yml b/metricbeat/module/kafka/broker/_meta/fields.yml index 4231ee0cf295..4fa195e7b298 100644 --- a/metricbeat/module/kafka/broker/_meta/fields.yml +++ b/metricbeat/module/kafka/broker/_meta/fields.yml @@ -11,25 +11,25 @@ type: long - name: request.produce.failed_per_second description: The rate of failed produce requests per second - type: long + type: float - name: request.fetch.failed_per_second description: The rate of client fetch request failures per second - type: long + type: float - name: replication.leader_elections description: The leader election rate - type: long + type: float - name: replication.unclean_leader_elections description: The unclean leader election rate - type: long + type: float - name: session.zookeeper.disconnect description: The ZooKeeper closed sessions per second - type: long + type: float - name: session.zookeeper.expire description: The ZooKeeper expired sessions per second - type: long + type: float - name: session.zookeeper.readonly description: The ZooKeeper readonly sessions per second - type: long + type: float - name: session.zookeeper.sync description: The ZooKeeper client connections per second type: float @@ -38,13 +38,13 @@ type: float - name: topic.net.bytes_in description: The incoming byte rate - type: long + type: float - name: topic.net.bytes_out description: The outgoing byte rate - type: long + type: float - name: topic.net.bytes_rejected description: The rejected byte rate - type: long + type: float - name: topic.messages_in description: The incoming message rate - type: long + type: float diff --git a/metricbeat/module/kafka/consumer/_meta/fields.yml b/metricbeat/module/kafka/consumer/_meta/fields.yml index cc39470b617f..c9946d6d801c 100644 --- a/metricbeat/module/kafka/consumer/_meta/fields.yml +++ b/metricbeat/module/kafka/consumer/_meta/fields.yml @@ -17,4 +17,4 @@ type: float - name: bytes_in description: The rate of bytes coming in to the consumer - type: long + type: float diff --git a/metricbeat/module/kafka/fields.go b/metricbeat/module/kafka/fields.go index 0aab1054bfd9..4176d06ada76 100644 --- a/metricbeat/module/kafka/fields.go +++ b/metricbeat/module/kafka/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKafka returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/kafka. func AssetKafka() string { - return "eJzUWk2P2zYQve+vGOS0OUQ5tYc9FGiTotgmaYI0BYpeBJocWexKpEJS3ji/viApybI+KEpeB82e1rI475HzweGjX8ADHu/ggWQP5AbAcFPgHTx7Yz8/uwFgqKnileFS3MFPNwAA7jsoJasLvAHQuVQmpVJkfH8HGSm0faqwQKLxDvbWbMaxYPrODX8BgpR4grR/5ljZV5Wsq+bJBO65mb6pnZIPqLrHU/Zmbfq/X5wFeCWFrktU8JsdCvcik6okdgDk5ICwQxSgkDDIlCzhthmWE8EKLvZnJk2OQFt7jsrzpPfCcC79+XB29ridTyEHEMEp9abF2c0kDmFModaTYA94fJRqSCQOj7ADKsM1sg5i5DMjK04T+//Ib2PoAOwna8fZnMNApaRKqGRjpMGKLsI4U2BNJWO0iijD7djkzH9rkT60ZoCzIIqbXTqBFV6/M7C/BP9cI3AGMnMRW53QhXvg13CZh8/Bb0MHiGDukwdNnqQgNLFbolGcap/gvtQ13/z+7u/e2K7A7dCQyLwud0hEKKPe2RfA5MSAybkGPKAwwLVFIwYZGBmdrC2ows81apPQnAiBRfK5xhoTzb9iiMmnHMG+0zqisQJudFx1GhKolGQ1xSQjvECWVqhSjVSKYI2xPBQxjocfCI2d1q6GChVMWooklqGh+XZatODWS85Kt07WWq3wEnJVwanbe5ICCUOVYoHUfh4W7BEz/z607zum29FrQQskIl3Lohn3BGw0am2ZfJXyAbFClTCuqRQCqVli8Y+Ub9wYoIW0O1JjbLtnxmzwS8XVYjqdmPj3r0LFNidSFMd4Mu2Ia7DRR0HXOMglUuPYGCpZIYmZ5FLIfZIVtc7TiXAbJ4zcg3t7PjjnoZpOBk2yOxrUKQ+WeAvHBZUlF3uwAzbkwxBR1otpIGuzl08IqfBfpAaXi2Tz3mW4JWpN9uvWthkTidnitR37ut6hOzdMdA/dd99p/+D2tagsKrngZV36jZEYeMw5zc/PQRoF0+dbpQYjgYxbNohIPR+LjfXFWCQHVDYiRF3uUNmt241v2THIpAICukLKM06bXnNzCVJIpWKX0GssnAieuExy3bR6yynVNjrtarn0sv24PHPu+iQb5tTqY/ucoXP1of0LpVYg/IbGPadKIbUpdQc/Jj9sOiU/pdgAy4LD3ApASHiAkPgQMVU4EyHaJ7P404IELIgS63gMMdYoMBGiyNCrr8NA83k7hxUdeD1lJEihO2PH7cyrOJzEjKWFkFmmcdjGxOleXdB7G8CFkT3lYIe2XtlECjMozzdiWON1WmsjyxMTawsYMQS0Uf1knUSeVKji5z8Od6etdmxentaiL2AFKflefEU9DPL5WWu+F8jaFt86wzrF9SdNtzQXhhfUrMhy8cqTun8Nt37hNBpj6Xm2CWfPl0tXLvVwubYSOTM1C1iibRPSy6fPhUElSDHYPBqAfuKGKsbqvXvKyPp9O1A2tsTpgfCC7Aps7OpW/trzA4qe5rkyRgU+YiA8tu+tfzjDbeEbiqZDmr1lK9h1CL13hpcJbdiJNvjztP3YWvwtuqHQzhhBGEYXAO3TWZZeXruCK9963Y4zuPXd8aid7C2VnmdwUdP2lrtosgDAmZ5n0CiWV1iHj97y9ELMr4jQR0HTJVo7KYvxoT6S2b1g3EabBp61C2BP+1zQombI2ksTLl5YMp2qi3aHg9v7Pz9GzUSnCzF2lUmYTshepjjbQMET+P/XrmfyjYpTL2x7EFvWAteQQX4XHTLN6MJyntrgFhae7gwy5hV7Kmmu9tZchV9+JOfTEljDZct1+eWcZs+r47bMX4yt1C4/NKOmtMvuu+9UuyRtP5fu6ixDlToJK3hodnfMhhRASlkLt/f4sbYflmp4sbIgX++IoXmq+VdMySF4iAsJlHruKBbQ9k7AJfmyBFySL065jQYOXBxSqViqUbAozXhe9LTYl4qvqUKjjpuJGMWRNaYa6fpSQq4g/48IaXfh3Ujx13XWYsyszJJ2Gca/ZIj0xdrsWAJc+qFBlNtPa9uW8+gfOoQmrCspNG5n4MdfQIHL9JHwxRDrIO9fvgc7AAyfaU6WLjYibkXPbzb8BamsjbtcMT028+7+LwAA///1XQeS" + return "eJzUWk+P27YTve+nGOS0OUQ5/X6HPRRok6LYJmmCNAWKXgSaHFnsUqRCUt44n74gKcmy/lCS5U2bPa1lcd7jcGY4fPQLeMDjHTyQ7IHcAFhuBd7Bszfu87MbAIaGal5aruQd/HADAOC/g0KxSuANgMmVtilVMuP7O8iIMO6pRoHE4B3sndmMo2Dmzg9/AZIUeIJ0f/ZYule1qsr6yQjuuZmuqZ1WD6jbx2P2Jm2Gv5+8BXilpKkK1PCLGwr3MlO6IG4A5OSAsEOUoJEwyLQq4LYelhPJBJf7M5M2R6CNPU/ledJ5oT+X7nw4O3vczEeoHkR0Sp1pcXYzikMY02jMKNgDHh+V7hNZhkfYAbXlBlkLMVgzq0pOE/f/YN2G0BHYT86OtzmFgVornVDFhkg9j87CeFPgTCVDtJJoy93Y5Gz91iJ9aMwAZ1EUP7t0BCvuvzOwPyT/XCFwBirzEVue0KV/EHw4zyPk4LehA0Qy/ymAJlcpCHXsFmg1pyYkeCh19Te/vvuzM7YtcDu0ZGFeFzskMpZR79wLYHNiwebcAB5QWuDGoRGLDKxanKwNqMbPFRqb0JxIiSL5XGGFieFfMcbkU47g3mkWorYCfvSy6tQnUGrFKopJRrhAlpaoU4NUyWiNcTw0sZ5HGAi1ncaugRI1jFoKxDKhiI0yy9DS/HJeVHC3TN5K6yhnrdK4iV0pOPW7TyKQMNQpCqTuc79kD6iF96F531PdAF9JKpDIdC2Netw16Bg0xlH5qtQDYok6YdxQJSVSO0fjL6Xe+DFAhXK7Um1sw+IM6eCXkuvZnDpRCe8/DRfXoigpjsvZNCOehI45SrpmjXw61Wu7jYtQ+yQTlcnTkZAbZo3ag3/7kgCtGxq0ye5o0aQ8WukdHJdUFVzuwQ24BqSqZlNBVXavromp8W+kFueLZf3eRuACjSH7de6txywFbQCb5n1dG9EeIUYaifa777SV8DvcokwquORFVYQtklh4zDnNz49EBiUz55umAauADLu3JeERorG2PhuN5IDahYSsih1qt4n78Q07BpnSQMCUSHnGad12btjKqdJsC73awongicso14u8N59TTcvTeMvnl2vN1dniXpBl/aRafYSfMnSuRDR/sdyKxF/feOBUaqQup+7g/8n/LjoxX1N4gHnxYcoDEBMhICZELJgqnAkSzZNJ/HFxAmYEinU8+hhr1JgFAkl/VV/HgaYTdwprceB1VJIohfa8veyIt4rDSdiYc4TKMoP9VmaZBtYGfbABXFrVURF26AqWS6Q4g+J8J4Y1q04rY1VxYuJsASOWgLG6m6yjyKNq1fL5D8Pd66wtm5cnX3TFrCil0JCvqIdRPj8aw/cSWdPnu8Vwi+IblLpdmgrDDTVrYbl4FUjdv4bb4DiD1jp6gW3C2fP50pUr03fXpUTOTE0CFuj6hHT79Lm0qCURvc2jBugmbqxirN67x4ys37cjZeOSOD0QLshOYG3XNFLYnh9QdvTPlTEq8REj4XH53vqbN9wUvr6A2qfZcZtgT0PovTc8T+iCneiC9TxtP64Wf4tuKLYzLiAMg8uA5ukky6CzPcFSvg0CHmdwG7rjQTvZcZWZZrCpaXvLfTQ5AODMTDOopcsn8MPHYHncEdMekeYoaTpHa6eUGJ7qFzK7l4y7aDPAs8YB7rjPJRUVQ9ZcoHD5wpFp5V10Oxzc3v/+cdFMTDoTY08yCdtK2vMUJxsouML6/9z2TKFR8fKFaw+WlrXIlWSU36ZDph1cXk5T693IwvXOIENeS08l9TXfmmvx7UdyPq6B1VwuuTrfzmnyvDpsy8Il2Urx8kM9aky8bL/7TsVL0vRz6a7KMtSp17Cih2Z/32yJAFKoSvq9J4x1/bDS/euVWZGNWJqnhn/FlByip7iYRGmmzmKLgAvyZQ64IF+8drsYOHLzS5VmqUHJFqnG07Knw94qv6YarT5eTMRqjqw2VYvXWwn5ivwfImT85Xctxv/Li7U2TRo/DH/XsAZwRXrMAc79HmLRup+c2xT0q/zswZRKGrycQRi/gQJX6SPhszHWQt6/fA9uAFg+0Z7M3W0suBs9v9wI16Sqsv5+xXbYRPD/CQAA//8NHQ2M" } diff --git a/metricbeat/module/kafka/producer/_meta/fields.yml b/metricbeat/module/kafka/producer/_meta/fields.yml index e0c3eaf14a02..1edd723e0a4b 100644 --- a/metricbeat/module/kafka/producer/_meta/fields.yml +++ b/metricbeat/module/kafka/producer/_meta/fields.yml @@ -8,7 +8,7 @@ type: keyword - name: available_buffer_bytes description: The total amount of buffer memory - type: long + type: float - name: batch_size_avg description: The average number of bytes sent type: float @@ -26,7 +26,7 @@ type: float - name: records_per_request description: The average number of records sent per second - type: long + type: float - name: record_size_avg description: The average record size type: float @@ -44,4 +44,4 @@ type: float - name: bytes_out description: The rate of bytes going out for the producer - type: long + type: float diff --git a/x-pack/filebeat/modules.d/googlecloud.yml.disabled b/x-pack/filebeat/modules.d/googlecloud.yml.disabled new file mode 100644 index 000000000000..d0be252b97aa --- /dev/null +++ b/x-pack/filebeat/modules.d/googlecloud.yml.disabled @@ -0,0 +1,39 @@ +# Module: googlecloud +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-googlecloud.html + +- module: googlecloud + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: googlecloud-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-googlecloud-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: googlecloud-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-googlecloud-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json