From 5e7dbc74afe632d50c4a1fca7431a44fc379711b Mon Sep 17 00:00:00 2001 From: Patrick Cullen Date: Thu, 12 Mar 2015 20:00:04 -0400 Subject: [PATCH] [kafka] upgraded jmx keys for >= 0.8.2 [skip ci] kafka updated their bean names. See the documentation: http://kafka.apache.org/082/documentation.html#monitoring http://kafka.apache.org/081/documentation.html#monitoring This won't work on older Kafka version so we added a warning to link back to the old Kafka sample file if needed [leo@datadoghq.com] Rebased on master and reindented + added warning --- conf.d/kafka.yaml.example | 107 +++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 43 deletions(-) diff --git a/conf.d/kafka.yaml.example b/conf.d/kafka.yaml.example index 47710eee47..3e1d326e9d 100644 --- a/conf.d/kafka.yaml.example +++ b/conf.d/kafka.yaml.example @@ -1,18 +1,25 @@ +########## +# WARNING +########## +# This sample works only for Kafka >= 0.8.2. +# If you are running a version older than that, you can refer to agent 5.2.x released +# sample files, https://raw.githubusercontent.com/DataDog/dd-agent/5.2.1/conf.d/kafka.yaml.example + instances: - host: localhost port: 9999 # This is the JMX port on which Kafka exposes its metrics (usually 9999) - # user: username - # password: password - # process_name_regex: .*process_name.* # Instead of specifying a host, and port. The agent can connect using the attach api. - # # This requires the JDK to be installed and the path to tools.jar to be set below. - # tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set - # name: kafka_instance - # # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable - # # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled - # # trust_store_password: password - # tags: - # env: stage - # newTag: test + # user: username + # password: password + # process_name_regex: .*process_name.* # Instead of specifying a host, and port. The agent can connect using the attach api. + # # This requires the JDK to be installed and the path to tools.jar to be set below. + # tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set + # name: kafka_instance + # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable + # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled + # trust_store_password: password + # tags: + # env: stage + # newTag: test init_config: is_jmx: true @@ -23,47 +30,54 @@ init_config: # Aggregate cluster stats # - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsBytesOutPerSec"' + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec' attribute: MeanRate: metric_type: gauge - alias: kafka.net.bytes_out + alias: kafka.net.bytes_out - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsBytesInPerSec"' + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec' attribute: MeanRate: metric_type: gauge alias: kafka.net.bytes_in - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsMessagesInPerSec"' + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec' attribute: MeanRate: metric_type: gauge alias: kafka.messages_in + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec' + attribute: + MeanRate: + metric_type: gauge + alias: kafka.net.bytes_rejected # # Request timings # - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsFailedFetchRequestsPerSec"' + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec' attribute: MeanRate: metric_type: gauge alias: kafka.request.fetch.failed - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsFailedProduceRequestsPerSec"' + domain: 'kafka.server' + bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec' attribute: MeanRate: metric_type: gauge alias: kafka.request.produce.failed - include: - domain: '"kafka.network"' - bean: '"kafka.network":type="RequestMetrics",name="Produce-TotalTimeMs"' + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce' attribute: Mean: metric_type: gauge @@ -72,8 +86,8 @@ init_config: metric_type: gauge alias: kafka.request.produce.time.99percentile - include: - domain: '"kafka.network"' - bean: '"kafka.network":type="RequestMetrics",name="Fetch-TotalTimeMs"' + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Fetch' attribute: Mean: metric_type: gauge @@ -82,8 +96,8 @@ init_config: metric_type: gauge alias: kafka.request.fetch.time.99percentile - include: - domain: '"kafka.network"' - bean: '"kafka.network":type="RequestMetrics",name="UpdateMetadata-TotalTimeMs"' + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=UpdateMetadata' attribute: Mean: metric_type: gauge @@ -92,8 +106,8 @@ init_config: metric_type: gauge alias: kafka.request.update_metadata.time.99percentile - include: - domain: '"kafka.network"' - bean: '"kafka.network":type="RequestMetrics",name="Metadata-TotalTimeMs"' + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Metadata' attribute: Mean: metric_type: gauge @@ -102,8 +116,8 @@ init_config: metric_type: gauge alias: kafka.request.metadata.time.99percentile - include: - domain: '"kafka.network"' - bean: '"kafka.network":type="RequestMetrics",name="Offsets-TotalTimeMs"' + domain: 'kafka.network' + bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Offsets' attribute: Mean: metric_type: gauge @@ -111,34 +125,41 @@ init_config: 99thPercentile: metric_type: gauge alias: kafka.request.offsets.time.99percentile + - include: + domain: 'kafka.server' + bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent' + attribute: + MeanRate: + metric_type: gauge + alias: kafka.request.handler.avg.idle.pct # # Replication stats # - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="ReplicaManager",name="ISRShrinksPerSec"' + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=IsrShrinksPerSec' attribute: MeanRate: metric_type: gauge alias: kafka.replication.isr_shrinks - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="ReplicaManager",name="ISRExpandsPerSec"' + domain: 'kafka.server' + bean: 'kafka.server:type=ReplicaManager,name=IsrExpandsPerSec' attribute: MeanRate: metric_type: gauge alias: kafka.replication.isr_expands - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="ControllerStats",name="LeaderElectionRateAndTimeMs"' + domain: 'kafka.controller' + bean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs' attribute: MeanRate: metric_type: gauge alias: kafka.replication.leader_elections - include: - domain: '"kafka.server"' - bean: '"kafka.server":type="ControllerStats",name="UncleanLeaderElectionsPerSec"' + domain: 'kafka.controller' + bean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec' attribute: MeanRate: metric_type: gauge @@ -148,8 +169,8 @@ init_config: # Log flush stats # - include: - domain: '"kafka.log"' - bean: '"kafka.log":type="LogFlushStats",name="LogFlushRateAndTimeMs"' + domain: 'kafka.log' + bean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs' attribute: MeanRate: metric_type: gauge