Skip to content

Commit

Permalink
[kafka] upgraded jmx keys for >= 0.8.2
Browse files Browse the repository at this point in the history
[skip ci]
kafka updated their bean names.
See the documentation:
http://kafka.apache.org/082/documentation.html#monitoring
http://kafka.apache.org/081/documentation.html#monitoring

This won't work on older Kafka version so we added a warning
to link back to the old Kafka sample file if needed

[leo@datadoghq.com] Rebased on master and reindented
+ added warning
  • Loading branch information
patrickbcullen authored and LeoCavaille committed Mar 17, 2015
1 parent d10deb2 commit 5e7dbc7
Showing 1 changed file with 64 additions and 43 deletions.
107 changes: 64 additions & 43 deletions conf.d/kafka.yaml.example
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
##########
# WARNING
##########
# This sample works only for Kafka >= 0.8.2.
# If you are running a version older than that, you can refer to agent 5.2.x released
# sample files, https://raw.githubusercontent.com/DataDog/dd-agent/5.2.1/conf.d/kafka.yaml.example

instances:
- host: localhost
port: 9999 # This is the JMX port on which Kafka exposes its metrics (usually 9999)
# user: username
# password: password
# process_name_regex: .*process_name.* # Instead of specifying a host, and port. The agent can connect using the attach api.
# # This requires the JDK to be installed and the path to tools.jar to be set below.
# tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
# name: kafka_instance
# # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
# # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
# # trust_store_password: password
# tags:
# env: stage
# newTag: test
# user: username
# password: password
# process_name_regex: .*process_name.* # Instead of specifying a host, and port. The agent can connect using the attach api.
# # This requires the JDK to be installed and the path to tools.jar to be set below.
# tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
# name: kafka_instance
# java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
# trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
# trust_store_password: password
# tags:
# env: stage
# newTag: test

init_config:
is_jmx: true
Expand All @@ -23,47 +30,54 @@ init_config:
# Aggregate cluster stats
#
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsBytesOutPerSec"'
domain: 'kafka.server'
bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.net.bytes_out
alias: kafka.net.bytes_out
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsBytesInPerSec"'
domain: 'kafka.server'
bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.net.bytes_in
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsMessagesInPerSec"'
domain: 'kafka.server'
bean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.messages_in
- include:
domain: 'kafka.server'
bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.net.bytes_rejected

#
# Request timings
#
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsFailedFetchRequestsPerSec"'
domain: 'kafka.server'
bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.request.fetch.failed
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="BrokerTopicMetrics",name="AllTopicsFailedProduceRequestsPerSec"'
domain: 'kafka.server'
bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.request.produce.failed
- include:
domain: '"kafka.network"'
bean: '"kafka.network":type="RequestMetrics",name="Produce-TotalTimeMs"'
domain: 'kafka.network'
bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce'
attribute:
Mean:
metric_type: gauge
Expand All @@ -72,8 +86,8 @@ init_config:
metric_type: gauge
alias: kafka.request.produce.time.99percentile
- include:
domain: '"kafka.network"'
bean: '"kafka.network":type="RequestMetrics",name="Fetch-TotalTimeMs"'
domain: 'kafka.network'
bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Fetch'
attribute:
Mean:
metric_type: gauge
Expand All @@ -82,8 +96,8 @@ init_config:
metric_type: gauge
alias: kafka.request.fetch.time.99percentile
- include:
domain: '"kafka.network"'
bean: '"kafka.network":type="RequestMetrics",name="UpdateMetadata-TotalTimeMs"'
domain: 'kafka.network'
bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=UpdateMetadata'
attribute:
Mean:
metric_type: gauge
Expand All @@ -92,8 +106,8 @@ init_config:
metric_type: gauge
alias: kafka.request.update_metadata.time.99percentile
- include:
domain: '"kafka.network"'
bean: '"kafka.network":type="RequestMetrics",name="Metadata-TotalTimeMs"'
domain: 'kafka.network'
bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Metadata'
attribute:
Mean:
metric_type: gauge
Expand All @@ -102,43 +116,50 @@ init_config:
metric_type: gauge
alias: kafka.request.metadata.time.99percentile
- include:
domain: '"kafka.network"'
bean: '"kafka.network":type="RequestMetrics",name="Offsets-TotalTimeMs"'
domain: 'kafka.network'
bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Offsets'
attribute:
Mean:
metric_type: gauge
alias: kafka.request.offsets.time.avg
99thPercentile:
metric_type: gauge
alias: kafka.request.offsets.time.99percentile
- include:
domain: 'kafka.server'
bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.request.handler.avg.idle.pct

#
# Replication stats
#
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="ReplicaManager",name="ISRShrinksPerSec"'
domain: 'kafka.server'
bean: 'kafka.server:type=ReplicaManager,name=IsrShrinksPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.replication.isr_shrinks
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="ReplicaManager",name="ISRExpandsPerSec"'
domain: 'kafka.server'
bean: 'kafka.server:type=ReplicaManager,name=IsrExpandsPerSec'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.replication.isr_expands
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="ControllerStats",name="LeaderElectionRateAndTimeMs"'
domain: 'kafka.controller'
bean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs'
attribute:
MeanRate:
metric_type: gauge
alias: kafka.replication.leader_elections
- include:
domain: '"kafka.server"'
bean: '"kafka.server":type="ControllerStats",name="UncleanLeaderElectionsPerSec"'
domain: 'kafka.controller'
bean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec'
attribute:
MeanRate:
metric_type: gauge
Expand All @@ -148,8 +169,8 @@ init_config:
# Log flush stats
#
- include:
domain: '"kafka.log"'
bean: '"kafka.log":type="LogFlushStats",name="LogFlushRateAndTimeMs"'
domain: 'kafka.log'
bean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs'
attribute:
MeanRate:
metric_type: gauge
Expand Down

0 comments on commit 5e7dbc7

Please sign in to comment.