Basic RabbitMQ queues metricset (#4788)

Basic RabbitMQ queue metricset

Related to #3887

CHANGELOG.asciidoc

@@ -100,6 +100,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add http server metricset to support push metrics via http. {pull}4770[4770]
 - Make config object public for graphite and http server {pull}4820[4820]
 - Add system uptime metricset. {issue}[4848[4848]
+- Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685]
+- Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788]
 
 *Packetbeat*
 

metricbeat/Makefile

@@ -27,7 +27,7 @@ kibana:
 
 # Collects all module and metricset fields
 .PHONY: fields
-fields:
+fields: python-env
 	@mkdir -p _meta
 	@cp ${ES_BEATS}/metricbeat/_meta/fields.common.yml _meta/fields.generated.yml
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml

metricbeat/docs/fields.asciidoc

@@ -7852,6 +7852,153 @@ type: long
 Node uptime.
 
 
+[float]
+== queue fields
+
+queue
+
+
+
+[float]
+=== `rabbitmq.queue.name`
+
+type: keyword
+
+The name of the queue with non-ASCII characters escaped as in C.
+
+
+[float]
+=== `rabbitmq.queue.vhost`
+
+type: keyword
+
+Virtual host name with non-ASCII characters escaped as in C.
+
+
+[float]
+=== `rabbitmq.queue.durable`
+
+type: boolean
+
+Whether or not the queue survives server restarts.
+
+
+[float]
+=== `rabbitmq.queue.auto_delete`
+
+type: boolean
+
+Whether the queue will be deleted automatically when no longer used.
+
+
+[float]
+=== `rabbitmq.queue.exclusive`
+
+type: boolean
+
+Whether the queue is exclusive (i.e. has owner_pid).
+
+
+[float]
+=== `rabbitmq.queue.node`
+
+type: keyword
+
+Node name.
+
+
+[float]
+=== `rabbitmq.queue.state`
+
+type: keyword
+
+The state of the queue. Normally 'running', but may be "{syncing, MsgCount}" if the queue is synchronising. Queues which are located on cluster nodes that are currently down will be shown with a status of 'down'.
+
+
+[float]
+=== `rabbitmq.queue.arguments.max_priority`
+
+type: long
+
+Maximum number of priority levels for the queue to support.
+
+
+[float]
+=== `rabbitmq.queue.consumers.count`
+
+type: long
+
+Number of consumers.
+
+
+[float]
+=== `rabbitmq.queue.consumers.utilisation.pct`
+
+type: long
+
+format: percentage
+
+Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count.
+
+
+[float]
+=== `rabbitmq.queue.messages.total.count`
+
+type: long
+
+Sum of ready and unacknowledged messages (queue depth).
+
+
+[float]
+=== `rabbitmq.queue.messages.ready.count`
+
+type: long
+
+Number of messages ready to be delivered to clients.
+
+
+[float]
+=== `rabbitmq.queue.messages.unacknowledged.count`
+
+type: long
+
+Number of messages delivered to clients but not yet acknowledged.
+
+
+[float]
+=== `rabbitmq.queue.messages.persistent.count`
+
+type: long
+
+Total number of persistent messages in the queue (will always be 0 for transient queues).
+
+
+[float]
+=== `rabbitmq.queue.memory.bytes`
+
+type: long
+
+format: bytes
+
+Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures.
+
+
+[float]
+=== `rabbitmq.queue.disk.reads.count`
+
+type: long
+
+Total number of times messages have been read from disk by this queue since it started.
+
+
+[float]
+=== `rabbitmq.queue.disk.writes.count`
+
+type: long
+
+Total number of times messages have been written to disk by this queue since it started.
+
+
 [[exported-fields-redis]]
 == Redis fields
 

metricbeat/docs/modules/rabbitmq.asciidoc

@@ -19,7 +19,7 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: rabbitmq
-  metricsets: ["node"]
+  metricsets: ["node", "queue"]
   period: 10s
   hosts: ["localhost:15672"]
 
@@ -34,5 +34,9 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-rabbitmq-node,node>>
 
+* <<metricbeat-metricset-rabbitmq-queue,queue>>
+
 include::rabbitmq/node.asciidoc[]
 
+include::rabbitmq/queue.asciidoc[]
+

metricbeat/docs/modules/rabbitmq/queue.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-rabbitmq-queue]]
+include::../../../module/rabbitmq/queue/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-rabbitmq,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/rabbitmq/queue/_meta/data.json[]
+----

metricbeat/include/list.go

@@ -87,6 +87,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/prometheus/stats"
 	_ "github.com/elastic/beats/metricbeat/module/rabbitmq"
 	_ "github.com/elastic/beats/metricbeat/module/rabbitmq/node"
+	_ "github.com/elastic/beats/metricbeat/module/rabbitmq/queue"
 	_ "github.com/elastic/beats/metricbeat/module/redis"
 	_ "github.com/elastic/beats/metricbeat/module/redis/info"
 	_ "github.com/elastic/beats/metricbeat/module/redis/keyspace"

metricbeat/metricbeat.reference.yml

@@ -378,7 +378,7 @@ metricbeat.modules:
 
 #------------------------------ RabbitMQ Module ------------------------------
 - module: rabbitmq
-  metricsets: ["node"]
+  metricsets: ["node", "queue"]
   period: 10s
   hosts: ["localhost:15672"]
 

metricbeat/module/rabbitmq/_meta/config.yml

@@ -1,5 +1,5 @@
 - module: rabbitmq
-  metricsets: ["node"]
+  metricsets: ["node", "queue"]
   period: 10s
   hosts: ["localhost:15672"]
 

metricbeat/module/rabbitmq/_meta/testdata/queue_sample_response.json

@@ -0,0 +1,145 @@
+[
+    {
+        "memory": 232720,
+        "message_stats": {
+            "disk_reads": 212,
+            "disk_reads_details": {
+                "rate": 0
+            },
+            "disk_writes": 121,
+            "disk_writes_details": {
+                "rate": 0
+            },
+            "deliver": 15,
+            "deliver_details": {
+                "rate": 0
+            },
+            "deliver_no_ack": 0,
+            "deliver_no_ack_details": {
+                "rate": 0
+            },
+            "get": 0,
+            "get_details": {
+                "rate": 0
+            },
+            "get_no_ack": 38,
+            "get_no_ack_details": {
+                "rate": 0
+            },
+            "publish": 121,
+            "publish_details": {
+                "rate": 0
+            },
+            "publish_in": 0,
+            "publish_in_details": {
+                "rate": 0
+            },
+            "publish_out": 0,
+            "publish_out_details": {
+                "rate": 0
+            },
+            "ack": 9,
+            "ack_details": {
+                "rate": 0
+            },
+            "deliver_get": 53,
+            "deliver_get_details": {
+                "rate": 0
+            },
+            "confirm": 0,
+            "confirm_details": {
+                "rate": 0
+            },
+            "return_unroutable": 0,
+            "return_unroutable_details": {
+                "rate": 0
+            },
+            "redeliver": 3,
+            "redeliver_details": {
+                "rate": 0
+            }
+        },
+        "reductions": 787128,
+        "reductions_details": {
+            "rate": 0
+        },
+        "messages": 74,
+        "messages_details": {
+            "rate": 0
+        },
+        "messages_ready": 71,
+        "messages_ready_details": {
+            "rate": 0
+        },
+        "messages_unacknowledged": 3,
+        "messages_unacknowledged_details": {
+            "rate": 0
+        },
+        "idle_since": "2017-07-28 23:45:52",
+        "consumer_utilisation": 0.7,
+        "policy": null,
+        "exclusive_consumer_tag": null,
+        "consumers": 3,
+        "recoverable_slaves": null,
+        "state": "running",
+        "garbage_collection": {
+            "min_bin_vheap_size": 46422,
+            "min_heap_size": 233,
+            "fullsweep_after": 65535,
+            "minor_gcs": 0
+        },
+        "messages_ram": 74,
+        "messages_ready_ram": 71,
+        "messages_unacknowledged_ram": 3,
+        "messages_persistent": 73,
+        "message_bytes": 101824,
+        "message_bytes_ready": 97696,
+        "message_bytes_unacknowledged": 4128,
+        "message_bytes_ram": 101824,
+        "message_bytes_persistent": 101824,
+        "head_message_timestamp": 1501250275,
+        "disk_reads": 212,
+        "disk_writes": 121,
+        "backing_queue_status": {
+            "priority_lengths": {
+                "0": 0,
+                "1": 71,
+                "2": 0,
+                "3": 0,
+                "4": 0,
+                "5": 0,
+                "6": 0,
+                "7": 0,
+                "8": 0,
+                "9": 0
+            },
+            "mode": "default",
+            "q1": 0,
+            "q2": 0,
+            "delta": [
+                "delta",
+                "todo",
+                "todo",
+                "todo"
+            ],
+            "q3": 0,
+            "q4": 71,
+            "len": 71,
+            "target_ram_count": "infinity",
+            "next_seq_id": 121,
+            "avg_ingress_rate": 0,
+            "avg_egress_rate": 0.00019793395296866087,
+            "avg_ack_ingress_rate": 0.00019793395296866087,
+            "avg_ack_egress_rate": 0.00019793395296866087
+        },
+        "node": "rabbit@localhost",
+        "arguments": {
+            "x-max-priority": 9
+        },
+        "exclusive": false,
+        "auto_delete": false,
+        "durable": true,
+        "vhost": "/",
+        "name": "queuenamehere"
+    }
+]