[Metricbeat]kube-state-metrics: add storage class support (#16145)

* add ksm storage class support
elastic · Feb 10, 2020 · 86434d6 · 86434d6
1 parent 327b14d
commit 86434d6
Show file tree

Hide file tree

Showing 21 changed files with 402 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -145,6 +145,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add collecting AuroraDB metrics in rds metricset. {issue}14142[14142] {pull}16004[16004]
 - Reuse connections in SQL module. {pull}16001[16001]
 - Improve the `logstash` module (when `xpack.enabled` is set to `true`) to use the override `cluster_uuid` returned by Logstash APIs. {issue}15772[15772] {pull}15795[15795]
+- Add kubernetes storage class support via kube-state-metrics. {pull}16145[16145]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -22495,6 +22495,58 @@ type: long
 
 --
 
+[float]
+=== storageclass
+
+kubernetes storage class metrics
+
+
+
+*`kubernetes.storageclass.name`*::
++
+--
+Storage class name.
+
+type: keyword
+
+--
+
+*`kubernetes.storageclass.provisioner`*::
++
+--
+Volume provisioner for the storage class.
+
+type: keyword
+
+--
+
+*`kubernetes.storageclass.reclaim_policy`*::
++
+--
+Reclaim policy for dynamically created volumes
+
+type: keyword
+
+--
+
+*`kubernetes.storageclass.volume_binding_mode`*::
++
+--
+Mode for default provisioning and binding
+
+type: keyword
+
+--
+
+*`kubernetes.storageclass.created`*::
++
+--
+Storage class creation date
+
+type: date
+
+--
+
 [float]
 === system
 

diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc
@@ -99,6 +99,7 @@ metricbeat.modules:
     - state_service
     - state_persistentvolume
     - state_persistentvolumeclaim
+    - state_storageclass
   period: 10s
   hosts: ["kube-state-metrics:8080"]
 
@@ -196,6 +197,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-kubernetes-state_statefulset,state_statefulset>>
 
+* <<metricbeat-metricset-kubernetes-state_storageclass,state_storageclass>>
+
 * <<metricbeat-metricset-kubernetes-system,system>>
 
 * <<metricbeat-metricset-kubernetes-volume,volume>>
@@ -236,6 +239,8 @@ include::kubernetes/state_service.asciidoc[]
 
 include::kubernetes/state_statefulset.asciidoc[]
 
+include::kubernetes/state_storageclass.asciidoc[]
+
 include::kubernetes/system.asciidoc[]
 
 include::kubernetes/volume.asciidoc[]

diff --git a/metricbeat/docs/modules/kubernetes/state_storageclass.asciidoc b/metricbeat/docs/modules/kubernetes/state_storageclass.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-kubernetes-state_storageclass]]
+=== Kubernetes state_storageclass metricset
+
+experimental[]
+
+include::../../../module/kubernetes/state_storageclass/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-kubernetes,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/kubernetes/state_storageclass/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -121,7 +121,7 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-kibana-stats,stats>>   
 |<<metricbeat-metricset-kibana-status,status>>   
 |<<metricbeat-module-kubernetes,Kubernetes>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.20+| .20+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
+.21+| .21+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
 |<<metricbeat-metricset-kubernetes-container,container>>   
 |<<metricbeat-metricset-kubernetes-controllermanager,controllermanager>>   
 |<<metricbeat-metricset-kubernetes-event,event>>   
@@ -139,6 +139,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-kubernetes-state_resourcequota,state_resourcequota>>   
 |<<metricbeat-metricset-kubernetes-state_service,state_service>> experimental[]  
 |<<metricbeat-metricset-kubernetes-state_statefulset,state_statefulset>>   
+|<<metricbeat-metricset-kubernetes-state_storageclass,state_storageclass>> experimental[]  
 |<<metricbeat-metricset-kubernetes-system,system>>   
 |<<metricbeat-metricset-kubernetes-volume,volume>>   
 |<<metricbeat-module-kvm,kvm>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  

diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -482,6 +482,7 @@ metricbeat.modules:
     - state_service
     - state_persistentvolume
     - state_persistentvolumeclaim
+    - state_storageclass
   period: 10s
   hosts: ["kube-state-metrics:8080"]
 

diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml
@@ -39,6 +39,7 @@
     - state_service
     - state_persistentvolume
     - state_persistentvolumeclaim
+    - state_storageclass
   period: 10s
   hosts: ["kube-state-metrics:8080"]
 

diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml
@@ -38,6 +38,7 @@
 #    - state_service
 #    - state_persistentvolume
 #    - state_persistentvolumeclaim
+#    - state_storageclass
 #  period: 10s
 #  hosts: ["kube-state-metrics:8080"]
 #  add_metadata: true

diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go
diff --git a/metricbeat/module/kubernetes/state_storageclass/README.md b/metricbeat/module/kubernetes/state_storageclass/README.md
@@ -0,0 +1,47 @@
+# Kube-state-metrics/StorageClass
+
+This metricset connects to kube-state-metrics endpoint to retrieve and report Storage Class metrics.
+
+Interestingly enough kube-state-metrics does not repport annotations, we are unable to inform which storage class is default. We can consider enriching adding that info, or contributing back to kube-state-metrics to add annotations.
+
+## Version history
+
+- February 2020, first release using kube-state-metrics `v1.8.0`.
+
+## Configuration
+
+See the metricset documentation for the configuration reference.
+
+## Manual testing
+
+Probably your kubernetes cluster already has a storage class. You can add extra SCs:
+
+Example:
+
+```bash
+kubectl apply -f - << EOF
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: beats-test-sc1
+  labels:
+    testl1: value1
+    testl2: value2
+provisioner: kubernetes.io/non-existing1
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: beats-test-sc2
+  labels:
+    testl3: value3
+    testl4: value4
+provisioner: kubernetes.io/non-existing2
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+EOF
+```
+
+Then run metricbeat pointing to the kube-state-metrics endpoint.