[kubernetes] Add last_terminated_reason_timestamp metric (#39200)

* add last_terminated_reason_timestamp metric Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * Update CHANGELOG.next.asciidoc Co-authored-by: Andrew Gizas <andreas.gkizas@elastic.co> * revert the metric name to status.last_terminated_reason_timestamp Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * rename field to last_terminated_timestamp Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * Update CHANGELOG.next.asciidoc --------- Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> Co-authored-by: Andrew Gizas <andreas.gkizas@elastic.co>
elastic · Apr 26, 2024 · 0c78454 · 0c78454
1 parent 074f221
commit 0c78454
Show file tree

Hide file tree

Showing 8 changed files with 102 additions and 79 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -281,6 +281,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new memory/cgroup metrics to Kibana module {pull}37232[37232]
 - Add SSL support to mysql module {pull}37997[37997]
 - Add SSL support for aerospike module {pull}38126[38126]
+- Add last_terminated_timestamp metric in kubernetes module {pull}39200[39200] {issue}3802[3802]
 
 
 *Metricbeat*

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -45985,6 +45985,16 @@ type: keyword
 
 --
 
+*`kubernetes.container.status.last_terminated_timestamp`*::
++
+--
+Last terminated time (epoch) of the container
+
+
+type: double
+
+--
+
 
 *`kubernetes.container.cpu.limit.cores`*::
 +

diff --git a/metricbeat/module/kubernetes/_meta/test/KSM/ksm.v2.12.0.plain b/metricbeat/module/kubernetes/_meta/test/KSM/ksm.v2.12.0.plain
@@ -552,10 +552,13 @@ kube_pod_container_state_started{namespace="kube-system",pod="kube-proxy-45qj9",
 kube_pod_container_state_started{namespace="kube-system",pod="kube-state-metrics-5bcd4898-bntgt",uid="38cad684-d5e5-4187-8a98-2999c0f5b252",container="kube-state-metrics"} 1.713873336e+09
 # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
 # TYPE kube_pod_container_status_last_terminated_reason gauge
+kube_pod_container_status_last_terminated_reason{namespace="kube-system",pod="kube-scheduler-kind-control-plane",uid="82154115-17ff-4943-8088-879d4d045b52",container="kube-scheduler",reason="Error"} 1
 # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
 # TYPE kube_pod_container_status_last_terminated_exitcode gauge
+kube_pod_container_status_last_terminated_exitcode{namespace="kube-system",pod="kube-scheduler-kind-control-plane",uid="82154115-17ff-4943-8088-879d4d045b52",container="kube-scheduler"} 1
 # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp.
 # TYPE kube_pod_container_status_last_terminated_timestamp gauge
+kube_pod_container_status_last_terminated_timestamp{namespace="kube-system",pod="kube-scheduler-kind-control-plane",uid="82154115-17ff-4943-8088-879d4d045b52",container="kube-scheduler"} 1.71393586e+09
 # HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded.
 # TYPE kube_pod_container_status_ready gauge
 kube_pod_container_status_ready{namespace="default",pod="hello-28564556-gkqsk",uid="888e4e15-5ffc-40cd-826d-af822fee80dd",container="hello"} 0

diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go
diff --git a/metricbeat/module/kubernetes/state_container/_meta/fields.yml b/metricbeat/module/kubernetes/state_container/_meta/fields.yml
@@ -32,6 +32,10 @@
           type: keyword
           description: >
             The last reason the container was in terminated state (Completed, ContainerCannotRun, Error or OOMKilled).
+        - name: last_terminated_timestamp
+          type: double
+          description: >
+            Last terminated time (epoch) of the container
     - name: cpu
       type: group
       fields:

diff --git a/metricbeat/module/kubernetes/state_container/_meta/test/ksm.v2.12.0.plain.expected b/metricbeat/module/kubernetes/state_container/_meta/test/ksm.v2.12.0.plain.expected
@@ -482,9 +482,9 @@
 	{
 		"RootFields": {
 			"container": {
-				"id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
+				"id": "1be2795038ca3a3a291a6aca5d7ecdfaf0b326453c74e471bab3064822f0bdfb",
 				"image": {
-					"name": "registry.k8s.io/kube-scheduler-amd64:v1.29.1"
+					"name": "registry.k8s.io/coredns/coredns:v1.11.1"
 				},
 				"runtime": "containerd"
 			}
@@ -495,7 +495,7 @@
 				"name": "kind-control-plane"
 			},
 			"pod": {
-				"name": "kube-scheduler-kind-control-plane"
+				"name": "coredns-76f75df574-v8skx"
 			}
 		},
 		"MetricSetFields": {
@@ -504,8 +504,16 @@
 					"cores": 0.1
 				}
 			},
-			"id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
-			"name": "kube-scheduler",
+			"id": "1be2795038ca3a3a291a6aca5d7ecdfaf0b326453c74e471bab3064822f0bdfb",
+			"memory": {
+				"limit": {
+					"bytes": 178257920
+				},
+				"request": {
+					"bytes": 73400320
+				}
+			},
+			"name": "coredns",
 			"status": {
 				"phase": "running",
 				"ready": true,
@@ -526,9 +534,9 @@
 	{
 		"RootFields": {
 			"container": {
-				"id": "1be2795038ca3a3a291a6aca5d7ecdfaf0b326453c74e471bab3064822f0bdfb",
+				"id": "9a40af12036875eb414555deeca2a09e73f37ccb7c8bbf787bfe9616cbbcbd98",
 				"image": {
-					"name": "registry.k8s.io/coredns/coredns:v1.11.1"
+					"name": "registry.k8s.io/kube-apiserver-amd64:v1.29.1"
 				},
 				"runtime": "containerd"
 			}
@@ -539,25 +547,17 @@
 				"name": "kind-control-plane"
 			},
 			"pod": {
-				"name": "coredns-76f75df574-v8skx"
+				"name": "kube-apiserver-kind-control-plane"
 			}
 		},
 		"MetricSetFields": {
 			"cpu": {
 				"request": {
-					"cores": 0.1
-				}
-			},
-			"id": "1be2795038ca3a3a291a6aca5d7ecdfaf0b326453c74e471bab3064822f0bdfb",
-			"memory": {
-				"limit": {
-					"bytes": 178257920
-				},
-				"request": {
-					"bytes": 73400320
+					"cores": 0.25
 				}
 			},
-			"name": "coredns",
+			"id": "9a40af12036875eb414555deeca2a09e73f37ccb7c8bbf787bfe9616cbbcbd98",
+			"name": "kube-apiserver",
 			"status": {
 				"phase": "running",
 				"ready": true,
@@ -578,9 +578,9 @@
 	{
 		"RootFields": {
 			"container": {
-				"id": "9a40af12036875eb414555deeca2a09e73f37ccb7c8bbf787bfe9616cbbcbd98",
+				"id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
 				"image": {
-					"name": "registry.k8s.io/kube-apiserver-amd64:v1.29.1"
+					"name": "registry.k8s.io/kube-scheduler-amd64:v1.29.1"
 				},
 				"runtime": "containerd"
 			}
@@ -591,18 +591,20 @@
 				"name": "kind-control-plane"
 			},
 			"pod": {
-				"name": "kube-apiserver-kind-control-plane"
+				"name": "kube-scheduler-kind-control-plane"
 			}
 		},
 		"MetricSetFields": {
 			"cpu": {
 				"request": {
-					"cores": 0.25
+					"cores": 0.1
 				}
 			},
-			"id": "9a40af12036875eb414555deeca2a09e73f37ccb7c8bbf787bfe9616cbbcbd98",
-			"name": "kube-apiserver",
+			"id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
+			"name": "kube-scheduler",
 			"status": {
+				"last_terminated_reason": "Error",
+				"last_terminated_timestamp": 1713935860,
 				"phase": "running",
 				"ready": true,
 				"restarts": 0

diff --git a/metricbeat/module/kubernetes/state_container/_meta/testdata/ksm.v2.12.0.plain-expected.json b/metricbeat/module/kubernetes/state_container/_meta/testdata/ksm.v2.12.0.plain-expected.json
@@ -269,6 +269,53 @@
             "type": "kubernetes"
         }
     },
+    {
+        "container": {
+            "id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
+            "image": {
+                "name": "registry.k8s.io/kube-scheduler-amd64:v1.29.1"
+            },
+            "runtime": "containerd"
+        },
+        "event": {
+            "dataset": "kubernetes.container",
+            "duration": 115000,
+            "module": "kubernetes"
+        },
+        "kubernetes": {
+            "container": {
+                "cpu": {
+                    "request": {
+                        "cores": 0.1
+                    }
+                },
+                "id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
+                "name": "kube-scheduler",
+                "status": {
+                    "last_terminated_reason": "Error",
+                    "last_terminated_timestamp": 1713935860,
+                    "phase": "running",
+                    "ready": true,
+                    "restarts": 0
+                }
+            },
+            "namespace": "kube-system",
+            "node": {
+                "name": "kind-control-plane"
+            },
+            "pod": {
+                "name": "kube-scheduler-kind-control-plane"
+            }
+        },
+        "metricset": {
+            "name": "state_container",
+            "period": 10000
+        },
+        "service": {
+            "address": "127.0.0.1:55555",
+            "type": "kubernetes"
+        }
+    },
     {
         "container": {
             "id": "be5708afe249a81a67f812bfda962b8c60022ec47250f9e84a2bcfd0d74e94bc",
@@ -581,51 +628,6 @@
             "type": "kubernetes"
         }
     },
-    {
-        "container": {
-            "id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
-            "image": {
-                "name": "registry.k8s.io/kube-scheduler-amd64:v1.29.1"
-            },
-            "runtime": "containerd"
-        },
-        "event": {
-            "dataset": "kubernetes.container",
-            "duration": 115000,
-            "module": "kubernetes"
-        },
-        "kubernetes": {
-            "container": {
-                "cpu": {
-                    "request": {
-                        "cores": 0.1
-                    }
-                },
-                "id": "cd52120cda244a98ed01e9259381cbcd439bc087115d93c7573bada504dbb620",
-                "name": "kube-scheduler",
-                "status": {
-                    "phase": "running",
-                    "ready": true,
-                    "restarts": 0
-                }
-            },
-            "namespace": "kube-system",
-            "node": {
-                "name": "kind-control-plane"
-            },
-            "pod": {
-                "name": "kube-scheduler-kind-control-plane"
-            }
-        },
-        "metricset": {
-            "name": "state_container",
-            "period": 10000
-        },
-        "service": {
-            "address": "127.0.0.1:55555",
-            "type": "kubernetes"
-        }
-    },
     {
         "container": {
             "id": "54c671509ea7b99312164a5a49297e48eb0b724a8cb2459d2d3917a25ce0c585",

diff --git a/metricbeat/module/kubernetes/state_container/state_container.go b/metricbeat/module/kubernetes/state_container/state_container.go
@@ -59,14 +59,15 @@ var (
 				},
 			)),
 
-			"kube_pod_container_status_ready":                  p.BooleanMetric("status.ready"),
-			"kube_pod_container_status_restarts_total":         p.Metric("status.restarts"),
-			"kube_pod_container_status_running":                p.KeywordMetric("status.phase", "running"),
-			"kube_pod_container_status_terminated":             p.KeywordMetric("status.phase", "terminated"),
-			"kube_pod_container_status_waiting":                p.KeywordMetric("status.phase", "waiting"),
-			"kube_pod_container_status_terminated_reason":      p.LabelMetric("status.reason", "reason"),
-			"kube_pod_container_status_waiting_reason":         p.LabelMetric("status.reason", "reason"),
-			"kube_pod_container_status_last_terminated_reason": p.LabelMetric("status.last_terminated_reason", "reason"),
+			"kube_pod_container_status_ready":                     p.BooleanMetric("status.ready"),
+			"kube_pod_container_status_restarts_total":            p.Metric("status.restarts"),
+			"kube_pod_container_status_running":                   p.KeywordMetric("status.phase", "running"),
+			"kube_pod_container_status_terminated":                p.KeywordMetric("status.phase", "terminated"),
+			"kube_pod_container_status_waiting":                   p.KeywordMetric("status.phase", "waiting"),
+			"kube_pod_container_status_terminated_reason":         p.LabelMetric("status.reason", "reason"),
+			"kube_pod_container_status_waiting_reason":            p.LabelMetric("status.reason", "reason"),
+			"kube_pod_container_status_last_terminated_reason":    p.LabelMetric("status.last_terminated_reason", "reason"),
+			"kube_pod_container_status_last_terminated_timestamp": p.Metric("status.last_terminated_timestamp"),
 		},
 
 		Labels: map[string]p.LabelMap{