From 51ccb665c40afbfec430800911258eb8b40d5dcf Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Thu, 28 Dec 2023 15:04:49 +0100 Subject: [PATCH 1/4] feat: add kube_pod_container_status_last_terminated_timestamp metric Signed-off-by: Tetiana Kravchenko --- docs/pod-metrics.md | 1 + internal/store/pod.go | 27 +++++++++++++++++++++++++++ internal/store/pod_test.go | 7 +++++++ 3 files changed, 35 insertions(+) diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index ee30c7f668..a57e57cafb 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -23,6 +23,7 @@ | kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_timestamp | Gauge | Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | | kube_pod_status_initialized_time | Gauge | Time when the pod is initialized. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | | kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | diff --git a/internal/store/pod.go b/internal/store/pod.go index 09a83d171b..9daec2d70e 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -49,6 +49,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodContainerStateStartedFamilyGenerator(), createPodContainerStatusLastTerminatedReasonFamilyGenerator(), createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(), + createPodContainerStatusLastTerminatedTimestampFamilyGenerator(), createPodContainerStatusReadyFamilyGenerator(), createPodContainerStatusRestartsTotalFamilyGenerator(), createPodContainerStatusRunningFamilyGenerator(), @@ -375,6 +376,32 @@ func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.F ) } +func createPodContainerStatusLastTerminatedTimestampFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_last_terminated_timestamp", + "Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses)) + for _, cs := range p.Status.ContainerStatuses { + if cs.LastTerminationState.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: float64(cs.LastTerminationState.Terminated.FinishedAt.Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_container_status_ready", diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 84f63cadb4..642efb0762 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -720,6 +720,9 @@ func TestPodStore(t *testing.T) { Terminated: &v1.ContainerStateTerminated{ Reason: "OOMKilled", ExitCode: 137, + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, }, }, }, @@ -729,6 +732,7 @@ func TestPodStore(t *testing.T) { Want: ` # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. + # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. @@ -737,6 +741,7 @@ func TestPodStore(t *testing.T) { # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # TYPE kube_pod_container_status_last_terminated_reason gauge # TYPE kube_pod_container_status_last_terminated_exitcode gauge + # TYPE kube_pod_container_status_last_terminated_timestamp gauge # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge @@ -749,10 +754,12 @@ func TestPodStore(t *testing.T) { kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0 kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1 kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137 + kube_pod_container_status_last_terminated_timestamp{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 1.501779547e+09 `, MetricNames: []string{ "kube_pod_container_status_last_terminated_reason", "kube_pod_container_status_last_terminated_exitcode", + "kube_pod_container_status_last_terminated_timestamp", "kube_pod_container_status_running", "kube_pod_container_state_started", "kube_pod_container_status_terminated", From 8876d40ecd79fa5e728b295d0caf93d67819a3c2 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 22 Jan 2024 15:36:58 +0100 Subject: [PATCH 2/4] add kube_pod_container_status_last_terminated_timestamp to the unit tests Signed-off-by: Tetiana Kravchenko --- pkg/app/server_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index f3d71aab8a..0d235a8836 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -209,6 +209,7 @@ func TestFullScrapeCycle(t *testing.T) { # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. +# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. # HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded. # HELP kube_pod_container_status_restarts_total [STABLE] The number of container restarts per container. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. @@ -261,6 +262,7 @@ func TestFullScrapeCycle(t *testing.T) { # TYPE kube_pod_container_state_started gauge # TYPE kube_pod_container_status_last_terminated_exitcode gauge # TYPE kube_pod_container_status_last_terminated_reason gauge +# TYPE kube_pod_container_status_last_terminated_timestamp gauge # TYPE kube_pod_container_status_ready gauge # TYPE kube_pod_container_status_restarts_total counter # TYPE kube_pod_container_status_running gauge @@ -323,6 +325,7 @@ kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0", kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08 kube_pod_container_status_last_terminated_exitcode{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 137 kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="OOMKilled"} 1 +kube_pod_container_status_last_terminated_timestamp{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 1.501779547e+09 kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0 kube_pod_container_status_restarts_total{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 @@ -834,6 +837,9 @@ func pod(client *fake.Clientset, index int) error { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, }, From dbdb77c920700aa06b932c00593c48a65f35edf3 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 22 Jan 2024 16:32:52 +0100 Subject: [PATCH 3/4] fix benchmark test; add missing kube_pod_container_status_last_terminated_timestamp in pod tests Signed-off-by: Tetiana Kravchenko --- internal/store/pod_test.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 642efb0762..95d30291b6 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -690,7 +690,6 @@ func TestPodStore(t *testing.T) { }, }, { - Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod6", @@ -797,6 +796,9 @@ func TestPodStore(t *testing.T) { Terminated: &v1.ContainerStateTerminated{ Reason: "DeadlineExceeded", ExitCode: 143, + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, }, }, }, @@ -806,6 +808,7 @@ func TestPodStore(t *testing.T) { Want: ` # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. + # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. @@ -814,6 +817,7 @@ func TestPodStore(t *testing.T) { # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. # TYPE kube_pod_container_status_last_terminated_exitcode gauge # TYPE kube_pod_container_status_last_terminated_reason gauge + # TYPE kube_pod_container_status_last_terminated_timestamp gauge # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_state_started gauge # TYPE kube_pod_container_status_terminated gauge @@ -823,6 +827,7 @@ func TestPodStore(t *testing.T) { kube_pod_container_state_started{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501777018e+09 kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 143 kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded",uid="uid7"} 1 + kube_pod_container_status_last_terminated_timestamp{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501779547e+09 kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1 kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0 kube_pod_container_status_waiting{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0 @@ -832,6 +837,7 @@ func TestPodStore(t *testing.T) { "kube_pod_container_state_started", "kube_pod_container_status_terminated", "kube_pod_container_status_terminated_reason", + "kube_pod_container_status_last_terminated_timestamp", "kube_pod_container_status_waiting", "kube_pod_container_status_last_terminated_reason", "kube_pod_container_status_last_terminated_exitcode", @@ -2217,6 +2223,9 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, }, @@ -2234,6 +2243,9 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, }, @@ -2251,6 +2263,9 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, }, @@ -2260,7 +2275,7 @@ func BenchmarkPodStore(b *testing.B) { }, } - expectedFamilies := 53 + expectedFamilies := 54 for n := 0; n < b.N; n++ { families := f(pod) if len(families) != expectedFamilies { From 393430ed86941c502e179deffff625fee1864f3b Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 4 Mar 2024 11:40:34 +0100 Subject: [PATCH 4/4] fix the description of the 'kube_pod_container_status_last_terminated_timestamp' metric Signed-off-by: Tetiana Kravchenko --- docs/pod-metrics.md | 2 +- internal/store/pod.go | 2 +- internal/store/pod_test.go | 4 ++-- pkg/app/server_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index a57e57cafb..995c220ceb 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -23,7 +23,7 @@ | kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | -| kube_pod_container_status_last_terminated_timestamp | Gauge | Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_timestamp | Gauge | Last terminated time for a pod container in unix timestamp. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | | kube_pod_status_initialized_time | Gauge | Time when the pod is initialized. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | | kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | diff --git a/internal/store/pod.go b/internal/store/pod.go index 9daec2d70e..82da1a43b3 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -379,7 +379,7 @@ func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.F func createPodContainerStatusLastTerminatedTimestampFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_container_status_last_terminated_timestamp", - "Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp.", + "Last terminated time for a pod container in unix timestamp.", metric.Gauge, basemetrics.ALPHA, "", diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 95d30291b6..49a5cd942e 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -731,7 +731,7 @@ func TestPodStore(t *testing.T) { Want: ` # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. - # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. + # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. @@ -808,7 +808,7 @@ func TestPodStore(t *testing.T) { Want: ` # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. - # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. + # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index 0d235a8836..88bf39d49e 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -209,7 +209,7 @@ func TestFullScrapeCycle(t *testing.T) { # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. -# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp. +# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp. # HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded. # HELP kube_pod_container_status_restarts_total [STABLE] The number of container restarts per container. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state.