diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index ee30c7f668..995c220ceb 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -23,6 +23,7 @@ | kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_timestamp | Gauge | Last terminated time for a pod container in unix timestamp. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | | kube_pod_status_initialized_time | Gauge | Time when the pod is initialized. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | | kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | diff --git a/internal/store/pod.go b/internal/store/pod.go index 09a83d171b..82da1a43b3 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -49,6 +49,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodContainerStateStartedFamilyGenerator(), createPodContainerStatusLastTerminatedReasonFamilyGenerator(), createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(), + createPodContainerStatusLastTerminatedTimestampFamilyGenerator(), createPodContainerStatusReadyFamilyGenerator(), createPodContainerStatusRestartsTotalFamilyGenerator(), createPodContainerStatusRunningFamilyGenerator(), @@ -375,6 +376,32 @@ func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.F ) } +func createPodContainerStatusLastTerminatedTimestampFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_last_terminated_timestamp", + "Last terminated time for a pod container in unix timestamp.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses)) + for _, cs := range p.Status.ContainerStatuses { + if cs.LastTerminationState.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: float64(cs.LastTerminationState.Terminated.FinishedAt.Unix()), + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_container_status_ready", diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 84f63cadb4..49a5cd942e 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -690,7 +690,6 @@ func TestPodStore(t *testing.T) { }, }, { - Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod6", @@ -720,6 +719,9 @@ func TestPodStore(t *testing.T) { Terminated: &v1.ContainerStateTerminated{ Reason: "OOMKilled", ExitCode: 137, + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, }, }, }, @@ -729,6 +731,7 @@ func TestPodStore(t *testing.T) { Want: ` # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. + # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. @@ -737,6 +740,7 @@ func TestPodStore(t *testing.T) { # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # TYPE kube_pod_container_status_last_terminated_reason gauge # TYPE kube_pod_container_status_last_terminated_exitcode gauge + # TYPE kube_pod_container_status_last_terminated_timestamp gauge # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge @@ -749,10 +753,12 @@ func TestPodStore(t *testing.T) { kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0 kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1 kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137 + kube_pod_container_status_last_terminated_timestamp{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 1.501779547e+09 `, MetricNames: []string{ "kube_pod_container_status_last_terminated_reason", "kube_pod_container_status_last_terminated_exitcode", + "kube_pod_container_status_last_terminated_timestamp", "kube_pod_container_status_running", "kube_pod_container_state_started", "kube_pod_container_status_terminated", @@ -790,6 +796,9 @@ func TestPodStore(t *testing.T) { Terminated: &v1.ContainerStateTerminated{ Reason: "DeadlineExceeded", ExitCode: 143, + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, }, }, }, @@ -799,6 +808,7 @@ func TestPodStore(t *testing.T) { Want: ` # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. + # HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. @@ -807,6 +817,7 @@ func TestPodStore(t *testing.T) { # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. # TYPE kube_pod_container_status_last_terminated_exitcode gauge # TYPE kube_pod_container_status_last_terminated_reason gauge + # TYPE kube_pod_container_status_last_terminated_timestamp gauge # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_state_started gauge # TYPE kube_pod_container_status_terminated gauge @@ -816,6 +827,7 @@ func TestPodStore(t *testing.T) { kube_pod_container_state_started{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501777018e+09 kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 143 kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded",uid="uid7"} 1 + kube_pod_container_status_last_terminated_timestamp{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501779547e+09 kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1 kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0 kube_pod_container_status_waiting{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0 @@ -825,6 +837,7 @@ func TestPodStore(t *testing.T) { "kube_pod_container_state_started", "kube_pod_container_status_terminated", "kube_pod_container_status_terminated_reason", + "kube_pod_container_status_last_terminated_timestamp", "kube_pod_container_status_waiting", "kube_pod_container_status_last_terminated_reason", "kube_pod_container_status_last_terminated_exitcode", @@ -2210,6 +2223,9 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, }, @@ -2227,6 +2243,9 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, }, @@ -2244,6 +2263,9 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, }, @@ -2253,7 +2275,7 @@ func BenchmarkPodStore(b *testing.B) { }, } - expectedFamilies := 53 + expectedFamilies := 54 for n := 0; n < b.N; n++ { families := f(pod) if len(families) != expectedFamilies { diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index f3d71aab8a..88bf39d49e 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -209,6 +209,7 @@ func TestFullScrapeCycle(t *testing.T) { # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. +# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp. # HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded. # HELP kube_pod_container_status_restarts_total [STABLE] The number of container restarts per container. # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. @@ -261,6 +262,7 @@ func TestFullScrapeCycle(t *testing.T) { # TYPE kube_pod_container_state_started gauge # TYPE kube_pod_container_status_last_terminated_exitcode gauge # TYPE kube_pod_container_status_last_terminated_reason gauge +# TYPE kube_pod_container_status_last_terminated_timestamp gauge # TYPE kube_pod_container_status_ready gauge # TYPE kube_pod_container_status_restarts_total counter # TYPE kube_pod_container_status_running gauge @@ -323,6 +325,7 @@ kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0", kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08 kube_pod_container_status_last_terminated_exitcode{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 137 kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="OOMKilled"} 1 +kube_pod_container_status_last_terminated_timestamp{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 1.501779547e+09 kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0 kube_pod_container_status_restarts_total{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 @@ -834,6 +837,9 @@ func pod(client *fake.Clientset, index int) error { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ + FinishedAt: metav1.Time{ + Time: time.Unix(1501779547, 0), + }, Reason: "OOMKilled", ExitCode: 137, },