From 997b5cedfe6d83308366eb581c9198c8d464d136 Mon Sep 17 00:00:00 2001 From: Mario Constanti Date: Tue, 21 Jun 2022 05:41:09 +0200 Subject: [PATCH 1/2] add new kube_endpoint_address metric As KSM shouldn't be responsible for computation of metrics, a new metric is needed to expose available and unavailable addresses of an endpoint. This new metric is generated as a future replacement for kube_endpoint_address_available and kube_endpoint_address_not_ready. Signed-off-by: Mario Constanti --- docs/endpoint-metrics.md | 1 + internal/store/endpoint.go | 28 ++++++++++++++++++++++ internal/store/endpoint_test.go | 42 +++++++++++++++++++++++++++++---- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/docs/endpoint-metrics.md b/docs/endpoint-metrics.md index f4ec67baee..db69887d38 100644 --- a/docs/endpoint-metrics.md +++ b/docs/endpoint-metrics.md @@ -9,3 +9,4 @@ | kube_endpoint_labels | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`label_ENDPOINT_LABEL`=<ENDPOINT_LABEL> | STABLE | | kube_endpoint_created | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | STABLE | | kube_endpoint_ports | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`port_name`=<endpoint-port-name>
`port_protocol`=<endpoint-port-protocol>
`port_number`=<endpoint-port-number> | EXPERIMENTAL | +| kube_endpoint_address | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`available_ip`=<endpoint-available-ip>
`unavailable_ip`=<endpoint-unavailable_ip> | EXPERIMENTAL | diff --git a/internal/store/endpoint.go b/internal/store/endpoint.go index 4cd55a5eaa..785e57e8f5 100644 --- a/internal/store/endpoint.go +++ b/internal/store/endpoint.go @@ -151,6 +151,34 @@ func endpointMetricFamilies(allowAnnotationsList, allowLabelsList []string) []ge } }), ), + *generator.NewFamilyGenerator( + "kube_endpoint_address", + "Information about Endpoint available and non available addresses.", + metric.Gauge, + "", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + ms := []*metric.Metric{} + for _, s := range e.Subsets { + for _, available := range s.Addresses { + ms = append(ms, &metric.Metric{ + LabelValues: []string{available.IP}, + LabelKeys: []string{"available_ip"}, + Value: 1, + }) + } + for _, notReadyAddresses := range s.NotReadyAddresses { + ms = append(ms, &metric.Metric{ + LabelValues: []string{notReadyAddresses.IP}, + LabelKeys: []string{"unavailable_ip"}, + Value: 1, + }) + } + } + return &metric.Family{ + Metrics: ms, + } + }), + ), *generator.NewFamilyGenerator( "kube_endpoint_ports", "Information about the Endpoint ports.", diff --git a/internal/store/endpoint_test.go b/internal/store/endpoint_test.go index 57c0250e70..838b055b1c 100644 --- a/internal/store/endpoint_test.go +++ b/internal/store/endpoint_test.go @@ -44,6 +44,8 @@ func TestEndpointStore(t *testing.T) { # TYPE kube_endpoint_labels gauge # HELP kube_endpoint_ports Information about the Endpoint ports. # TYPE kube_endpoint_ports gauge + # HELP kube_endpoint_address Information about Endpoint available and non available addresses. + # TYPE kube_endpoint_address gauge ` cases := []generateMetricsTestCase{ { @@ -61,6 +63,9 @@ func TestEndpointStore(t *testing.T) { Addresses: []v1.EndpointAddress{ {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, Ports: []v1.EndpointPort{ {Port: 8080, Name: "http", Protocol: v1.ProtocolTCP}, {Port: 8081, Name: "app", Protocol: v1.ProtocolTCP}, }, @@ -86,7 +91,7 @@ func TestEndpointStore(t *testing.T) { Want: metadata + ` kube_endpoint_annotations{endpoint="test-endpoint",namespace="default"} 1 kube_endpoint_address_available{endpoint="test-endpoint",namespace="default"} 6 - kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 4 + kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 6 kube_endpoint_created{endpoint="test-endpoint",namespace="default"} 1.5e+09 kube_endpoint_info{endpoint="test-endpoint",namespace="default"} 1 kube_endpoint_labels{endpoint="test-endpoint",namespace="default"} 1 @@ -96,6 +101,12 @@ func TestEndpointStore(t *testing.T) { kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="127.0.0.1"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="10.0.0.1"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="172.22.23.202"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.1.3"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.2.2"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 `, }, { @@ -113,6 +124,9 @@ func TestEndpointStore(t *testing.T) { Addresses: []v1.EndpointAddress{ {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, Ports: []v1.EndpointPort{ {Port: 8080, Protocol: v1.ProtocolTCP}, }, @@ -122,11 +136,14 @@ func TestEndpointStore(t *testing.T) { Want: metadata + ` kube_endpoint_annotations{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_address_available{endpoint="single-port-endpoint",namespace="default"} 2 - kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 0 + kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_created{endpoint="single-port-endpoint",namespace="default"} 1.5e+09 kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_labels{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="127.0.0.1"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="10.0.0.1"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 `, }, } @@ -157,6 +174,8 @@ func TestEndpointStoreWithLabels(t *testing.T) { # TYPE kube_endpoint_labels gauge # HELP kube_endpoint_ports Information about the Endpoint ports. # TYPE kube_endpoint_ports gauge + # HELP kube_endpoint_address Information about Endpoint available and non available addresses. + # TYPE kube_endpoint_address gauge ` cases := []generateMetricsTestCase{ { @@ -177,6 +196,9 @@ func TestEndpointStoreWithLabels(t *testing.T) { Addresses: []v1.EndpointAddress{ {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, Ports: []v1.EndpointPort{ {Port: 8080, Name: "http", Protocol: v1.ProtocolTCP}, {Port: 8081, Name: "app", Protocol: v1.ProtocolTCP}, }, @@ -201,7 +223,7 @@ func TestEndpointStoreWithLabels(t *testing.T) { }, Want: metadata + ` kube_endpoint_address_available{endpoint="test-endpoint",namespace="default"} 6 - kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 4 + kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 6 kube_endpoint_annotations{endpoint="test-endpoint",annotation_app="foobar",namespace="default"} 1 kube_endpoint_created{endpoint="test-endpoint",namespace="default"} 1.5e+09 kube_endpoint_info{endpoint="test-endpoint",namespace="default"} 1 @@ -212,6 +234,12 @@ func TestEndpointStoreWithLabels(t *testing.T) { kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="127.0.0.1"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="10.0.0.1"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="172.22.23.202"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.1.3"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.2.2"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 `, }, { @@ -232,6 +260,9 @@ func TestEndpointStoreWithLabels(t *testing.T) { Addresses: []v1.EndpointAddress{ {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, Ports: []v1.EndpointPort{ {Port: 8080, Protocol: v1.ProtocolTCP}, }, @@ -241,11 +272,14 @@ func TestEndpointStoreWithLabels(t *testing.T) { Want: metadata + ` kube_endpoint_annotations{endpoint="single-port-endpoint",annotation_app="single-foobar",namespace="default"} 1 kube_endpoint_address_available{endpoint="single-port-endpoint",namespace="default"} 2 - kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 0 + kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_created{endpoint="single-port-endpoint",namespace="default"} 1.5e+09 kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_labels{endpoint="single-port-endpoint",label_app="single-foobar",namespace="default"} 1 kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="127.0.0.1"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="10.0.0.1"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 `, }, } From 6082c0335c7cfb33c84f8aebe9e4eb44d8ebaae9 Mon Sep 17 00:00:00 2001 From: Mario Constanti Date: Wed, 22 Jun 2022 13:35:33 +0200 Subject: [PATCH 2/2] kube_endpoint_address now has a ready label Signed-off-by: Mario Constanti --- docs/endpoint-metrics.md | 2 +- internal/store/endpoint.go | 8 ++++---- internal/store/endpoint_test.go | 36 ++++++++++++++++----------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/endpoint-metrics.md b/docs/endpoint-metrics.md index db69887d38..6afc817488 100644 --- a/docs/endpoint-metrics.md +++ b/docs/endpoint-metrics.md @@ -9,4 +9,4 @@ | kube_endpoint_labels | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`label_ENDPOINT_LABEL`=<ENDPOINT_LABEL> | STABLE | | kube_endpoint_created | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | STABLE | | kube_endpoint_ports | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`port_name`=<endpoint-port-name>
`port_protocol`=<endpoint-port-protocol>
`port_number`=<endpoint-port-number> | EXPERIMENTAL | -| kube_endpoint_address | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`available_ip`=<endpoint-available-ip>
`unavailable_ip`=<endpoint-unavailable_ip> | EXPERIMENTAL | +| kube_endpoint_address | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`ip`=<endpoint-ip>
`ready`=<true if available, false if unavailalbe> | EXPERIMENTAL | diff --git a/internal/store/endpoint.go b/internal/store/endpoint.go index 785e57e8f5..a2ab994339 100644 --- a/internal/store/endpoint.go +++ b/internal/store/endpoint.go @@ -161,15 +161,15 @@ func endpointMetricFamilies(allowAnnotationsList, allowLabelsList []string) []ge for _, s := range e.Subsets { for _, available := range s.Addresses { ms = append(ms, &metric.Metric{ - LabelValues: []string{available.IP}, - LabelKeys: []string{"available_ip"}, + LabelValues: []string{available.IP, "true"}, + LabelKeys: []string{"ip", "ready"}, Value: 1, }) } for _, notReadyAddresses := range s.NotReadyAddresses { ms = append(ms, &metric.Metric{ - LabelValues: []string{notReadyAddresses.IP}, - LabelKeys: []string{"unavailable_ip"}, + LabelValues: []string{notReadyAddresses.IP, "false"}, + LabelKeys: []string{"ip", "ready"}, Value: 1, }) } diff --git a/internal/store/endpoint_test.go b/internal/store/endpoint_test.go index 838b055b1c..69af58e791 100644 --- a/internal/store/endpoint_test.go +++ b/internal/store/endpoint_test.go @@ -101,12 +101,12 @@ func TestEndpointStore(t *testing.T) { kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="127.0.0.1"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="10.0.0.1"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="172.22.23.202"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.1.3"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.2.2"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="172.22.23.202",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.1.3",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.2.2",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 `, }, { @@ -141,9 +141,9 @@ func TestEndpointStore(t *testing.T) { kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_labels{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1 - kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="127.0.0.1"} 1 - kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="10.0.0.1"} 1 - kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 `, }, } @@ -234,12 +234,12 @@ func TestEndpointStoreWithLabels(t *testing.T) { kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1 kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="127.0.0.1"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="10.0.0.1"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",available_ip="172.22.23.202"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.1.3"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="192.168.2.2"} 1 - kube_endpoint_address{endpoint="test-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="172.22.23.202",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.1.3",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.2.2",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 `, }, { @@ -277,9 +277,9 @@ func TestEndpointStoreWithLabels(t *testing.T) { kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1 kube_endpoint_labels{endpoint="single-port-endpoint",label_app="single-foobar",namespace="default"} 1 kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1 - kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="127.0.0.1"} 1 - kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",available_ip="10.0.0.1"} 1 - kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",unavailable_ip="10.0.0.10"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 `, }, }