From 865165ee4b930e68e956a86a9105dc8ecb541336 Mon Sep 17 00:00:00 2001 From: Slavik Panasovets Date: Tue, 7 Feb 2023 15:33:54 +0000 Subject: [PATCH] This PR adds counting service per ipFamilies, ipFamilyPolicy and Status for L4 NetLB DualStack --- pkg/metrics/l4metrics_test.go | 104 +++++++++++++++++++++++++++++++++- pkg/metrics/metrics.go | 76 ++++++++++++++++++++++--- pkg/metrics/types.go | 21 +++++-- 3 files changed, 186 insertions(+), 15 deletions(-) diff --git a/pkg/metrics/l4metrics_test.go b/pkg/metrics/l4metrics_test.go index be7759f93e..7dc02fd967 100644 --- a/pkg/metrics/l4metrics_test.go +++ b/pkg/metrics/l4metrics_test.go @@ -379,6 +379,100 @@ func TestComputeL4NetLBMetrics(t *testing.T) { } } +func TestComputeL4NetLBDualStackMetrics(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + desc string + serviceStates []L4NetLBDualStackServiceState + expectL4NetLBDualStackCount map[L4NetLBDualStackServiceState]int + }{ + { + desc: "empty input", + serviceStates: []L4NetLBDualStackServiceState{}, + expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{}, + }, + { + desc: "one l4 NetLB dual-stack service", + serviceStates: []L4NetLBDualStackServiceState{ + newL4NetLBDualStackServiceState("IPv4", "SingleStack", StatusSuccess), + }, + expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{ + L4NetLBDualStackServiceState{ + "IPv4", + "SingleStack", + StatusSuccess, + }: 1, + }, + }, + { + desc: "l4 NetLB dual-stack service in error state", + serviceStates: []L4NetLBDualStackServiceState{ + newL4NetLBDualStackServiceState("IPv4", "SingleStack", StatusError), + }, + expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{ + L4NetLBDualStackServiceState{ + "IPv4", + "SingleStack", + StatusError, + }: 1, + }, + }, + { + desc: "L4 NetLB dual-stack service with IPv4,IPv6 Families", + serviceStates: []L4NetLBDualStackServiceState{ + newL4NetLBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + }, + expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{ + L4NetLBDualStackServiceState{ + "IPv4,IPv6", + "RequireDualStack", + StatusSuccess, + }: 1, + }, + }, + { + desc: "many l4 NetLB dual-stack services", + serviceStates: []L4NetLBDualStackServiceState{ + newL4NetLBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + newL4NetLBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + newL4NetLBDualStackServiceState("IPv4", "SingleStack", StatusError), + newL4NetLBDualStackServiceState("IPv6", "SingleStack", StatusSuccess), + newL4NetLBDualStackServiceState("IPv6", "SingleStack", StatusSuccess), + }, + expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{ + L4NetLBDualStackServiceState{ + "IPv4,IPv6", + "RequireDualStack", + StatusSuccess, + }: 2, + L4NetLBDualStackServiceState{ + "IPv4", + "SingleStack", + StatusError, + }: 1, + L4NetLBDualStackServiceState{ + "IPv6", + "SingleStack", + StatusSuccess, + }: 2, + }, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + newMetrics := FakeControllerMetrics() + for i, serviceState := range tc.serviceStates { + newMetrics.SetL4NetLBDualStackService(fmt.Sprint(i), serviceState) + } + got := newMetrics.computeL4NetLBDualStackMetrics() + if diff := cmp.Diff(tc.expectL4NetLBDualStackCount, got); diff != "" { + t.Fatalf("Got diff for L4 NetLB Dual-Stack service counts (-want +got):\n%s", diff) + } + }) + } +} + func newL4NetLBServiceState(inSuccess, managed, premium, userError bool, errorTimestamp *time.Time) L4NetLBServiceState { return L4NetLBServiceState{ IsPremiumTier: premium, @@ -525,10 +619,18 @@ func TestComputeL4ILBDualStackMetrics(t *testing.T) { } } -func newL4ILBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4ILBDualStackServiceStateStatus) L4ILBDualStackServiceState { +func newL4ILBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4DualStackServiceStateStatus) L4ILBDualStackServiceState { return L4ILBDualStackServiceState{ IPFamilies: ipFamilies, IPFamilyPolicy: ipFamilyPolicy, Status: status, } } + +func newL4NetLBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4DualStackServiceStateStatus) L4NetLBDualStackServiceState { + return L4NetLBDualStackServiceState{ + IPFamilies: ipFamilies, + IPFamilyPolicy: ipFamilyPolicy, + Status: status, + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a6201ef746..cfa7a911d9 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -76,6 +76,13 @@ var ( }, []string{"ip_families", "ip_family_policy", "status"}, ) + l4NetLBDualStackCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "number_of_l4_dual_stack_netlbs", + Help: "Number of L4 NetLBs with DualStack enabled", + }, + []string{"ip_families", "ip_family_policy", "status"}, + ) l4NetLBCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "number_of_l4_netlbs", @@ -140,6 +147,9 @@ func init() { klog.V(3).Infof("Registering L4 NetLB usage metrics %v", l4NetLBCount) prometheus.MustRegister(l4NetLBCount) + klog.V(3).Infof("Registering L4 NetLB Dual Stack usage metrics %v", l4NetLBDualStackCount) + prometheus.MustRegister(l4NetLBDualStackCount) + klog.V(3).Infof("Registering PSC usage metrics %v", serviceAttachmentCount) prometheus.MustRegister(serviceAttachmentCount) prometheus.MustRegister(serviceCount) @@ -167,6 +177,8 @@ type ControllerMetrics struct { l4ILBDualStackServiceMap map[string]L4ILBDualStackServiceState // l4NetLBServiceMap is a map between service key and L4 NetLB service state. l4NetLBServiceMap map[string]L4NetLBServiceState + // l4NetLBDualStackServiceMap is a map between service key and L4 NetLB DualStack service state. + l4NetLBDualStackServiceMap map[string]L4NetLBDualStackServiceState // pscMap is a map between the service attachment key and PSC state pscMap map[string]pscmetrics.PSCState // ServiceMap track the number of services in this cluster @@ -182,15 +194,16 @@ type ControllerMetrics struct { // NewControllerMetrics initializes ControllerMetrics and starts a go routine to compute and export metrics periodically. func NewControllerMetrics(exportInterval, l4NetLBProvisionDeadline time.Duration) *ControllerMetrics { return &ControllerMetrics{ - ingressMap: make(map[string]IngressState), - negMap: make(map[string]NegServiceState), - l4ILBServiceMap: make(map[string]L4ILBServiceState), - l4ILBDualStackServiceMap: make(map[string]L4ILBDualStackServiceState), - l4NetLBServiceMap: make(map[string]L4NetLBServiceState), - pscMap: make(map[string]pscmetrics.PSCState), - serviceMap: make(map[string]struct{}), - metricsInterval: exportInterval, - l4NetLBProvisionDeadline: l4NetLBProvisionDeadline, + ingressMap: make(map[string]IngressState), + negMap: make(map[string]NegServiceState), + l4ILBServiceMap: make(map[string]L4ILBServiceState), + l4ILBDualStackServiceMap: make(map[string]L4ILBDualStackServiceState), + l4NetLBServiceMap: make(map[string]L4NetLBServiceState), + l4NetLBDualStackServiceMap: make(map[string]L4NetLBDualStackServiceState), + pscMap: make(map[string]pscmetrics.PSCState), + serviceMap: make(map[string]struct{}), + metricsInterval: exportInterval, + l4NetLBProvisionDeadline: l4NetLBProvisionDeadline, } } @@ -330,6 +343,25 @@ func (im *ControllerMetrics) DeleteL4NetLBService(svcKey string) { delete(im.l4NetLBServiceMap, svcKey) } +// SetL4NetLBDualStackService implements L4NetLBMetricsCollector. +func (im *ControllerMetrics) SetL4NetLBDualStackService(svcKey string, state L4NetLBDualStackServiceState) { + im.Lock() + defer im.Unlock() + + if im.l4NetLBDualStackServiceMap == nil { + klog.Fatalf("L4 NetLB DualStack Metrics failed to initialize correctly.") + } + im.l4NetLBDualStackServiceMap[svcKey] = state +} + +// DeleteL4NetLBDualStackService implements L4NetLBMetricsCollector. +func (im *ControllerMetrics) DeleteL4NetLBDualStackService(svcKey string) { + im.Lock() + defer im.Unlock() + + delete(im.l4NetLBDualStackServiceMap, svcKey) +} + // SetServiceAttachment adds sa state to the map to be counted during metrics computation. // SetServiceAttachment implements PSCMetricsCollector. func (im *ControllerMetrics) SetServiceAttachment(saKey string, state pscmetrics.PSCState) { @@ -415,6 +447,17 @@ func (im *ControllerMetrics) export() { klog.V(3).Infof("L4 NetLB usage metrics exported.") + netlbDualStackCount := im.computeL4NetLBDualStackMetrics() + klog.V(3).Infof("Exporting L4 NetLB DualStack usage metrics: %#v", netlbDualStackCount) + for state, count := range netlbDualStackCount { + l4NetLBDualStackCount.With(prometheus.Labels{ + "ip_families": state.IPFamilies, + "ip_family_policy": state.IPFamilyPolicy, + "status": string(state.Status), + }).Set(float64(count)) + } + klog.V(3).Infof("L4 Netlb DualStack usage metrics exported.") + saCount := im.computePSCMetrics() klog.V(3).Infof("Exporting PSC Usage Metrics: %#v", saCount) for feature, count := range saCount { @@ -616,6 +659,21 @@ func (im *ControllerMetrics) computeL4NetLBMetrics() netLBFeatureCount { return counts } +// computeL4NetLBDualStackMetrics aggregates L4 NetLB DualStack metrics in the cache. +func (im *ControllerMetrics) computeL4NetLBDualStackMetrics() map[L4NetLBDualStackServiceState]int { + im.Lock() + defer im.Unlock() + klog.V(4).Infof("Computing L4 DualStack NetLB usage metrics from service state map: %#v", im.l4NetLBDualStackServiceMap) + counts := map[L4NetLBDualStackServiceState]int{} + + for key, state := range im.l4NetLBDualStackServiceMap { + klog.V(6).Infof("NetLB Service %s has IPFamilies: %v, IPFamilyPolicy: %t, Status: %v", key, state.IPFamilies, state.IPFamilyPolicy, state.Status) + counts[state]++ + } + klog.V(4).Info("L4 NetLB usage metrics computed.") + return counts +} + func (im *ControllerMetrics) computePSCMetrics() map[feature]int { im.Lock() defer im.Unlock() diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go index 5eb825abaa..b2f3cf813d 100644 --- a/pkg/metrics/types.go +++ b/pkg/metrics/types.go @@ -73,11 +73,11 @@ type L4ILBServiceState struct { IsUserError bool } -type L4ILBDualStackServiceStateStatus string +type L4DualStackServiceStateStatus string -const StatusSuccess = L4ILBDualStackServiceStateStatus("Success") -const StatusUserError = L4ILBDualStackServiceStateStatus("UserError") -const StatusError = L4ILBDualStackServiceStateStatus("Error") +const StatusSuccess = L4DualStackServiceStateStatus("Success") +const StatusUserError = L4DualStackServiceStateStatus("UserError") +const StatusError = L4DualStackServiceStateStatus("Error") // L4ILBDualStackServiceState defines ipFamilies, ipFamilyPolicy and status // of L4 ILB DualStack service @@ -87,7 +87,18 @@ type L4ILBDualStackServiceState struct { // IPFamilyPolicy specifies spec.IPFamilyPolicy of Service IPFamilyPolicy string // Status specifies status of L4 ILB DualStack - Status L4ILBDualStackServiceStateStatus + Status L4DualStackServiceStateStatus +} + +// L4NetLBDualStackServiceState defines ipFamilies, ipFamilyPolicy and status +// of L4 NetLB DualStack service +type L4NetLBDualStackServiceState struct { + // IPFamilies stores spec.ipFamilies of Service + IPFamilies string + // IPFamilyPolicy specifies spec.IPFamilyPolicy of Service + IPFamilyPolicy string + // Status specifies status of L4 ILB DualStack + Status L4DualStackServiceStateStatus } // L4NetLBServiceState defines if network tier is premium and