This PR adds counting service per ipFamilies, ipFamilyPolicy and Stat…

…us for L4 NetLB DualStack
kubernetes · Feb 7, 2023 · 865165e · 865165e
1 parent d4ccf21
commit 865165e
Show file tree

Hide file tree

Showing 3 changed files with 186 additions and 15 deletions.
diff --git a/pkg/metrics/l4metrics_test.go b/pkg/metrics/l4metrics_test.go
@@ -379,6 +379,100 @@ func TestComputeL4NetLBMetrics(t *testing.T) {
 	}
 }
 
+func TestComputeL4NetLBDualStackMetrics(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		desc                        string
+		serviceStates               []L4NetLBDualStackServiceState
+		expectL4NetLBDualStackCount map[L4NetLBDualStackServiceState]int
+	}{
+		{
+			desc:                        "empty input",
+			serviceStates:               []L4NetLBDualStackServiceState{},
+			expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{},
+		},
+		{
+			desc: "one l4 NetLB dual-stack service",
+			serviceStates: []L4NetLBDualStackServiceState{
+				newL4NetLBDualStackServiceState("IPv4", "SingleStack", StatusSuccess),
+			},
+			expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{
+				L4NetLBDualStackServiceState{
+					"IPv4",
+					"SingleStack",
+					StatusSuccess,
+				}: 1,
+			},
+		},
+		{
+			desc: "l4 NetLB dual-stack service in error state",
+			serviceStates: []L4NetLBDualStackServiceState{
+				newL4NetLBDualStackServiceState("IPv4", "SingleStack", StatusError),
+			},
+			expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{
+				L4NetLBDualStackServiceState{
+					"IPv4",
+					"SingleStack",
+					StatusError,
+				}: 1,
+			},
+		},
+		{
+			desc: "L4 NetLB dual-stack service with IPv4,IPv6 Families",
+			serviceStates: []L4NetLBDualStackServiceState{
+				newL4NetLBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
+			},
+			expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{
+				L4NetLBDualStackServiceState{
+					"IPv4,IPv6",
+					"RequireDualStack",
+					StatusSuccess,
+				}: 1,
+			},
+		},
+		{
+			desc: "many l4 NetLB dual-stack services",
+			serviceStates: []L4NetLBDualStackServiceState{
+				newL4NetLBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
+				newL4NetLBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
+				newL4NetLBDualStackServiceState("IPv4", "SingleStack", StatusError),
+				newL4NetLBDualStackServiceState("IPv6", "SingleStack", StatusSuccess),
+				newL4NetLBDualStackServiceState("IPv6", "SingleStack", StatusSuccess),
+			},
+			expectL4NetLBDualStackCount: map[L4NetLBDualStackServiceState]int{
+				L4NetLBDualStackServiceState{
+					"IPv4,IPv6",
+					"RequireDualStack",
+					StatusSuccess,
+				}: 2,
+				L4NetLBDualStackServiceState{
+					"IPv4",
+					"SingleStack",
+					StatusError,
+				}: 1,
+				L4NetLBDualStackServiceState{
+					"IPv6",
+					"SingleStack",
+					StatusSuccess,
+				}: 2,
+			},
+		},
+	} {
+		tc := tc
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
+			newMetrics := FakeControllerMetrics()
+			for i, serviceState := range tc.serviceStates {
+				newMetrics.SetL4NetLBDualStackService(fmt.Sprint(i), serviceState)
+			}
+			got := newMetrics.computeL4NetLBDualStackMetrics()
+			if diff := cmp.Diff(tc.expectL4NetLBDualStackCount, got); diff != "" {
+				t.Fatalf("Got diff for L4 NetLB Dual-Stack service counts (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
 func newL4NetLBServiceState(inSuccess, managed, premium, userError bool, errorTimestamp *time.Time) L4NetLBServiceState {
 	return L4NetLBServiceState{
 		IsPremiumTier:      premium,
@@ -525,10 +619,18 @@ func TestComputeL4ILBDualStackMetrics(t *testing.T) {
 	}
 }
 
-func newL4ILBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4ILBDualStackServiceStateStatus) L4ILBDualStackServiceState {
+func newL4ILBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4DualStackServiceStateStatus) L4ILBDualStackServiceState {
 	return L4ILBDualStackServiceState{
 		IPFamilies:     ipFamilies,
 		IPFamilyPolicy: ipFamilyPolicy,
 		Status:         status,
 	}
 }
+
+func newL4NetLBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4DualStackServiceStateStatus) L4NetLBDualStackServiceState {
+	return L4NetLBDualStackServiceState{
+		IPFamilies:     ipFamilies,
+		IPFamilyPolicy: ipFamilyPolicy,
+		Status:         status,
+	}
+}
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -76,6 +76,13 @@ var (
 		},
 		[]string{"ip_families", "ip_family_policy", "status"},
 	)
+	l4NetLBDualStackCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "number_of_l4_dual_stack_netlbs",
+			Help: "Number of L4 NetLBs with DualStack enabled",
+		},
+		[]string{"ip_families", "ip_family_policy", "status"},
+	)
 	l4NetLBCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "number_of_l4_netlbs",
@@ -140,6 +147,9 @@ func init() {
 	klog.V(3).Infof("Registering L4 NetLB usage metrics %v", l4NetLBCount)
 	prometheus.MustRegister(l4NetLBCount)
 
+	klog.V(3).Infof("Registering L4 NetLB Dual Stack usage metrics %v", l4NetLBDualStackCount)
+	prometheus.MustRegister(l4NetLBDualStackCount)
+
 	klog.V(3).Infof("Registering PSC usage metrics %v", serviceAttachmentCount)
 	prometheus.MustRegister(serviceAttachmentCount)
 	prometheus.MustRegister(serviceCount)
@@ -167,6 +177,8 @@ type ControllerMetrics struct {
 	l4ILBDualStackServiceMap map[string]L4ILBDualStackServiceState
 	// l4NetLBServiceMap is a map between service key and L4 NetLB service state.
 	l4NetLBServiceMap map[string]L4NetLBServiceState
+	// l4NetLBDualStackServiceMap is a map between service key and L4 NetLB DualStack service state.
+	l4NetLBDualStackServiceMap map[string]L4NetLBDualStackServiceState
 	// pscMap is a map between the service attachment key and PSC state
 	pscMap map[string]pscmetrics.PSCState
 	// ServiceMap track the number of services in this cluster
@@ -182,15 +194,16 @@ type ControllerMetrics struct {
 // NewControllerMetrics initializes ControllerMetrics and starts a go routine to compute and export metrics periodically.
 func NewControllerMetrics(exportInterval, l4NetLBProvisionDeadline time.Duration) *ControllerMetrics {
 	return &ControllerMetrics{
-		ingressMap:               make(map[string]IngressState),
-		negMap:                   make(map[string]NegServiceState),
-		l4ILBServiceMap:          make(map[string]L4ILBServiceState),
-		l4ILBDualStackServiceMap: make(map[string]L4ILBDualStackServiceState),
-		l4NetLBServiceMap:        make(map[string]L4NetLBServiceState),
-		pscMap:                   make(map[string]pscmetrics.PSCState),
-		serviceMap:               make(map[string]struct{}),
-		metricsInterval:          exportInterval,
-		l4NetLBProvisionDeadline: l4NetLBProvisionDeadline,
+		ingressMap:                 make(map[string]IngressState),
+		negMap:                     make(map[string]NegServiceState),
+		l4ILBServiceMap:            make(map[string]L4ILBServiceState),
+		l4ILBDualStackServiceMap:   make(map[string]L4ILBDualStackServiceState),
+		l4NetLBServiceMap:          make(map[string]L4NetLBServiceState),
+		l4NetLBDualStackServiceMap: make(map[string]L4NetLBDualStackServiceState),
+		pscMap:                     make(map[string]pscmetrics.PSCState),
+		serviceMap:                 make(map[string]struct{}),
+		metricsInterval:            exportInterval,
+		l4NetLBProvisionDeadline:   l4NetLBProvisionDeadline,
 	}
 }
 
@@ -330,6 +343,25 @@ func (im *ControllerMetrics) DeleteL4NetLBService(svcKey string) {
 	delete(im.l4NetLBServiceMap, svcKey)
 }
 
+// SetL4NetLBDualStackService implements L4NetLBMetricsCollector.
+func (im *ControllerMetrics) SetL4NetLBDualStackService(svcKey string, state L4NetLBDualStackServiceState) {
+	im.Lock()
+	defer im.Unlock()
+
+	if im.l4NetLBDualStackServiceMap == nil {
+		klog.Fatalf("L4 NetLB DualStack Metrics failed to initialize correctly.")
+	}
+	im.l4NetLBDualStackServiceMap[svcKey] = state
+}
+
+// DeleteL4NetLBDualStackService implements L4NetLBMetricsCollector.
+func (im *ControllerMetrics) DeleteL4NetLBDualStackService(svcKey string) {
+	im.Lock()
+	defer im.Unlock()
+
+	delete(im.l4NetLBDualStackServiceMap, svcKey)
+}
+
 // SetServiceAttachment adds sa state to the map to be counted during metrics computation.
 // SetServiceAttachment implements PSCMetricsCollector.
 func (im *ControllerMetrics) SetServiceAttachment(saKey string, state pscmetrics.PSCState) {
@@ -415,6 +447,17 @@ func (im *ControllerMetrics) export() {
 
 	klog.V(3).Infof("L4 NetLB usage metrics exported.")
 
+	netlbDualStackCount := im.computeL4NetLBDualStackMetrics()
+	klog.V(3).Infof("Exporting L4 NetLB DualStack usage metrics: %#v", netlbDualStackCount)
+	for state, count := range netlbDualStackCount {
+		l4NetLBDualStackCount.With(prometheus.Labels{
+			"ip_families":      state.IPFamilies,
+			"ip_family_policy": state.IPFamilyPolicy,
+			"status":           string(state.Status),
+		}).Set(float64(count))
+	}
+	klog.V(3).Infof("L4 Netlb DualStack usage metrics exported.")
+
 	saCount := im.computePSCMetrics()
 	klog.V(3).Infof("Exporting PSC Usage Metrics: %#v", saCount)
 	for feature, count := range saCount {
@@ -616,6 +659,21 @@ func (im *ControllerMetrics) computeL4NetLBMetrics() netLBFeatureCount {
 	return counts
 }
 
+// computeL4NetLBDualStackMetrics aggregates L4 NetLB DualStack metrics in the cache.
+func (im *ControllerMetrics) computeL4NetLBDualStackMetrics() map[L4NetLBDualStackServiceState]int {
+	im.Lock()
+	defer im.Unlock()
+	klog.V(4).Infof("Computing L4 DualStack NetLB usage metrics from service state map: %#v", im.l4NetLBDualStackServiceMap)
+	counts := map[L4NetLBDualStackServiceState]int{}
+
+	for key, state := range im.l4NetLBDualStackServiceMap {
+		klog.V(6).Infof("NetLB Service %s has IPFamilies: %v, IPFamilyPolicy: %t, Status: %v", key, state.IPFamilies, state.IPFamilyPolicy, state.Status)
+		counts[state]++
+	}
+	klog.V(4).Info("L4 NetLB usage metrics computed.")
+	return counts
+}
+
 func (im *ControllerMetrics) computePSCMetrics() map[feature]int {
 	im.Lock()
 	defer im.Unlock()

diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go
@@ -73,11 +73,11 @@ type L4ILBServiceState struct {
 	IsUserError bool
 }
 
-type L4ILBDualStackServiceStateStatus string
+type L4DualStackServiceStateStatus string
 
-const StatusSuccess = L4ILBDualStackServiceStateStatus("Success")
-const StatusUserError = L4ILBDualStackServiceStateStatus("UserError")
-const StatusError = L4ILBDualStackServiceStateStatus("Error")
+const StatusSuccess = L4DualStackServiceStateStatus("Success")
+const StatusUserError = L4DualStackServiceStateStatus("UserError")
+const StatusError = L4DualStackServiceStateStatus("Error")
 
 // L4ILBDualStackServiceState defines ipFamilies, ipFamilyPolicy and status
 // of L4 ILB DualStack service
@@ -87,7 +87,18 @@ type L4ILBDualStackServiceState struct {
 	// IPFamilyPolicy specifies spec.IPFamilyPolicy of Service
 	IPFamilyPolicy string
 	// Status specifies status of L4 ILB DualStack
-	Status L4ILBDualStackServiceStateStatus
+	Status L4DualStackServiceStateStatus
+}
+
+// L4NetLBDualStackServiceState defines ipFamilies, ipFamilyPolicy and status
+// of L4 NetLB DualStack service
+type L4NetLBDualStackServiceState struct {
+	// IPFamilies stores spec.ipFamilies of Service
+	IPFamilies string
+	// IPFamilyPolicy specifies spec.IPFamilyPolicy of Service
+	IPFamilyPolicy string
+	// Status specifies status of L4 ILB DualStack
+	Status L4DualStackServiceStateStatus
 }
 
 // L4NetLBServiceState defines if network tier is premium and