improve L4FailedHealthCheckCount metric (do no report multiple times …

…the same case)
kubernetes · Feb 15, 2023 · 9a6e586 · 9a6e586
1 parent 41b2aa7
commit 9a6e586
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/pkg/l4lb/l4controller.go b/pkg/l4lb/l4controller.go
@@ -151,10 +151,14 @@ func (l4c *L4Controller) checkHealth() error {
 	// This indicates that the controller was stuck handling a previous update, or sync function did not get invoked.
 	syncTimeLatest := lastEnqueueTime.Add(enqueueToSyncDelayThreshold)
 	if lastSyncTime.After(syncTimeLatest) {
-		msg := fmt.Sprintf("L4 ILB Sync happened at time %v - %v after enqueue time, threshold is %v", lastSyncTime, lastSyncTime.Sub(lastEnqueueTime), enqueueToSyncDelayThreshold)
+		msg := fmt.Sprintf("L4 ILB Sync happened at time %v, last enqueue time %v- %v after enqueue time, threshold is %v", lastSyncTime, lastEnqueueTime, lastSyncTime.Sub(lastEnqueueTime), enqueueToSyncDelayThreshold)
 		// Log here, context/http handler do no log the error.
 		klog.Error(msg)
 		l4metrics.PublishL4FailedHealthCheckCount(l4ILBControllerName)
+		// Reset trackers. Otherwise, if there is nothing in the queue then it will report the FailedHealthCheckCount every time the checkHealth is called
+		// If checkHealth returned error (as it is meant to) then container would be restarted and trackers would be reset either
+		l4c.enqueueTracker.Track()
+		l4c.syncTracker.Track()
 	}
 	return nil
 }

diff --git a/pkg/l4lb/l4netlbcontroller.go b/pkg/l4lb/l4netlbcontroller.go
@@ -355,10 +355,14 @@ func (lc *L4NetLBController) checkHealth() error {
 	// This indicates that the controller was stuck handling a previous update, or sync function did not get invoked.
 	syncTimeLatest := lastEnqueueTime.Add(enqueueToSyncDelayThreshold)
 	if lastSyncTime.After(syncTimeLatest) {
-		msg := fmt.Sprintf("L4 External LoadBalancer Sync happened at time %v - %v after enqueue time, threshold is %v", lastSyncTime, lastSyncTime.Sub(lastEnqueueTime), enqueueToSyncDelayThreshold)
+		msg := fmt.Sprintf("L4 NetLB Sync happened at time %v, last enqueue time %v- %v after enqueue time, threshold is %v", lastSyncTime, lastEnqueueTime, lastSyncTime.Sub(lastEnqueueTime), enqueueToSyncDelayThreshold)
 		// Log here, context/http handler do no log the error.
 		klog.Error(msg)
 		metrics.PublishL4FailedHealthCheckCount(l4NetLBControllerName)
+		// Reset trackers. Otherwise, if there is nothing in the queue then it will report the FailedHealthCheckCount every time the checkHealth is called
+		// If checkHealth returned error (as it is meant to) then container would be restarted and trackers would be reset either
+		lc.enqueueTracker.Track()
+		lc.syncTracker.Track()
 	}
 	return nil
 }