fix: add profile metric refresh

bytedance · Nov 11, 2024 · ff2b266 · ff2b266
1 parent 6f13fda
commit ff2b266
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 5 deletions.
diff --git a/internal/status/api/v1/manager.go b/internal/status/api/v1/manager.go
@@ -653,7 +653,7 @@ func (m *StatusManager) Run(stopCh <-chan struct{}) {
 	go m.reconcileStatus(stopCh)
 	go wait.Until(m.statusWorker, time.Second, stopCh)
 	go wait.Until(m.dataWorker, time.Second, stopCh)
-
+	go m.syncStatusMetricsLoop()
 	<-stopCh
 }
 

diff --git a/internal/status/api/v1/status.go b/internal/status/api/v1/status.go
@@ -86,11 +86,35 @@ func (m *StatusManager) HandleProfileStatusUpdate(status varmortypes.ProfileStat
 	}
 
 	m.profileChangeCount.Add(ctx, 1, metric.WithAttributeSet(attrSet))
+}
+func (m *StatusManager) syncStatusMetricsLoop() {
+	ctx := context.Background()
+	for {
+		time.Sleep(time.Second * 60)
+		logger := m.log.WithName("syncStatusMetricsLoop()")
+		logger.Info("start syncing status metrics")
+
+		for key, status := range m.PolicyStatuses {
+			namespace, name, err := PolicyStatusKeyGetInfo(key)
+			if err != nil {
+				logger.Error(err, "PolicyStatusKeyGetInfo()")
+				continue
+			}
+			for nodeName, nodeMessage := range status.NodeMessages {
+				labels := []attribute.KeyValue{
+					attribute.String("namespace", namespace),
+					attribute.String("profile_name", name),
+					attribute.String("node_name", nodeName),
+				}
+				attrSet := attribute.NewSet(labels...)
+				if nodeMessage == string(varmortypes.ArmorProfileReady) {
+					m.profileStatusPerNode.Record(ctx, 1, metric.WithAttributeSet(attrSet)) // 1 mean success
+				} else {
+					m.profileStatusPerNode.Record(ctx, 0, metric.WithAttributeSet(attrSet)) // 0 mean failure
+				}
+			}
 
-	if status.Status == varmortypes.Succeeded {
-		m.profileStatusPerNode.Record(ctx, 1, metric.WithAttributeSet(attrSet)) // 1 mean success
-	} else {
-		m.profileStatusPerNode.Record(ctx, 0, metric.WithAttributeSet(attrSet)) // 0 mean failure
+		}
 	}
 }
 

diff --git a/internal/status/api/v1/utils.go b/internal/status/api/v1/utils.go
@@ -59,6 +59,20 @@ func generatePolicyStatusKey(profileStatus *varmortypes.ProfileStatus) (string,
 		return "", fmt.Errorf("profileStatus.ProfileName is illegal")
 	}
 }
+func PolicyStatusKeyGetInfo(key string) (string, string, error) {
+
+	keyList := strings.Split(key, "/")
+	if len(keyList) == 2 {
+		profileNamePrefix := fmt.Sprintf(varmorprofile.ProfileNameTemplate, keyList[0], keyList[1])
+		return keyList[0], profileNamePrefix, nil
+	} else if len(keyList) == 1 {
+		clusterProfileNamePrefix := fmt.Sprintf(varmorprofile.ClusterProfileNameTemplate, "", keyList[0])
+		return "", clusterProfileNamePrefix, nil
+	} else {
+		return "", key, fmt.Errorf("PolicyStatusKey is illegal")
+	}
+
+}
 
 func generatePolicyStatusKeyWithArmorProfile(ap *varmor.ArmorProfile) (string, error) {
 	clusterProfileNamePrefix := fmt.Sprintf(varmorprofile.ClusterProfileNameTemplate, ap.Namespace, "")