HPA Observer Controller

cmd/craned/app/manager.go

@@ -212,7 +212,7 @@ func initializationControllers(ctx context.Context, mgr ctrl.Manager, opts *opti
 		Client: mgr.GetClient(),
 	}
 	if err := podOOMRecorder.SetupWithManager(mgr); err != nil {
-		klog.Exit(err, "unable to create controller", "controller", "PodOOMRecorder")
+		klog.Exit(err, "Unable to create controller", "PodOOMRecorder")
 	}
 	go func() {
 		if err := podOOMRecorder.Run(ctx.Done()); err != nil {
@@ -242,13 +242,13 @@ func initializationControllers(ctx context.Context, mgr ctrl.Manager, opts *opti
 			klog.Exit(err, "unable to create controller", "controller", "SubstituteController")
 		}
 
-		if err := (&ehpa.HPAReplicasController{
+		if err := (&ehpa.HPAObserverController{
 			Client:     mgr.GetClient(),
 			Scheme:     mgr.GetScheme(),
 			RestMapper: mgr.GetRESTMapper(),
 			Recorder:   mgr.GetEventRecorderFor("hpareplicas-controller"),
 		}).SetupWithManager(mgr); err != nil {
-			klog.Exit(err, "unable to create controller", "controller", "HPAReplicasController")
+			klog.Exit(err, "unable to create controller", "controller", "HPAObserverController")
 		}
 
 		if err := (&evpa.EffectiveVPAController{

pkg/controller/ehpa/effective_hpa_controller.go

@@ -184,7 +184,7 @@ func setCondition(status *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus,
 func RecordMetrics(ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) {
 	if ehpa.Status.ExpectReplicas != nil {
 		labels := map[string]string{
-			"identity": klog.KObj(ehpa).String(),
+			"resourceName": klog.KObj(ehpa).String(),
 			"strategy": string(ehpa.Spec.ScaleStrategy),
 		}
 		metrics.EHPAReplicas.With(labels).Set(float64(*ehpa.Status.ExpectReplicas))

pkg/controller/ehpa/hpa_event_handler.go

@@ -0,0 +1,64 @@
+package ehpa
+
+import (
+	"fmt"
+	"strings"
+
+	autoscalingv2 "k8s.io/api/autoscaling/v2beta2"
+	"k8s.io/client-go/util/workqueue"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+
+	"github.com/gocrane/crane/pkg/metrics"
+)
+
+type hpaEventHandler struct {
+	enqueueHandler handler.EnqueueRequestForObject
+}
+
+func (h *hpaEventHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
+	pod := evt.Object.(*autoscalingv2.HorizontalPodAutoscaler)
+	if pod.DeletionTimestamp != nil {
+		h.Delete(event.DeleteEvent{Object: evt.Object}, q)
+		return
+	}
+
+	h.enqueueHandler.Create(evt, q)
+}
+
+func (h *hpaEventHandler) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
+	h.enqueueHandler.Delete(evt, q)
+}
+
+func (h *hpaEventHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
+	newHpa := evt.ObjectNew.(*autoscalingv2.HorizontalPodAutoscaler)
+	oldHpa := evt.ObjectOld.(*autoscalingv2.HorizontalPodAutoscaler)
+	if oldHpa.Status.DesiredReplicas != newHpa.Status.DesiredReplicas {
+		for _, cond := range newHpa.Status.Conditions {
+			if cond.Reason == "SucceededRescale" || cond.Reason == "SucceededOverloadRescale" {
+				scaleType := "hpa"
+				if strings.HasPrefix("ehpa-", newHpa.Name) {
+					scaleType = "ehpa"
+				}
+
+				direction := "Down"
+				if newHpa.Status.DesiredReplicas > oldHpa.Status.DesiredReplicas {
+					direction = "Up"
+				}
+
+				labels := map[string]string{
+					"resourceName": fmt.Sprintf("%s/%s", newHpa.Namespace, newHpa.Name),
+					"type":         scaleType,
+					"direction":    direction,
+				}
+				metrics.HPAScaleCount.With(labels).Add(1)
+
+				break
+			}
+		}
+	}
+	h.enqueueHandler.Update(evt, q)
+}
+
+func (h *hpaEventHandler) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
+}

pkg/controller/ehpa/hpa_replicas_controller.go → pkg/controller/ehpa/hpa_observer_controller.go

@@ -10,19 +10,22 @@ import (
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/gocrane/crane/pkg/metrics"
 )
 
-// HPAReplicasController is responsible for monitor and export replicas for hpa
-type HPAReplicasController struct {
+// HPAObserverController is responsible for observer metrics for hpa
+type HPAObserverController struct {
 	client.Client
 	Scheme     *runtime.Scheme
 	RestMapper meta.RESTMapper
 	Recorder   record.EventRecorder
 }
 
-func (c *HPAReplicasController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (c *HPAObserverController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	klog.V(8).Infof("Got hpa %s", req.NamespacedName)
 
 	hpa := &autoscalingv2.HorizontalPodAutoscaler{}
@@ -31,15 +34,28 @@ func (c *HPAReplicasController) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	labels := map[string]string{
-		"identity": klog.KObj(hpa).String(),
+		"resourceName": klog.KObj(hpa).String(),
 	}
 	metrics.HPAReplicas.With(labels).Set(float64(hpa.Status.DesiredReplicas))
 
 	return ctrl.Result{}, nil
 }
 
-func (c *HPAReplicasController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&autoscalingv2.HorizontalPodAutoscaler{}).
-		Complete(c)
+func (c *HPAObserverController) SetupWithManager(mgr ctrl.Manager) error {
+	// Create a new controller
+	controller, err := controller.New("broadcastjob-controller", mgr, controller.Options{
+		Reconciler: c})
+	if err != nil {
+		return err
+	}
+
+	// Watch for changes to HPA
+	err = controller.Watch(&source.Kind{Type: &autoscalingv2.HorizontalPodAutoscaler{}}, &hpaEventHandler{
+		enqueueHandler: handler.EnqueueRequestForObject{},
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
 }

pkg/controller/evpa/effective_vpa_controller.go

@@ -124,7 +124,7 @@ func (c *EffectiveVPAController) SetupWithManager(mgr ctrl.Manager) error {
 
 func recordMetric(evpa *autoscalingapi.EffectiveVerticalPodAutoscaler, status *autoscalingapi.EffectiveVerticalPodAutoscalerStatus, podTemplate *v1.PodTemplateSpec) {
 	labels := map[string]string{
-		"target": fmt.Sprintf("%s/%s", evpa.Namespace, evpa.Spec.TargetRef.Name),
+		"resourceName": fmt.Sprintf("%s/%s", evpa.Namespace, evpa.Spec.TargetRef.Name),
 	}
 
 	for _, container := range status.Recommendation.ContainerRecommendations {

pkg/metrics/autoscaling.go

@@ -13,7 +13,7 @@ var (
 			Name:      "hpa_replicas",
 			Help:      "Replicas for HPA",
 		},
-		[]string{"identity"},
+		[]string{"resourceName"},
 	)
 	EHPAReplicas = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -22,7 +22,16 @@ var (
 			Name:      "effective_hpa_replicas",
 			Help:      "Replicas for Effective HPA",
 		},
-		[]string{"identity", "strategy"},
+		[]string{"resourceName", "strategy"},
+	)
+	HPAScaleCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "crane",
+			Subsystem: "autoscaling",
+			Name:      "hpa_scale_count",
+			Help:      "Scale count for HPA",
+		},
+		[]string{"resourceName", "type", "direction"},
 	)
 	OOMCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
@@ -44,7 +53,7 @@ var (
 			Help:      "The cpu scale up for Effective VPA",
 		},
 		[]string{
-			"target",
+			"resourceName",
 		},
 	)
 	EVPACpuScaleDownMilliCores = prometheus.NewGaugeVec(
@@ -55,7 +64,7 @@ var (
 			Help:      "The cpu scale down for Effective VPA",
 		},
 		[]string{
-			"target",
+			"resourceName",
 		},
 	)
 	EVPAMemoryScaleUpMB = prometheus.NewGaugeVec(
@@ -66,7 +75,7 @@ var (
 			Help:      "The memory scale up for Effective VPA",
 		},
 		[]string{
-			"target",
+			"resourceName",
 		},
 	)
 	EVPAMemoryScaleDownMB = prometheus.NewGaugeVec(
@@ -77,7 +86,7 @@ var (
 			Help:      "The memory scale down for Effective VPA",
 		},
 		[]string{
-			"target",
+			"resourceName",
 		},
 	)
 )