idle node recommender

cmd/craned/app/manager.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -18,6 +19,7 @@ import (
 	"k8s.io/client-go/scale"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 
 	analysisapi "github.com/gocrane/api/analysis/v1alpha1"
@@ -49,6 +51,7 @@ import (
 	"github.com/gocrane/crane/pkg/recommendation/config"
 	recommender "github.com/gocrane/crane/pkg/recommendation/recommender"
 	"github.com/gocrane/crane/pkg/recommendation/recommender/hpa"
+	"github.com/gocrane/crane/pkg/recommendation/recommender/idlenode"
 	"github.com/gocrane/crane/pkg/recommendation/recommender/replicas"
 	"github.com/gocrane/crane/pkg/recommendation/recommender/resource"
 	"github.com/gocrane/crane/pkg/server"
@@ -125,6 +128,7 @@ func Run(ctx context.Context, opts *options.Options) error {
 	recommenderMgr := initRecommenderManager(recommenders, realtimeDataSources, historyDataSources)
 
 	initScheme()
+	initFieldIndexer(mgr)
 	initWebhooks(mgr, opts)
 	initControllers(ctx, mgr, opts, predictorMgr, recommenderMgr, historyDataSources[providers.PrometheusDataSource])
 	// initialize custom collector metrics
@@ -157,6 +161,12 @@ func initRecommenders(opts *options.Options) (map[string]recommender.Recommender
 			recommenders[recommender.HPARecommender] = hpaRecommender
 		case recommender.ResourceRecommender:
 			recommenders[recommender.ResourceRecommender] = resource.NewResourceRecommender(r)
+		case recommender.IdleNodeRecommender:
+			idleNodeRecommender, err := idlenode.NewIdleNodeRecommender(r)
+			if err != nil {
+				return nil, err
+			}
+			recommenders[recommender.IdleNodeRecommender] = idleNodeRecommender
 		default:
 			return nil, fmt.Errorf("unknown recommender name: %s", r.Name)
 		}
@@ -184,6 +194,23 @@ func initScheme() {
 	}
 }
 
+func initFieldIndexer(mgr ctrl.Manager) {
+	// register nodeName indexer
+	if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &corev1.Pod{}, "spec.nodeName", func(obj client.Object) []string {
+		pod, ok := obj.(*corev1.Pod)
+		if !ok {
+			return []string{}
+		}
+		if len(pod.Spec.NodeName) == 0 {
+			return []string{}
+		} else {
+			return []string{pod.Spec.NodeName}
+		}
+	}); err != nil {
+		panic(err)
+	}
+}
+
 func initMetricCollector(mgr ctrl.Manager) {
 	// register as prometheus metric collector
 	metrics.CustomCollectorRegister(metrics.NewTspMetricCollector(mgr.GetClient()))

examples/analytics/config_set.yaml

@@ -7,12 +7,5 @@ configs:
       replicas.workload-min-replicas: "3"
       replicas.pod-min-ready-seconds: "30"
       replicas.pod-available-ratio: "0.5"
-      replicas.default-min-replicas: "3"
-      replicas.max-replicas-factor: "3"
-      replicas.min-cpu-usage-threshold: "1"
-      replicas.fluctuation-threshold: "1.5"
-      replicas.min-cpu-target-utilization: "30"
-      replicas.max-cpu-target-utilization: "75"
-      replicas.cpu-target-utilization: "50"
-      replicas.cpu-percentile: "95"
-      replicas.reference-hpa: "true"
+      replicas.cpu-target-utilization: "0.5"
+      replicas.cpu-percentile: "95"

examples/analytics/recommendation-configuration.yaml

@@ -12,4 +12,8 @@ recommenders:
       - kind: Deployment
         apiVersion: apps/v1
       - kind: StatefulSet
-        apiVersion: apps/v1
+        apiVersion: apps/v1
+  - name: IdleNode
+    acceptedResources:
+      - kind: Node
+        apiVersion: v1

examples/analytics/workloads-rule.yaml

@@ -16,3 +16,21 @@ spec:
   recommenders:                               # 使用 Workload 的副本和资源推荐器
     - name: Replicas
     - name: Resource
+
+---
+
+apiVersion: analysis.crane.io/v1alpha1
+kind: RecommendationRule
+metadata:
+  name: nodes-rule
+  labels:
+    analysis.crane.io/recommendation-rule-preinstall: "true"
+spec:
+  runInterval: 24h                            # 每24h运行一次
+  resourceSelectors:                          # 资源的信息
+    - kind: Node
+      apiVersion: v1
+  namespaceSelector:
+    any: true                                 # 扫描所有namespace
+  recommenders:
+    - name: IdleNode

pkg/controller/recommendation/recommendation_rule_controller.go

@@ -239,10 +239,15 @@ func (c *RecommendationRuleController) doReconcile(ctx context.Context, recommen
 func (c *RecommendationRuleController) CreateRecommendationObject(recommendationRule *analysisv1alph1.RecommendationRule,
 	target corev1.ObjectReference, id ObjectIdentity, recommenderName string) *analysisv1alph1.Recommendation {
 
+	namespace := known.CraneSystemNamespace
+	if id.Namespace != "" {
+		namespace = id.Namespace
+	}
+
 	recommendation := &analysisv1alph1.Recommendation{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: fmt.Sprintf("%s-%s-", recommendationRule.Name, strings.ToLower(recommenderName)),
-			Namespace:    id.Namespace,
+			Namespace:    namespace,
 			OwnerReferences: []metav1.OwnerReference{
 				*newOwnerRef(recommendationRule),
 			},
@@ -260,6 +265,9 @@ func (c *RecommendationRuleController) CreateRecommendationObject(recommendation
 	recommendation.Labels[known.RecommendationRuleNameLabel] = recommendationRule.Name
 	recommendation.Labels[known.RecommendationRuleUidLabel] = string(recommendationRule.UID)
 	recommendation.Labels[known.RecommendationRuleRecommenderLabel] = recommenderName
+	recommendation.Labels[known.RecommendationRuleTargetKindLabel] = target.Kind
+	recommendation.Labels[known.RecommendationRuleTargetApiVersionLabel] = target.APIVersion
+	recommendation.Labels[known.RecommendationRuleTargetNameLabel] = target.Name
 
 	return recommendation
 }

pkg/known/label.go

@@ -17,7 +17,10 @@ const (
 )
 
 const (
-	RecommendationRuleNameLabel        = "analysis.crane.io/recommendation-rule-name"
-	RecommendationRuleUidLabel         = "analysis.crane.io/recommendation-rule-uid"
-	RecommendationRuleRecommenderLabel = "analysis.crane.io/recommendation-rule-recommender"
+	RecommendationRuleNameLabel             = "analysis.crane.io/recommendation-rule-name"
+	RecommendationRuleUidLabel              = "analysis.crane.io/recommendation-rule-uid"
+	RecommendationRuleRecommenderLabel      = "analysis.crane.io/recommendation-rule-recommender"
+	RecommendationRuleTargetKindLabel       = "analysis.crane.io/recommendation-target-kind"
+	RecommendationRuleTargetApiVersionLabel = "analysis.crane.io/recommendation-target-apiversion"
+	RecommendationRuleTargetNameLabel       = "analysis.crane.io/recommendation-target-name"
 )

pkg/recommendation/framework/context.go

@@ -142,11 +142,11 @@ func RetrieveScale(ctx *RecommendationContext) error {
 }
 
 func RetrievePods(ctx *RecommendationContext) error {
-	if ctx.Recommendation.Spec.TargetRef.Kind != "DaemonSet" {
-		pods, err := utils.GetPodsFromScale(ctx.Client, ctx.Scale)
+	if ctx.Recommendation.Spec.TargetRef.Kind == "Node" {
+		pods, err := utils.GetNodePods(ctx.Client, ctx.Recommendation.Spec.TargetRef.Name)
 		ctx.Pods = pods
 		return err
-	} else {
+	} else if ctx.Recommendation.Spec.TargetRef.Kind == "DaemonSet" {
 		var daemonSet appsv1.DaemonSet
 		err := ObjectConversion(ctx.Object, &daemonSet)
 		if err != nil {
@@ -155,6 +155,10 @@ func RetrievePods(ctx *RecommendationContext) error {
 		pods, err := utils.GetDaemonSetPods(ctx.Client, ctx.Recommendation.Spec.TargetRef.Namespace, ctx.Recommendation.Spec.TargetRef.Name)
 		ctx.Pods = pods
 		return err
+	} else {
+		pods, err := utils.GetPodsFromScale(ctx.Client, ctx.Scale)
+		ctx.Pods = pods
+		return err
 	}
 }
 

pkg/recommendation/recommender/base/filter.go

@@ -21,13 +21,18 @@ func (br *BaseRecommender) Filter(ctx *framework.RecommendationContext) error {
 	// 3. if not support, abort the recommendation flow
 	supported := IsIdentitySupported(identity, accepted)
 	if !supported {
-		return fmt.Errorf("recommender %s is failed at fliter, your kubernetes resource is not supported for recommender %s.", br.Name(), br.Name())
+		return fmt.Errorf("recommender %s is failed at filter, your kubernetes resource is not supported for recommender %s ", br.Name(), br.Name())
 	}
 
 	// 4. skip the objects that just created
-	creationCheckingTime := ctx.Object.GetCreationTimestamp().Add(CoolDownForCreation)
+	creationCheckingTime := ctx.Object.GetCreationTimestamp().Add(br.CreationCoolDown)
 	if time.Now().Before(creationCheckingTime) {
+		return fmt.Errorf("recommender %s is failed at filter, Creation Cool Down %s ", br.Name(), ctx.Object.GetCreationTimestamp())
+	}
 
+	// 5. skip the objects that deleting
+	if ctx.Object.GetDeletionTimestamp() != nil {
+		return fmt.Errorf("recommender %s is failed at filter, Is deleting ", br.Name())
 	}
 
 	return nil

pkg/recommendation/recommender/base/registry.go

@@ -9,15 +9,15 @@ import (
 
 var _ recommender.Recommender = &BaseRecommender{}
 
-const DefaultCreationCoolDown = time.Minute * 5
+const DefaultCreationCoolDown = time.Minute * 3
 
 type BaseRecommender struct {
 	apis.Recommender
 	CreationCoolDown time.Duration
 }
 
 func (br *BaseRecommender) Name() string {
-	return recommender.ReplicasRecommender
+	return ""
 }
 
 // NewBaseRecommender create a new base recommender.

pkg/recommendation/recommender/idlenode/filter.go

@@ -1,8 +1,6 @@
-package resource
+package idlenode
 
 import (
-	"fmt"
-
 	"github.com/gocrane/crane/pkg/recommendation/framework"
 )
 
@@ -15,27 +13,9 @@ func (inr *IdleNodeRecommender) Filter(ctx *framework.RecommendationContext) err
 		return err
 	}
 
-	if err = framework.RetrievePodTemplate(ctx); err != nil {
-		return err
-	}
-
-	if err = framework.RetrieveScale(ctx); err != nil {
-		return err
-	}
-
 	if err = framework.RetrievePods(ctx); err != nil {
 		return err
 	}
 
-	// filter workloads that are downing
-	if len(ctx.Pods) == 0 {
-		return fmt.Errorf("pod not found")
-	}
-
-	pod := ctx.Pods[0]
-	if len(pod.OwnerReferences) == 0 {
-		return fmt.Errorf("owner reference not found")
-	}
-
 	return nil
 }

pkg/recommendation/recommender/idlenode/observe.go

@@ -1,60 +1,10 @@
-package resource
+package idlenode
 
 import (
-	"encoding/json"
-	"fmt"
-
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-
-	"github.com/gocrane/crane/pkg/metrics"
 	"github.com/gocrane/crane/pkg/recommendation/framework"
 )
 
 // Observe enhance the observability.
 func (inr *IdleNodeRecommender) Observe(ctx *framework.RecommendationContext) error {
-	// get new PodTemplate
-	var newObject map[string]interface{}
-	if err := json.Unmarshal([]byte(ctx.Recommendation.Status.RecommendedInfo), &newObject); err != nil {
-		return err
-	}
-
-	podTemplateObject, found, err := unstructured.NestedMap(newObject, "spec", "template")
-	if !found || err != nil {
-		return fmt.Errorf("get template from unstructed object failed. ")
-	}
-
-	var newPodTemplate v1.PodTemplateSpec
-	err = framework.ObjectConversion(podTemplateObject, &newPodTemplate)
-	if err != nil {
-		return err
-	}
-
-	for _, container := range newPodTemplate.Spec.Containers {
-		inr.recordResourceRecommendation(ctx, container.Name, v1.ResourceCPU, container.Resources.Requests[v1.ResourceCPU])
-		inr.recordResourceRecommendation(ctx, container.Name, v1.ResourceMemory, container.Resources.Requests[v1.ResourceMemory])
-	}
-
 	return nil
 }
-
-func (inr *IdleNodeRecommender) recordResourceRecommendation(ctx *framework.RecommendationContext, containerName string, resName v1.ResourceName, quantity resource.Quantity) {
-	labels := map[string]string{
-		"apiversion": ctx.Recommendation.Spec.TargetRef.APIVersion,
-		"owner_kind": ctx.Recommendation.Spec.TargetRef.Kind,
-		"namespace":  ctx.Recommendation.Spec.TargetRef.Namespace,
-		"owner_name": ctx.Recommendation.Spec.TargetRef.Name,
-		"container":  containerName,
-		"resource":   resName.String(),
-	}
-
-	labels["owner_replicas"] = fmt.Sprintf("%d", len(ctx.Pods))
-
-	switch resName {
-	case v1.ResourceCPU:
-		metrics.ResourceRecommendation.With(labels).Set(float64(quantity.MilliValue()) / 1000.)
-	case v1.ResourceMemory:
-		metrics.ResourceRecommendation.With(labels).Set(float64(quantity.Value()))
-	}
-}

pkg/recommendation/recommender/idlenode/prepare.go

@@ -1,4 +1,4 @@
-package resource
+package idlenode
 
 import (
 	"github.com/gocrane/crane/pkg/recommendation/framework"