add examples for evicting all be pods when high load

gocrane · Aug 30, 2022 · 47cebe6 · 47cebe6
1 parent 0dea480
commit 47cebe6
Show file tree

Hide file tree

Showing 9 changed files with 69 additions and 70 deletions.
diff --git a/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml b/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml
@@ -0,0 +1,47 @@
+apiVersion: ensurance.crane.io/v1alpha1
+kind: PodQOS
+metadata:
+  name: all-be-pods
+spec:
+  allowedActions:
+    - eviction
+  resourceQOS:
+    cpuQOS:
+      cpuPriority: 7
+      htIsolation:
+        enable: false
+  scopeSelector:
+    matchExpressions:
+      - operator: In
+        scopeName: QOSClass
+        values:
+          - BestEffort
+---
+apiVersion: ensurance.crane.io/v1alpha1
+kind: NodeQOS
+metadata:
+  name: eviction-on-high-usage
+spec:
+  nodeQualityProbe:
+    nodeLocalGet:
+      localCacheTTLSeconds: 60
+    timeoutSeconds: 10
+  rules:
+    - actionName: eviction
+      avoidanceThreshold: 2
+      metricRule:
+        name: cpu_total_usage
+        value: 5000
+      name: cpu-usage
+      restoreThreshold: 2
+      strategy: None
+---
+apiVersion: ensurance.crane.io/v1alpha1
+kind: AvoidanceAction
+metadata:
+  name: eviction
+spec:
+  coolDownSeconds: 300
+  description: evict low priority pods
+  eviction:
+    terminationGracePeriodSeconds: 30
diff --git a/pkg/ensurance/analyzer/podqos-fetcher.go b/pkg/ensurance/analyzer/podqos-fetcher.go
@@ -4,13 +4,13 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-
-	ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1"
+
 	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/klog/v2"
+
+	ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1"
 )
 
 type ObjectIdentity struct {
@@ -21,56 +21,6 @@ type ObjectIdentity struct {
 	Labels     map[string]string
 }
 
-func objRefKey(kind, apiVersion, namespace, name string) string {
-	return fmt.Sprintf("%s#%s#%s#%s", kind, apiVersion, namespace, name)
-}
-
-func labelMatch(labelSelector metav1.LabelSelector, matchLabels map[string]string) bool {
-	for k, v := range labelSelector.MatchLabels {
-		if matchLabels[k] != v {
-			return false
-		}
-	}
-
-	for _, expr := range labelSelector.MatchExpressions {
-		switch expr.Operator {
-		case metav1.LabelSelectorOpExists:
-			if _, exists := matchLabels[expr.Key]; !exists {
-				return false
-			}
-		case metav1.LabelSelectorOpDoesNotExist:
-			if _, exists := matchLabels[expr.Key]; exists {
-				return false
-			}
-		case metav1.LabelSelectorOpIn:
-			if v, exists := matchLabels[expr.Key]; !exists {
-				return false
-			} else {
-				var found bool
-				for i := range expr.Values {
-					if expr.Values[i] == v {
-						found = true
-						break
-					}
-				}
-				if !found {
-					return false
-				}
-			}
-		case metav1.LabelSelectorOpNotIn:
-			if v, exists := matchLabels[expr.Key]; exists {
-				for i := range expr.Values {
-					if expr.Values[i] == v {
-						return false
-					}
-				}
-			}
-		}
-	}
-
-	return true
-}
-
 func match(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool {
 
 	if podQOS.Spec.ScopeSelector == nil &&

diff --git a/pkg/ensurance/collector/types/types.go b/pkg/ensurance/collector/types/types.go
@@ -68,6 +68,7 @@ func GetCgroupPath(p *v1.Pod, cgroupDriver string) string {
 		return ""
 	}
 }
+
 func GetCgroupName(p *v1.Pod) cm.CgroupName {
 	switch p.Status.QOSClass {
 	case v1.PodQOSGuaranteed:

diff --git a/pkg/ensurance/executor/evict.go b/pkg/ensurance/executor/evict.go
@@ -98,7 +98,7 @@ func (e *EvictExecutor) Avoid(ctx *ExecuteContext) error {
 				}
 				for !ctx.ToBeEvict.TargetGapsRemoved(m) {
 					klog.V(2).Infof("For metric %s, there is more gap to watermarks: %f of %s", m, ctx.ToBeEvict[m], m)
-					if podinfo.ContainsPendingPod(e.EvictPods) {
+					if podinfo.ContainsNoExecutedPod(e.EvictPods) {
 						index := podinfo.GetFirstPendingPod(e.EvictPods)
 						errKeys, released = metricMap[m].EvictFunc(&wg, ctx, index, &totalReleased, e.EvictPods)
 						errPodKeys = append(errPodKeys, errKeys...)

diff --git a/pkg/ensurance/executor/podinfo/pod_info.go b/pkg/ensurance/executor/podinfo/pod_info.go
@@ -94,7 +94,7 @@ type PodContext struct {
 	Executed    bool
 }
 
-func ContainsPendingPod(pods []PodContext) bool {
+func ContainsNoExecutedPod(pods []PodContext) bool {
 	for _, p := range pods {
 		if p.Executed == false {
 			return true

diff --git a/pkg/ensurance/executor/watermark.go b/pkg/ensurance/executor/watermark.go
@@ -1,9 +1,10 @@
 package executor
 
 import (
+	"math"
+
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/klog/v2"
-	"math"
 
 	"github.com/gocrane/crane/pkg/common"
 	"github.com/gocrane/crane/pkg/ensurance/collector/types"

diff --git a/pkg/utils/node.go b/pkg/utils/node.go
@@ -2,14 +2,14 @@ package utils
 
 import (
 	"fmt"
-	"k8s.io/klog/v2"
 
 	"golang.org/x/net/context"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/klog/v2"
 )
 
 const defaultRetryTimes = 3

diff --git a/pkg/webhooks/ensurance/validate_test.go b/pkg/webhooks/ensurance/validate_test.go
@@ -129,37 +129,37 @@ func TestValidateMetricRule(t *testing.T) {
 	}
 }
 
-func TestValidateObjectiveEnsurances(t *testing.T) {
+func TestValidateRules(t *testing.T) {
 	cases := map[string]struct {
-		objects       []ensuranceapi.ObjectiveEnsurance
+		objects       []ensuranceapi.Rule
 		httpGetEnable bool
 		expectErr     bool
 		errorType     field.ErrorType
 		errorDetail   string
 	}{
 		"object is required": {
-			objects:       []ensuranceapi.ObjectiveEnsurance{},
+			objects:       []ensuranceapi.Rule{},
 			httpGetEnable: false,
 			errorType:     field.ErrorTypeRequired,
 			errorDetail:   "",
 			expectErr:     true,
 		},
 		"object name is  required": {
-			objects:       []ensuranceapi.ObjectiveEnsurance{{Name: ""}},
+			objects:       []ensuranceapi.Rule{{Name: ""}},
 			httpGetEnable: false,
 			errorType:     field.ErrorTypeRequired,
 			errorDetail:   "",
 			expectErr:     true,
 		},
 		"object name is invalid": {
-			objects:       []ensuranceapi.ObjectiveEnsurance{{Name: "aaa.bbb"}},
+			objects:       []ensuranceapi.Rule{{Name: "aaa.bbb"}},
 			httpGetEnable: false,
 			errorType:     field.ErrorTypeInvalid,
 			errorDetail:   "",
 			expectErr:     true,
 		},
 		"object name is duplicate": {
-			objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa",
+			objects: []ensuranceapi.Rule{{Name: "aaa",
 				AvoidanceActionName: "eviction",
 				AvoidanceThreshold:  known.DefaultAvoidedThreshold,
 				RestoreThreshold:    known.DefaultRestoredThreshold,
@@ -173,29 +173,29 @@ func TestValidateObjectiveEnsurances(t *testing.T) {
 			expectErr:     true,
 		},
 		"object action name is  required": {
-			objects:       []ensuranceapi.ObjectiveEnsurance{{Name: "aaa"}},
+			objects:       []ensuranceapi.Rule{{Name: "aaa"}},
 			httpGetEnable: false,
 			errorType:     field.ErrorTypeRequired,
 			errorDetail:   "",
 			expectErr:     true,
 		},
 		"object action name is invalid": {
-			objects:       []ensuranceapi.ObjectiveEnsurance{{Name: "aaa", AvoidanceActionName: "eviction.aa"}},
+			objects:       []ensuranceapi.Rule{{Name: "aaa", AvoidanceActionName: "eviction.aa"}},
 			httpGetEnable: false,
 			errorType:     field.ErrorTypeInvalid,
 			errorDetail:   "",
 			expectErr:     true,
 		},
 		"object metric rule is required": {
-			objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa", AvoidanceActionName: "eviction",
+			objects: []ensuranceapi.Rule{{Name: "aaa", AvoidanceActionName: "eviction",
 				AvoidanceThreshold: known.DefaultAvoidedThreshold, RestoreThreshold: known.DefaultRestoredThreshold}},
 			httpGetEnable: false,
 			errorType:     field.ErrorTypeRequired,
 			errorDetail:   "",
 			expectErr:     true,
 		},
 		"object valid": {
-			objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa",
+			objects: []ensuranceapi.Rule{{Name: "aaa",
 				AvoidanceActionName: "eviction",
 				AvoidanceThreshold:  known.DefaultAvoidedThreshold,
 				RestoreThreshold:    known.DefaultRestoredThreshold,
@@ -209,7 +209,7 @@ func TestValidateObjectiveEnsurances(t *testing.T) {
 
 	for k, v := range cases {
 		t.Run(k, func(t *testing.T) {
-			errs := validateObjectiveEnsurances(v.objects, field.NewPath("objectiveEnsurances"), v.httpGetEnable)
+			errs := validateRules(v.objects, field.NewPath("objectiveEnsurances"), v.httpGetEnable)
 			t.Logf("%s: len %d", k, len(errs))
 			if v.expectErr && len(errs) > 0 {
 				if errs[0].Type != v.errorType || !strings.Contains(errs[0].Detail, v.errorDetail) {

diff --git a/pkg/webhooks/ensurance/validating.go b/pkg/webhooks/ensurance/validating.go
@@ -46,7 +46,7 @@ func (p *NodeQOSValidationAdmission) ValidateCreate(ctx context.Context, req run
 	if nodeQOS.Spec.NodeQualityProbe.HTTPGet != nil {
 		httpGetEnable = true
 	}
-	allErrs = append(allErrs, validateObjectiveEnsurances(nodeQOS.Spec.Rules, field.NewPath("objectiveEnsurances"), httpGetEnable)...)
+	allErrs = append(allErrs, validateRules(nodeQOS.Spec.Rules, field.NewPath("objectiveEnsurances"), httpGetEnable)...)
 
 	if len(allErrs) != 0 {
 		return allErrs.ToAggregate()
@@ -98,7 +98,7 @@ func validateHTTPGetAction(http *corev1.HTTPGetAction, fldPath *field.Path) fiel
 	return allErrs
 }
 
-func validateObjectiveEnsurances(objects []ensuranceapi.Rule, fldPath *field.Path, httpGetEnable bool) field.ErrorList {
+func validateRules(objects []ensuranceapi.Rule, fldPath *field.Path, httpGetEnable bool) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	if len(objects) == 0 {