From 47cebe676663eea7daccfa7a40d7038cb31c78d7 Mon Sep 17 00:00:00 2001 From: mfanjie Date: Mon, 22 Aug 2022 17:38:39 +0800 Subject: [PATCH] add examples for evicting all be pods when high load --- .../evict-on-cpu-usage-total/be-rules.yaml | 47 ++++++++++++++++ pkg/ensurance/analyzer/podqos-fetcher.go | 56 +------------------ pkg/ensurance/collector/types/types.go | 1 + pkg/ensurance/executor/evict.go | 2 +- pkg/ensurance/executor/podinfo/pod_info.go | 2 +- pkg/ensurance/executor/watermark.go | 3 +- pkg/utils/node.go | 2 +- pkg/webhooks/ensurance/validate_test.go | 22 ++++---- pkg/webhooks/ensurance/validating.go | 4 +- 9 files changed, 69 insertions(+), 70 deletions(-) create mode 100644 examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml diff --git a/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml b/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml new file mode 100644 index 000000000..a03f0f2e6 --- /dev/null +++ b/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml @@ -0,0 +1,47 @@ +apiVersion: ensurance.crane.io/v1alpha1 +kind: PodQOS +metadata: + name: all-be-pods +spec: + allowedActions: + - eviction + resourceQOS: + cpuQOS: + cpuPriority: 7 + htIsolation: + enable: false + scopeSelector: + matchExpressions: + - operator: In + scopeName: QOSClass + values: + - BestEffort +--- +apiVersion: ensurance.crane.io/v1alpha1 +kind: NodeQOS +metadata: + name: eviction-on-high-usage +spec: + nodeQualityProbe: + nodeLocalGet: + localCacheTTLSeconds: 60 + timeoutSeconds: 10 + rules: + - actionName: eviction + avoidanceThreshold: 2 + metricRule: + name: cpu_total_usage + value: 5000 + name: cpu-usage + restoreThreshold: 2 + strategy: None +--- +apiVersion: ensurance.crane.io/v1alpha1 +kind: AvoidanceAction +metadata: + name: eviction +spec: + coolDownSeconds: 300 + description: evict low priority pods + eviction: + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/pkg/ensurance/analyzer/podqos-fetcher.go b/pkg/ensurance/analyzer/podqos-fetcher.go index ea923ab9d..95fbbbc0d 100644 --- a/pkg/ensurance/analyzer/podqos-fetcher.go +++ b/pkg/ensurance/analyzer/podqos-fetcher.go @@ -4,13 +4,13 @@ import ( "fmt" "strconv" "strings" - - ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" + v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/klog/v2" + + ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" ) type ObjectIdentity struct { @@ -21,56 +21,6 @@ type ObjectIdentity struct { Labels map[string]string } -func objRefKey(kind, apiVersion, namespace, name string) string { - return fmt.Sprintf("%s#%s#%s#%s", kind, apiVersion, namespace, name) -} - -func labelMatch(labelSelector metav1.LabelSelector, matchLabels map[string]string) bool { - for k, v := range labelSelector.MatchLabels { - if matchLabels[k] != v { - return false - } - } - - for _, expr := range labelSelector.MatchExpressions { - switch expr.Operator { - case metav1.LabelSelectorOpExists: - if _, exists := matchLabels[expr.Key]; !exists { - return false - } - case metav1.LabelSelectorOpDoesNotExist: - if _, exists := matchLabels[expr.Key]; exists { - return false - } - case metav1.LabelSelectorOpIn: - if v, exists := matchLabels[expr.Key]; !exists { - return false - } else { - var found bool - for i := range expr.Values { - if expr.Values[i] == v { - found = true - break - } - } - if !found { - return false - } - } - case metav1.LabelSelectorOpNotIn: - if v, exists := matchLabels[expr.Key]; exists { - for i := range expr.Values { - if expr.Values[i] == v { - return false - } - } - } - } - } - - return true -} - func match(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool { if podQOS.Spec.ScopeSelector == nil && diff --git a/pkg/ensurance/collector/types/types.go b/pkg/ensurance/collector/types/types.go index 085c8f22e..a29cfdc39 100644 --- a/pkg/ensurance/collector/types/types.go +++ b/pkg/ensurance/collector/types/types.go @@ -68,6 +68,7 @@ func GetCgroupPath(p *v1.Pod, cgroupDriver string) string { return "" } } + func GetCgroupName(p *v1.Pod) cm.CgroupName { switch p.Status.QOSClass { case v1.PodQOSGuaranteed: diff --git a/pkg/ensurance/executor/evict.go b/pkg/ensurance/executor/evict.go index 436da8114..984bb2b76 100644 --- a/pkg/ensurance/executor/evict.go +++ b/pkg/ensurance/executor/evict.go @@ -98,7 +98,7 @@ func (e *EvictExecutor) Avoid(ctx *ExecuteContext) error { } for !ctx.ToBeEvict.TargetGapsRemoved(m) { klog.V(2).Infof("For metric %s, there is more gap to watermarks: %f of %s", m, ctx.ToBeEvict[m], m) - if podinfo.ContainsPendingPod(e.EvictPods) { + if podinfo.ContainsNoExecutedPod(e.EvictPods) { index := podinfo.GetFirstPendingPod(e.EvictPods) errKeys, released = metricMap[m].EvictFunc(&wg, ctx, index, &totalReleased, e.EvictPods) errPodKeys = append(errPodKeys, errKeys...) diff --git a/pkg/ensurance/executor/podinfo/pod_info.go b/pkg/ensurance/executor/podinfo/pod_info.go index 928b0d1c2..011e8fae0 100644 --- a/pkg/ensurance/executor/podinfo/pod_info.go +++ b/pkg/ensurance/executor/podinfo/pod_info.go @@ -94,7 +94,7 @@ type PodContext struct { Executed bool } -func ContainsPendingPod(pods []PodContext) bool { +func ContainsNoExecutedPod(pods []PodContext) bool { for _, p := range pods { if p.Executed == false { return true diff --git a/pkg/ensurance/executor/watermark.go b/pkg/ensurance/executor/watermark.go index a42008bfe..5ad7fe8a8 100644 --- a/pkg/ensurance/executor/watermark.go +++ b/pkg/ensurance/executor/watermark.go @@ -1,9 +1,10 @@ package executor import ( + "math" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/klog/v2" - "math" "github.com/gocrane/crane/pkg/common" "github.com/gocrane/crane/pkg/ensurance/collector/types" diff --git a/pkg/utils/node.go b/pkg/utils/node.go index 1a959a9b9..f2891e1ba 100644 --- a/pkg/utils/node.go +++ b/pkg/utils/node.go @@ -2,7 +2,6 @@ package utils import ( "fmt" - "k8s.io/klog/v2" "golang.org/x/net/context" v1 "k8s.io/api/core/v1" @@ -10,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" ) const defaultRetryTimes = 3 diff --git a/pkg/webhooks/ensurance/validate_test.go b/pkg/webhooks/ensurance/validate_test.go index 4a8c98091..31eaa6443 100644 --- a/pkg/webhooks/ensurance/validate_test.go +++ b/pkg/webhooks/ensurance/validate_test.go @@ -129,37 +129,37 @@ func TestValidateMetricRule(t *testing.T) { } } -func TestValidateObjectiveEnsurances(t *testing.T) { +func TestValidateRules(t *testing.T) { cases := map[string]struct { - objects []ensuranceapi.ObjectiveEnsurance + objects []ensuranceapi.Rule httpGetEnable bool expectErr bool errorType field.ErrorType errorDetail string }{ "object is required": { - objects: []ensuranceapi.ObjectiveEnsurance{}, + objects: []ensuranceapi.Rule{}, httpGetEnable: false, errorType: field.ErrorTypeRequired, errorDetail: "", expectErr: true, }, "object name is required": { - objects: []ensuranceapi.ObjectiveEnsurance{{Name: ""}}, + objects: []ensuranceapi.Rule{{Name: ""}}, httpGetEnable: false, errorType: field.ErrorTypeRequired, errorDetail: "", expectErr: true, }, "object name is invalid": { - objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa.bbb"}}, + objects: []ensuranceapi.Rule{{Name: "aaa.bbb"}}, httpGetEnable: false, errorType: field.ErrorTypeInvalid, errorDetail: "", expectErr: true, }, "object name is duplicate": { - objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa", + objects: []ensuranceapi.Rule{{Name: "aaa", AvoidanceActionName: "eviction", AvoidanceThreshold: known.DefaultAvoidedThreshold, RestoreThreshold: known.DefaultRestoredThreshold, @@ -173,21 +173,21 @@ func TestValidateObjectiveEnsurances(t *testing.T) { expectErr: true, }, "object action name is required": { - objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa"}}, + objects: []ensuranceapi.Rule{{Name: "aaa"}}, httpGetEnable: false, errorType: field.ErrorTypeRequired, errorDetail: "", expectErr: true, }, "object action name is invalid": { - objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa", AvoidanceActionName: "eviction.aa"}}, + objects: []ensuranceapi.Rule{{Name: "aaa", AvoidanceActionName: "eviction.aa"}}, httpGetEnable: false, errorType: field.ErrorTypeInvalid, errorDetail: "", expectErr: true, }, "object metric rule is required": { - objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa", AvoidanceActionName: "eviction", + objects: []ensuranceapi.Rule{{Name: "aaa", AvoidanceActionName: "eviction", AvoidanceThreshold: known.DefaultAvoidedThreshold, RestoreThreshold: known.DefaultRestoredThreshold}}, httpGetEnable: false, errorType: field.ErrorTypeRequired, @@ -195,7 +195,7 @@ func TestValidateObjectiveEnsurances(t *testing.T) { expectErr: true, }, "object valid": { - objects: []ensuranceapi.ObjectiveEnsurance{{Name: "aaa", + objects: []ensuranceapi.Rule{{Name: "aaa", AvoidanceActionName: "eviction", AvoidanceThreshold: known.DefaultAvoidedThreshold, RestoreThreshold: known.DefaultRestoredThreshold, @@ -209,7 +209,7 @@ func TestValidateObjectiveEnsurances(t *testing.T) { for k, v := range cases { t.Run(k, func(t *testing.T) { - errs := validateObjectiveEnsurances(v.objects, field.NewPath("objectiveEnsurances"), v.httpGetEnable) + errs := validateRules(v.objects, field.NewPath("objectiveEnsurances"), v.httpGetEnable) t.Logf("%s: len %d", k, len(errs)) if v.expectErr && len(errs) > 0 { if errs[0].Type != v.errorType || !strings.Contains(errs[0].Detail, v.errorDetail) { diff --git a/pkg/webhooks/ensurance/validating.go b/pkg/webhooks/ensurance/validating.go index 723a892fb..065e486e4 100644 --- a/pkg/webhooks/ensurance/validating.go +++ b/pkg/webhooks/ensurance/validating.go @@ -46,7 +46,7 @@ func (p *NodeQOSValidationAdmission) ValidateCreate(ctx context.Context, req run if nodeQOS.Spec.NodeQualityProbe.HTTPGet != nil { httpGetEnable = true } - allErrs = append(allErrs, validateObjectiveEnsurances(nodeQOS.Spec.Rules, field.NewPath("objectiveEnsurances"), httpGetEnable)...) + allErrs = append(allErrs, validateRules(nodeQOS.Spec.Rules, field.NewPath("objectiveEnsurances"), httpGetEnable)...) if len(allErrs) != 0 { return allErrs.ToAggregate() @@ -98,7 +98,7 @@ func validateHTTPGetAction(http *corev1.HTTPGetAction, fldPath *field.Path) fiel return allErrs } -func validateObjectiveEnsurances(objects []ensuranceapi.Rule, fldPath *field.Path, httpGetEnable bool) field.ErrorList { +func validateRules(objects []ensuranceapi.Rule, fldPath *field.Path, httpGetEnable bool) field.ErrorList { allErrs := field.ErrorList{} if len(objects) == 0 {