diff --git a/cmd/crane-agent/app/agent.go b/cmd/crane-agent/app/agent.go index cd48a79a0..a16fcce3e 100644 --- a/cmd/crane-agent/app/agent.go +++ b/cmd/crane-agent/app/agent.go @@ -100,12 +100,13 @@ func Run(ctx context.Context, opts *options.Options) error { nodeInformer := nodeInformerFactory.Core().V1().Nodes() craneInformerFactory := craneinformers.NewSharedInformerFactory(craneClient, informerSyncPeriod) - nepInformer := craneInformerFactory.Ensurance().V1alpha1().NodeQOSEnsurancePolicies() + nodeQOSInformer := craneInformerFactory.Ensurance().V1alpha1().NodeQOSs() + podQOSInformer := craneInformerFactory.Ensurance().V1alpha1().PodQOSs() actionInformer := craneInformerFactory.Ensurance().V1alpha1().AvoidanceActions() tspInformer := craneInformerFactory.Prediction().V1alpha1().TimeSeriesPredictions() newAgent, err := agent.NewAgent(ctx, hostname, opts.RuntimeEndpoint, opts.CgroupDriver, kubeClient, craneClient, podInformer, nodeInformer, - nepInformer, actionInformer, tspInformer, opts.NodeResourceReserved, opts.Ifaces, healthCheck, opts.CollectInterval, opts.ExecuteExcess) + nodeQOSInformer, podQOSInformer, actionInformer, tspInformer, opts.NodeResourceReserved, opts.Ifaces, healthCheck, opts.CollectInterval, opts.ExecuteExcess) if err != nil { return err diff --git a/cmd/crane-agent/app/options/option.go b/cmd/crane-agent/app/options/option.go index 830873bc1..4575b8ca4 100644 --- a/cmd/crane-agent/app/options/option.go +++ b/cmd/crane-agent/app/options/option.go @@ -26,7 +26,7 @@ type Options struct { // Ifaces is the network devices to collect metric Ifaces []string NodeResourceReserved map[string]string - // ExecuteExcess is the percentage of executions that exceed the gap between current usage and waterlines + // ExecuteExcess is the percentage of executions that exceed the gap between current usage and watermarks ExecuteExcess string } @@ -56,5 +56,5 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.StringArrayVar(&o.Ifaces, "ifaces", []string{"eth0"}, "The network devices to collect metric, use comma to separated, default: eth0") flags.Var(cliflag.NewMapStringString(&o.NodeResourceReserved), "node-resource-reserved", "A set of ResourceName=Percent (e.g. cpu=40%,memory=40%)") flags.DurationVar(&o.MaxInactivity, "max-inactivity", 5*time.Minute, "Maximum time from last recorded activity before automatic restart, default: 5min") - flags.StringVar(&o.ExecuteExcess, "execute-excess", "10%", "The percentage of executions that exceed the gap between current usage and waterlines, default: 10%.") + flags.StringVar(&o.ExecuteExcess, "execute-excess", "10%", "The percentage of executions that exceed the gap between current usage and watermarks, default: 10%.") } diff --git a/deploy/crane-agent/rbac.yaml b/deploy/crane-agent/rbac.yaml index b33f5e81a..7dbb33b06 100644 --- a/deploy/crane-agent/rbac.yaml +++ b/deploy/crane-agent/rbac.yaml @@ -6,73 +6,48 @@ rules: - apiGroups: - "" resources: - - pods/eviction - - pods/status - pods + - pods/status + - nodes + - nodes/status + - nodes/finalizers verbs: - - create - get - list - watch - update - patch - - delete - apiGroups: - "" resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events + - pods/eviction verbs: - - "*" + - create - apiGroups: - "" resources: - - nodes - - nodes/status - - nodes/finalizers - verbs: - - get - - list - - watch - - update - - patch - - apiGroups: - - "ensurance.crane.io" - resources: - - nodeqosensurancepolicies + - configmaps verbs: - get - list - watch - - update - - patch - apiGroups: - - "ensurance.crane.io" + - "" resources: - - podqosensurancepolicies + - events verbs: - - get - - list - - watch - - update - - patch + - "*" - apiGroups: - "ensurance.crane.io" resources: + - podqoss + - nodeqoss - avoidanceactions verbs: - get - list - watch - update - - patch - apiGroups: - "prediction.crane.io" resources: @@ -83,8 +58,8 @@ rules: - list - watch - create - - delete - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml b/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml new file mode 100644 index 000000000..a03f0f2e6 --- /dev/null +++ b/examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml @@ -0,0 +1,47 @@ +apiVersion: ensurance.crane.io/v1alpha1 +kind: PodQOS +metadata: + name: all-be-pods +spec: + allowedActions: + - eviction + resourceQOS: + cpuQOS: + cpuPriority: 7 + htIsolation: + enable: false + scopeSelector: + matchExpressions: + - operator: In + scopeName: QOSClass + values: + - BestEffort +--- +apiVersion: ensurance.crane.io/v1alpha1 +kind: NodeQOS +metadata: + name: eviction-on-high-usage +spec: + nodeQualityProbe: + nodeLocalGet: + localCacheTTLSeconds: 60 + timeoutSeconds: 10 + rules: + - actionName: eviction + avoidanceThreshold: 2 + metricRule: + name: cpu_total_usage + value: 5000 + name: cpu-usage + restoreThreshold: 2 + strategy: None +--- +apiVersion: ensurance.crane.io/v1alpha1 +kind: AvoidanceAction +metadata: + name: eviction +spec: + coolDownSeconds: 300 + description: evict low priority pods + eviction: + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/go.mod b/go.mod index deff5def1..876e1287b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/evanphx/json-patch v4.11.0+incompatible github.com/go-echarts/go-echarts/v2 v2.2.4 - github.com/gocrane/api v0.6.1-0.20220812033255-887f4b4e7d8b + github.com/gocrane/api v0.7.1-0.20220819080332-e4c0d60e812d github.com/google/cadvisor v0.39.2 github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12 github.com/prometheus/client_golang v1.11.0 diff --git a/go.sum b/go.sum index d363d840c..9e35c247e 100644 --- a/go.sum +++ b/go.sum @@ -310,10 +310,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.1.0-rc.5 h1:QOAag7FoBaBYYHRqzqkhhd8fq5RTubvI4v3Ft/gDVVQ= github.com/gobwas/ws v1.1.0-rc.5/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= -github.com/gocrane/api v0.6.1-0.20220809112454-68f0199a774e h1:pIocbZM7LchSMG7XBbfD9K+Im7zZtMZjVU7paVJOv6I= -github.com/gocrane/api v0.6.1-0.20220809112454-68f0199a774e/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= -github.com/gocrane/api v0.6.1-0.20220812033255-887f4b4e7d8b h1:ELyVltbne39izU2XaFrgJtqnhdeV+hBt+JBKooN7N4w= -github.com/gocrane/api v0.6.1-0.20220812033255-887f4b4e7d8b/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= +github.com/gocrane/api v0.7.1-0.20220819080332-e4c0d60e812d h1:qqPrNx1AETykgX80aWAmna/eQMDVWnUdSemWlfaZUNM= +github.com/gocrane/api v0.7.1-0.20220819080332-e4c0d60e812d/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 3af477e15..e52017482 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -58,7 +58,8 @@ func NewAgent(ctx context.Context, craneClient *craneclientset.Clientset, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer, - nepInformer v1alpha1.NodeQOSEnsurancePolicyInformer, + nodeQOSInformer v1alpha1.NodeQOSInformer, + podQOSInformer v1alpha1.PodQOSInformer, actionInformer v1alpha1.AvoidanceActionInformer, tspInformer predictionv1.TimeSeriesPredictionInformer, nodeResourceReserved map[string]string, @@ -85,9 +86,9 @@ func NewAgent(ctx context.Context, exclusiveCPUSet = cpuManager.GetExclusiveCpu managers = appendManagerIfNotNil(managers, cpuManager) } - stateCollector := collector.NewStateCollector(nodeName, nepInformer.Lister(), podInformer.Lister(), nodeInformer.Lister(), ifaces, healthCheck, CollectInterval, exclusiveCPUSet, cadvisorManager) + stateCollector := collector.NewStateCollector(nodeName, nodeQOSInformer.Lister(), podInformer.Lister(), nodeInformer.Lister(), ifaces, healthCheck, CollectInterval, exclusiveCPUSet, cadvisorManager) managers = appendManagerIfNotNil(managers, stateCollector) - analyzerManager := analyzer.NewAnormalyAnalyzer(kubeClient, nodeName, podInformer, nodeInformer, nepInformer, actionInformer, stateCollector.AnalyzerChann, noticeCh) + analyzerManager := analyzer.NewAnomalyAnalyzer(kubeClient, nodeName, podInformer, nodeInformer, nodeQOSInformer, podQOSInformer, actionInformer, stateCollector.AnalyzerChann, noticeCh) managers = appendManagerIfNotNil(managers, analyzerManager) avoidanceManager := executor.NewActionExecutor(kubeClient, nodeName, podInformer, nodeInformer, noticeCh, runtimeEndpoint, stateCollector.State, executeExcess) managers = appendManagerIfNotNil(managers, avoidanceManager) diff --git a/pkg/ensurance/analyzer/analyzer.go b/pkg/ensurance/analyzer/analyzer.go index 96727ae69..6a9c25a0a 100644 --- a/pkg/ensurance/analyzer/analyzer.go +++ b/pkg/ensurance/analyzer/analyzer.go @@ -25,13 +25,13 @@ import ( "github.com/gocrane/crane/pkg/ensurance/analyzer/evaluator" ecache "github.com/gocrane/crane/pkg/ensurance/cache" "github.com/gocrane/crane/pkg/ensurance/executor" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" "github.com/gocrane/crane/pkg/known" "github.com/gocrane/crane/pkg/metrics" "github.com/gocrane/crane/pkg/utils" ) -type AnormalyAnalyzer struct { +type AnomalyAnalyzer struct { nodeName string podLister corelisters.PodLister @@ -40,9 +40,12 @@ type AnormalyAnalyzer struct { nodeLister corelisters.NodeLister nodeSynced cache.InformerSynced - nodeQOSLister ensurancelisters.NodeQOSEnsurancePolicyLister + nodeQOSLister ensurancelisters.NodeQOSLister nodeQOSSynced cache.InformerSynced + podQOSLister ensurancelisters.PodQOSLister + podQOSSynced cache.InformerSynced + avoidanceActionLister ensurancelisters.AvoidanceActionLister avoidanceActionSynced cache.InformerSynced @@ -57,16 +60,17 @@ type AnormalyAnalyzer struct { lastTriggeredTime time.Time } -// NewAnormalyAnalyzer create an analyzer manager -func NewAnormalyAnalyzer(kubeClient *kubernetes.Clientset, +// NewAnomalyAnalyzer create an analyzer manager +func NewAnomalyAnalyzer(kubeClient *kubernetes.Clientset, nodeName string, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer, - nepInformer v1alpha1.NodeQOSEnsurancePolicyInformer, + nodeQOSInformer v1alpha1.NodeQOSInformer, + podQOSInformer v1alpha1.PodQOSInformer, actionInformer v1alpha1.AvoidanceActionInformer, stateChann chan map[string][]common.TimeSeries, noticeCh chan<- executor.AvoidanceExecutor, -) *AnormalyAnalyzer { +) *AnomalyAnalyzer { expressionEvaluator := evaluator.NewExpressionEvaluator() eventBroadcaster := record.NewBroadcaster() @@ -74,7 +78,7 @@ func NewAnormalyAnalyzer(kubeClient *kubernetes.Clientset, eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "crane-agent"}) - return &AnormalyAnalyzer{ + return &AnomalyAnalyzer{ nodeName: nodeName, evaluator: expressionEvaluator, actionCh: noticeCh, @@ -83,8 +87,10 @@ func NewAnormalyAnalyzer(kubeClient *kubernetes.Clientset, podSynced: podInformer.Informer().HasSynced, nodeLister: nodeInformer.Lister(), nodeSynced: nodeInformer.Informer().HasSynced, - nodeQOSLister: nepInformer.Lister(), - nodeQOSSynced: nepInformer.Informer().HasSynced, + nodeQOSLister: nodeQOSInformer.Lister(), + nodeQOSSynced: nodeQOSInformer.Informer().HasSynced, + podQOSLister: podQOSInformer.Lister(), + podQOSSynced: podQOSInformer.Informer().HasSynced, avoidanceActionLister: actionInformer.Lister(), avoidanceActionSynced: actionInformer.Informer().HasSynced, stateChann: stateChann, @@ -94,15 +100,15 @@ func NewAnormalyAnalyzer(kubeClient *kubernetes.Clientset, } } -func (s *AnormalyAnalyzer) Name() string { - return "AnormalyAnalyzer" +func (s *AnomalyAnalyzer) Name() string { + return "AnomalyAnalyzer" } -func (s *AnormalyAnalyzer) Run(stop <-chan struct{}) { - klog.Infof("Starting anormaly analyzer.") +func (s *AnomalyAnalyzer) Run(stop <-chan struct{}) { + klog.Infof("Starting anomaly analyzer.") // Wait for the caches to be synced before starting workers - if !cache.WaitForNamedCacheSync("anormaly-analyzer", + if !cache.WaitForNamedCacheSync("anomaly-analyzer", stop, s.podSynced, s.nodeSynced, @@ -117,11 +123,11 @@ func (s *AnormalyAnalyzer) Run(stop <-chan struct{}) { select { case state := <-s.stateChann: start := time.Now() - metrics.UpdateLastTime(string(known.ModuleAnormalyAnalyzer), metrics.StepMain, start) + metrics.UpdateLastTime(string(known.ModuleAnomalyAnalyzer), metrics.StepMain, start) s.Analyze(state) - metrics.UpdateDurationFromStart(string(known.ModuleAnormalyAnalyzer), metrics.StepMain, start) + metrics.UpdateDurationFromStart(string(known.ModuleAnomalyAnalyzer), metrics.StepMain, start) case <-stop: - klog.Infof("AnormalyAnalyzer exit") + klog.Infof("AnomalyAnalyzer exit") return } } @@ -130,28 +136,30 @@ func (s *AnormalyAnalyzer) Run(stop <-chan struct{}) { return } -func (s *AnormalyAnalyzer) Analyze(state map[string][]common.TimeSeries) { +func (s *AnomalyAnalyzer) Analyze(state map[string][]common.TimeSeries) { + klog.V(4).Infof("Starting anomaly analyze") node, err := s.nodeLister.Get(s.nodeName) if err != nil { klog.Errorf("Failed to get node: %v", err) return } - var neps []*ensuranceapi.NodeQOSEnsurancePolicy - allNeps, err := s.nodeQOSLister.List(labels.Everything()) + var nodeQOSs []*ensuranceapi.NodeQOS + allNodeQOSs, err := s.nodeQOSLister.List(labels.Everything()) if err != nil { klog.Errorf("Failed to list NodeQOS: %v", err) return } - for _, nep := range allNeps { - if matched, err := utils.LabelSelectorMatched(node.Labels, nep.Spec.Selector); err != nil || !matched { + for _, nodeQOS := range allNodeQOSs { + if matched, err := utils.LabelSelectorMatched(node.Labels, nodeQOS.Spec.Selector); err != nil || !matched { continue } - neps = append(neps, nep.DeepCopy()) + klog.V(6).Infof("NodeQOS %s matches the node", nodeQOS.Name) + nodeQOSs = append(nodeQOSs, nodeQOS.DeepCopy()) } - var avoidanceMaps = make(map[string]*ensuranceapi.AvoidanceAction) + var actionMap = make(map[string]*ensuranceapi.AvoidanceAction) allAvoidance, err := s.avoidanceActionLister.List(labels.Everything()) if err != nil { klog.Errorf("Failed to list AvoidanceActions, %v", err) @@ -159,33 +167,36 @@ func (s *AnormalyAnalyzer) Analyze(state map[string][]common.TimeSeries) { } for _, a := range allAvoidance { - avoidanceMaps[a.Name] = a + actionMap[a.Name] = a } - // step 2: do analyze for neps + // step 2: do analyze for nodeQOSs var actionContexts []ecache.ActionContext - for _, n := range neps { - for _, v := range n.Spec.ObjectiveEnsurances { - var key = strings.Join([]string{n.Name, v.Name}, ".") - ac, err := s.analyze(key, v, state) + for _, n := range nodeQOSs { + klog.V(6).Infof("Processing NodeQOS %s", n.Name) + + for _, r := range n.Spec.Rules { + var key = strings.Join([]string{n.Name, r.Name}, ".") + klog.V(6).Infof("Processing Rule %s", key) + actionContext, err := s.analyze(key, r, state) if err != nil { metrics.UpdateAnalyzerWithKeyStatus(metrics.AnalyzeTypeAnalyzeError, key, 1.0) klog.Errorf("Failed to analyze, %v.", err) } - metrics.UpdateAnalyzerWithKeyStatus(metrics.AnalyzeTypeAvoidance, key, float64(utils.Bool2Int32(ac.Triggered))) - metrics.UpdateAnalyzerWithKeyStatus(metrics.AnalyzeTypeRestore, key, float64(utils.Bool2Int32(ac.Restored))) + metrics.UpdateAnalyzerWithKeyStatus(metrics.AnalyzeTypeAvoidance, key, float64(utils.Bool2Int32(actionContext.Triggered))) + metrics.UpdateAnalyzerWithKeyStatus(metrics.AnalyzeTypeRestore, key, float64(utils.Bool2Int32(actionContext.Restored))) - ac.Nep = n - actionContexts = append(actionContexts, ac) + actionContext.NodeQOS = n + actionContexts = append(actionContexts, actionContext) } } klog.V(6).Infof("Analyze actionContexts: %#v", actionContexts) //step 3 : merge - avoidanceAction := s.merge(state, avoidanceMaps, actionContexts) + avoidanceAction := s.merge(state, actionMap, actionContexts) if err != nil { - klog.Errorf("Failed to merge, %v.", err) + klog.Errorf("Failed to merge actions, error: %v", err) return } @@ -195,7 +206,7 @@ func (s *AnormalyAnalyzer) Analyze(state map[string][]common.TimeSeries) { return } -func (s *AnormalyAnalyzer) getSeries(state []common.TimeSeries, selector *metav1.LabelSelector, metricName string) ([]common.TimeSeries, error) { +func (s *AnomalyAnalyzer) getSeries(state []common.TimeSeries, selector *metav1.LabelSelector, metricName string) ([]common.TimeSeries, error) { series := s.getTimeSeriesFromMap(state, selector) if len(series) == 0 { return []common.TimeSeries{}, fmt.Errorf("time series length is 0 for metric %s", metricName) @@ -203,126 +214,151 @@ func (s *AnormalyAnalyzer) getSeries(state []common.TimeSeries, selector *metav1 return series, nil } -func (s *AnormalyAnalyzer) trigger(series []common.TimeSeries, object ensuranceapi.ObjectiveEnsurance) bool { - var triggered, threshold bool +func (s *AnomalyAnalyzer) trigger(series []common.TimeSeries, object ensuranceapi.Rule) bool { + var aboveThreshold bool for _, ts := range series { - triggered = s.evaluator.EvalWithMetric(object.MetricRule.Name, float64(object.MetricRule.Value.Value()), ts.Samples[0].Value) + triggered := s.evaluator.EvalWithMetric(object.MetricRule.Name, float64(object.MetricRule.Value.Value()), ts.Samples[0].Value) - klog.V(4).Infof("Anormaly detection result %v, Name: %s, Value: %.2f, %s/%s", triggered, - object.MetricRule.Name, + klog.V(4).Infof("Evaluation result is %v, rule: %s, watermark: %f, current metrics: %f, metrics labels: %s/%s", + triggered, + object.Name+"."+object.MetricRule.Name, + float64(object.MetricRule.Value.Value()), ts.Samples[0].Value, common.GetValueByName(ts.Labels, common.LabelNamePodNamespace), common.GetValueByName(ts.Labels, common.LabelNamePodName)) if triggered { - threshold = true + klog.Warningf("Rule %s is triggered, watermark: %f, current metrics: %f, metrics labels: %s/%s", + object.Name+"."+object.MetricRule.Name, + float64(object.MetricRule.Value.Value()), + ts.Samples[0].Value, + common.GetValueByName(ts.Labels, common.LabelNamePodNamespace), + common.GetValueByName(ts.Labels, common.LabelNamePodName)) + aboveThreshold = true } } - return threshold + return aboveThreshold } -func (s *AnormalyAnalyzer) analyze(key string, object ensuranceapi.ObjectiveEnsurance, stateMap map[string][]common.TimeSeries) (ecache.ActionContext, error) { - var ac = ecache.ActionContext{Strategy: object.Strategy, ObjectiveEnsuranceName: object.Name, ActionName: object.AvoidanceActionName} +func (s *AnomalyAnalyzer) analyze(key string, rule ensuranceapi.Rule, stateMap map[string][]common.TimeSeries) (ecache.ActionContext, error) { + klog.V(4).Infof("Starting analyze") + var actionContext = ecache.ActionContext{Strategy: rule.Strategy, RuleName: rule.Name, ActionName: rule.AvoidanceActionName} - state, ok := stateMap[object.MetricRule.Name] + state, ok := stateMap[rule.MetricRule.Name] if !ok { - return ac, fmt.Errorf("metric %s not found", object.MetricRule.Name) + return actionContext, fmt.Errorf("metric %s not found", rule.MetricRule.Name) } //step1: get series from value - series, err := s.getSeries(state, object.MetricRule.Selector, object.MetricRule.Name) + series, err := s.getSeries(state, rule.MetricRule.Selector, rule.MetricRule.Name) if err != nil { - return ac, err + return actionContext, err } //step2: check if triggered for NodeQOSEnsurance - threshold := s.trigger(series, object) - - klog.V(4).Infof("For NodeQOS %s, metrics reach the threshold: %v", key, threshold) + aboveThreshold := s.trigger(series, rule) //step3: check is triggered action or restored, set the detection - s.computeActionContext(threshold, key, object, &ac) + s.computeActionContext(aboveThreshold, key, rule, &actionContext) - return ac, nil + return actionContext, nil } -func (s *AnormalyAnalyzer) computeActionContext(threshold bool, key string, object ensuranceapi.ObjectiveEnsurance, ac *ecache.ActionContext) { - if threshold { +func (s *AnomalyAnalyzer) computeActionContext(aboveThreshold bool, key string, object ensuranceapi.Rule, ac *ecache.ActionContext) { + if aboveThreshold { + // reset restore count s.restored[key] = 0 triggered := utils.GetUint64FromMaps(key, s.triggered) triggered++ s.triggered[key] = triggered if triggered >= uint64(object.AvoidanceThreshold) { + // log how many times the threshold is reached + s.triggered[key] = uint64(object.AvoidanceThreshold) + klog.Warningf("AvoidanceThreshold %d of %s is triggered, will take avoidance actions", object.AvoidanceThreshold, key) ac.Triggered = true } } else { s.triggered[key] = 0 restored := utils.GetUint64FromMaps(key, s.restored) restored++ + // log how many times the actual usage below threshold s.restored[key] = restored if restored >= uint64(object.RestoreThreshold) { + s.restored[key] = uint64(object.RestoreThreshold) + } + // only do restore action after trigger x times and restore y times + if s.triggered[key] >= uint64(object.AvoidanceThreshold) && + s.restored[key] >= uint64(object.RestoreThreshold) { + // reset trigger count when restored + s.triggered[key] = 0 + klog.Warningf("RestoreThreshold %d of %s is triggered, will take restore actions", object.RestoreThreshold, key) ac.Restored = true + } else { + // nothing happened + klog.V(2).Infof("The actual usage is below of watermark of rule %s, nothing happens", ac.RuleName) } } } -func (s *AnormalyAnalyzer) filterDryRun(acs []ecache.ActionContext) []ecache.ActionContext { +func (s *AnomalyAnalyzer) filterDryRun(actionContexts []ecache.ActionContext) []ecache.ActionContext { var dcsFiltered []ecache.ActionContext now := time.Now() - for _, ac := range acs { - s.logEvent(ac, now) - if !(ac.Strategy == ensuranceapi.AvoidanceActionStrategyPreview) { - dcsFiltered = append(dcsFiltered, ac) + for _, actionContext := range actionContexts { + s.logEvent(actionContext, now) + if !(actionContext.Strategy == ensuranceapi.AvoidanceActionStrategyPreview) { + dcsFiltered = append(dcsFiltered, actionContext) } } return dcsFiltered } -func (s *AnormalyAnalyzer) merge(stateMap map[string][]common.TimeSeries, avoidanceMaps map[string]*ensuranceapi.AvoidanceAction, acs []ecache.ActionContext) executor.AvoidanceExecutor { - var ae executor.AvoidanceExecutor +func (s *AnomalyAnalyzer) merge(stateMap map[string][]common.TimeSeries, actionMap map[string]*ensuranceapi.AvoidanceAction, actionContexts []ecache.ActionContext) executor.AvoidanceExecutor { + klog.V(6).Infof("Starting merge") + + var executor executor.AvoidanceExecutor //step1 filter dry run ActionContext - acsFiltered := s.filterDryRun(acs) + filteredActionContext := s.filterDryRun(actionContexts) //step2 do DisableScheduled merge - enableSchedule := s.disableSchedulingMerge(acsFiltered, avoidanceMaps, &ae) + s.mergeSchedulingActions(filteredActionContext, actionMap, &executor) - for _, ac := range acsFiltered { - action, ok := avoidanceMaps[ac.ActionName] + for _, context := range filteredActionContext { + action, ok := actionMap[context.ActionName] if !ok { - klog.Warningf("The action %s not found.", ac.ActionName) + klog.Warningf("Action %s is triggered, but the AvoidanceAction is not defined.", context.ActionName) continue } //step3 get and deduplicate throttlePods, throttleUpPods if action.Spec.Throttle != nil { - throttlePods, throttleUpPods := s.getThrottlePods(enableSchedule, ac, action, stateMap) + throttlePods, throttleUpPods := s.getThrottlePods(context, action, stateMap) - // combine the throttle waterline - combineThrottleWaterLine(&ae.ThrottleExecutor, ac, enableSchedule) + // combine the throttle watermark + combineThrottleWatermark(&executor.ThrottleExecutor, context) // combine the replicated pod - combineThrottleDuplicate(&ae.ThrottleExecutor, throttlePods, throttleUpPods) + combineThrottleDuplicate(&executor.ThrottleExecutor, throttlePods, throttleUpPods) } //step4 get and deduplicate evictPods if action.Spec.Eviction != nil { - evictPods := s.getEvictPods(ac.Triggered, action, stateMap) + evictPods := s.getEvictPods(context.Triggered, action, stateMap) - // combine the evict waterline - combineEvictWaterLine(&ae.EvictExecutor, ac) + // combine the evict watermark + combineEvictWatermark(&executor.EvictExecutor, context) // combine the replicated pod - combineEvictDuplicate(&ae.EvictExecutor, evictPods) + combineEvictDuplicate(&executor.EvictExecutor, evictPods) } } - ae.StateMap = stateMap + executor.StateMap = stateMap - klog.V(6).Infof("ThrottleExecutor is %#v, EvictExecutor is %#v", ae.ThrottleExecutor, ae.EvictExecutor) + klog.V(6).Infof("ThrottleExecutor is %#v, EvictExecutor is %#v", executor.ThrottleExecutor, executor.EvictExecutor) - return ae + return executor } -func (s *AnormalyAnalyzer) logEvent(ac ecache.ActionContext, now time.Time) { - var key = strings.Join([]string{ac.Nep.Name, ac.ObjectiveEnsuranceName}, "/") +func (s *AnomalyAnalyzer) logEvent(ac ecache.ActionContext, now time.Time) { + var key = strings.Join([]string{ac.NodeQOS.Name, ac.RuleName}, "/") if !(ac.Triggered || ac.Restored) { return @@ -352,7 +388,7 @@ func (s *AnormalyAnalyzer) logEvent(ac ecache.ActionContext, now time.Time) { return } -func (s *AnormalyAnalyzer) getTimeSeriesFromMap(state []common.TimeSeries, selector *metav1.LabelSelector) []common.TimeSeries { +func (s *AnomalyAnalyzer) getTimeSeriesFromMap(state []common.TimeSeries, selector *metav1.LabelSelector) []common.TimeSeries { var series []common.TimeSeries // step1: get the series from maps @@ -368,14 +404,14 @@ func (s *AnormalyAnalyzer) getTimeSeriesFromMap(state []common.TimeSeries, selec return series } -func (s *AnormalyAnalyzer) notify(as executor.AvoidanceExecutor) { +func (s *AnomalyAnalyzer) notify(as executor.AvoidanceExecutor) { //step1: notice by channel s.actionCh <- as return } -func (s *AnormalyAnalyzer) actionTriggered(ac ecache.ActionContext) bool { - var key = strings.Join([]string{ac.Nep.Name, ac.ObjectiveEnsuranceName}, "/") +func (s *AnomalyAnalyzer) actionTriggered(ac ecache.ActionContext) bool { + var key = strings.Join([]string{ac.NodeQOS.Name, ac.RuleName}, "/") if v, ok := s.actionEventStatus[key]; ok { if ac.Restored { @@ -388,12 +424,11 @@ func (s *AnormalyAnalyzer) actionTriggered(ac ecache.ActionContext) bool { return false } -func (s *AnormalyAnalyzer) getThrottlePods(enableSchedule bool, ac ecache.ActionContext, - action *ensuranceapi.AvoidanceAction, stateMap map[string][]common.TimeSeries) ([]podinfo.PodContext, []podinfo.PodContext) { +func (s *AnomalyAnalyzer) getThrottlePods(actionCtx ecache.ActionContext, action *ensuranceapi.AvoidanceAction, stateMap map[string][]common.TimeSeries) ([]podinfo.PodContext, []podinfo.PodContext) { throttlePods, throttleUpPods := []podinfo.PodContext{}, []podinfo.PodContext{} - if !ac.Triggered && !(enableSchedule && ac.Restored) { + if !actionCtx.Triggered && !actionCtx.Restored { return throttlePods, throttleUpPods } @@ -403,19 +438,24 @@ func (s *AnormalyAnalyzer) getThrottlePods(enableSchedule bool, ac ecache.Action return throttlePods, throttleUpPods } - for _, pod := range allPods { - if ac.Triggered { - throttlePods = append(throttlePods, podinfo.BuildPodBasicInfo(pod, stateMap, action, podinfo.ThrottleDown)) + filteredPods, err := s.filterPodQOSMatches(allPods, action.Name) + if err != nil { + klog.Errorf("Failed to filter all pods: %v.", err) + return throttlePods, throttleUpPods + } + for _, pod := range filteredPods { + if actionCtx.Triggered { + throttlePods = append(throttlePods, podinfo.BuildPodActionContext(pod, stateMap, action, podinfo.ThrottleDown)) } - if enableSchedule && ac.Restored { - throttleUpPods = append(throttleUpPods, podinfo.BuildPodBasicInfo(pod, stateMap, action, podinfo.ThrottleUp)) + if actionCtx.Restored { + throttleUpPods = append(throttleUpPods, podinfo.BuildPodActionContext(pod, stateMap, action, podinfo.ThrottleUp)) } } return throttlePods, throttleUpPods } -func (s *AnormalyAnalyzer) getEvictPods(triggered bool, action *ensuranceapi.AvoidanceAction, stateMap map[string][]common.TimeSeries) []podinfo.PodContext { +func (s *AnomalyAnalyzer) getEvictPods(triggered bool, action *ensuranceapi.AvoidanceAction, stateMap map[string][]common.TimeSeries) []podinfo.PodContext { evictPods := []podinfo.PodContext{} if triggered { @@ -424,69 +464,100 @@ func (s *AnormalyAnalyzer) getEvictPods(triggered bool, action *ensuranceapi.Avo klog.Errorf("Failed to list all pods: %v.", err) return evictPods } + filteredPods, err := s.filterPodQOSMatches(allPods, action.Name) + if err != nil { + klog.Errorf("Failed to filter all pods: %v.", err) + return evictPods + } + for _, pod := range filteredPods { + evictPods = append(evictPods, podinfo.BuildPodActionContext(pod, stateMap, action, podinfo.Evict)) - for _, pod := range allPods { - evictPods = append(evictPods, podinfo.BuildPodBasicInfo(pod, stateMap, action, podinfo.Evict)) } } return evictPods } -func (s *AnormalyAnalyzer) disableSchedulingMerge(acsFiltered []ecache.ActionContext, avoidanceMaps map[string]*ensuranceapi.AvoidanceAction, ae *executor.AvoidanceExecutor) bool { - var now = time.Now() +func (s *AnomalyAnalyzer) filterPodQOSMatches(pods []*v1.Pod, actionName string) ([]*v1.Pod, error) { + filteredPods := []*v1.Pod{} + podQOSList, err := s.podQOSLister.List(labels.Everything()) + // todo: not found error should be ignored + if err != nil { + klog.Errorf("Failed to list NodeQOS: %v", err) + return filteredPods, err + } + for _, qos := range podQOSList { + for _, pod := range pods { + if !match(pod, qos) { + klog.V(4).Infof("Pod %s/%s does not match PodQOS %s", pod.Namespace, pod.Name, qos.Name) + continue + + } + klog.V(4).Infof("Pod %s/%s matches PodQOS %s", pod.Namespace, pod.Name, qos.Name) + if !contains(qos.Spec.AllowedActions, actionName) { + klog.V(4).Infof("Action %s is not allowed for Pod %s/%s, PodQOS %s", actionName, pod.Namespace, pod.Name, qos.Name) + continue + } + filteredPods = append(filteredPods, pod) + } + } + return filteredPods, nil +} - // If any rules are triggered, the avoidance is true，otherwise the avoidance is false. - // If all rules are not triggered and some rules are restored, the restore is true，otherwise the restore is false. - // If the restore is true and the cool downtime reached, the enableScheduling is true，otherwise the enableScheduling is false. - var enableScheduling, avoidance, restore bool +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} - defer func() { - metrics.UpdateAnalyzerStatus(metrics.AnalyzeTypeEnableScheduling, float64(utils.Bool2Int32(enableScheduling))) - metrics.UpdateAnalyzerStatus(metrics.AnalyzeTypeAvoidance, float64(utils.Bool2Int32(avoidance))) - metrics.UpdateAnalyzerStatus(metrics.AnalyzeTypeRestore, float64(utils.Bool2Int32(restore))) - }() +func (s *AnomalyAnalyzer) mergeSchedulingActions(actionContexts []ecache.ActionContext, actionMap map[string]*ensuranceapi.AvoidanceAction, avoidanceExecutor *executor.AvoidanceExecutor) { + var now = time.Now() // If the ensurance rules are empty, it must be recovered soon. // So we set enableScheduling true - if len(acsFiltered) == 0 { - enableScheduling = true + if len(actionContexts) == 0 { + s.ToggleScheduleSetting(avoidanceExecutor, false) } else { - for _, ac := range acsFiltered { - action, ok := avoidanceMaps[ac.ActionName] - if !ok { - klog.Warningf("DoMerge for detection,but the action %s not found", ac.ActionName) - continue - } - + // If there is any actionContext is Triggered , or is Restored but in the CoolDownSeconds period, schedule should be disable; + // Otherwise, schedule shoule be able. + for _, ac := range actionContexts { if ac.Triggered { - avoidance = true - enableScheduling = false + metrics.UpdateAnalyzerStatus(metrics.AnalyzeTypeEnableScheduling, float64(0)) + s.ToggleScheduleSetting(avoidanceExecutor, true) + break } - if ac.Restored { - restore = true - if !avoidance && now.After(s.lastTriggeredTime.Add(time.Duration(action.Spec.CoolDownSeconds)*time.Second)) { - enableScheduling = true - } + action, ok := actionMap[ac.ActionName] + if !ok { + klog.Warningf("Action %s defined in nodeQOS %s is not found", ac.ActionName, ac.NodeQOS.Name) + continue + } + if ac.Restored && !now.After(s.lastTriggeredTime.Add(time.Duration(action.Spec.CoolDownSeconds)*time.Second)) { + metrics.UpdateAnalyzerStatus(metrics.AnalyzeTypeEnableScheduling, float64(0)) + s.ToggleScheduleSetting(avoidanceExecutor, true) + break } } - } - if avoidance { - s.lastTriggeredTime = now - ae.ScheduleExecutor.DisableClassAndPriority = &podinfo.ClassAndPriority{PodQOSClass: v1.PodQOSBestEffort, PriorityClassValue: 0} + metrics.UpdateAnalyzerStatus(metrics.AnalyzeTypeEnableScheduling, float64(1)) + s.ToggleScheduleSetting(avoidanceExecutor, false) } +} - if enableScheduling { - ae.ScheduleExecutor.RestoreClassAndPriority = &podinfo.ClassAndPriority{PodQOSClass: v1.PodQOSBestEffort, PriorityClassValue: 0} +func (s *AnomalyAnalyzer) ToggleScheduleSetting(ae *executor.AvoidanceExecutor, toBeDisable bool) { + if toBeDisable { + s.lastTriggeredTime = time.Now() } - - return enableScheduling + ae.ScheduleExecutor.ToBeDisable = toBeDisable + ae.ScheduleExecutor.ToBeRestore = !ae.ScheduleExecutor.ToBeDisable } +// todo to be refactered to deduplicate of two pod lists func combineThrottleDuplicate(e *executor.ThrottleExecutor, throttlePods, throttleUpPods executor.ThrottlePods) { for _, t := range throttlePods { - if i := e.ThrottleDownPods.Find(t.PodKey); i == -1 { + if i := e.ThrottleDownPods.Find(t.Key); i == -1 { e.ThrottleDownPods = append(e.ThrottleDownPods, t) } else { if t.CPUThrottle.MinCPURatio > e.ThrottleDownPods[i].CPUThrottle.MinCPURatio { @@ -500,7 +571,7 @@ func combineThrottleDuplicate(e *executor.ThrottleExecutor, throttlePods, thrott } for _, t := range throttleUpPods { - if i := e.ThrottleUpPods.Find(t.PodKey); i == -1 { + if i := e.ThrottleUpPods.Find(t.Key); i == -1 { e.ThrottleUpPods = append(e.ThrottleUpPods, t) } else { if t.CPUThrottle.MinCPURatio > e.ThrottleUpPods[i].CPUThrottle.MinCPURatio { @@ -516,7 +587,7 @@ func combineThrottleDuplicate(e *executor.ThrottleExecutor, throttlePods, thrott func combineEvictDuplicate(e *executor.EvictExecutor, evictPods executor.EvictPods) { for _, ep := range evictPods { - if i := e.EvictPods.Find(ep.PodKey); i == -1 { + if i := e.EvictPods.Find(ep.Key); i == -1 { e.EvictPods = append(e.EvictPods, ep) } else { if (ep.DeletionGracePeriodSeconds != nil) && ((e.EvictPods[i].DeletionGracePeriodSeconds == nil) || @@ -527,67 +598,67 @@ func combineEvictDuplicate(e *executor.EvictExecutor, evictPods executor.EvictPo } } -func combineThrottleWaterLine(e *executor.ThrottleExecutor, ac ecache.ActionContext, enableSchedule bool) { - if !ac.Triggered && !(enableSchedule && ac.Restored) { +func combineThrottleWatermark(e *executor.ThrottleExecutor, ac ecache.ActionContext) { + if !ac.Triggered && !ac.Restored { return } if ac.Triggered { - for _, ensurance := range ac.Nep.Spec.ObjectiveEnsurances { - if ensurance.Name == ac.ObjectiveEnsuranceName { - if e.ThrottleDownWaterLine == nil { - e.ThrottleDownWaterLine = make(map[executor.WaterLineMetric]*executor.WaterLine) + for _, ensurance := range ac.NodeQOS.Spec.Rules { + if ensurance.Name == ac.RuleName { + if e.ThrottleDownWatermark == nil { + e.ThrottleDownWatermark = make(map[executor.WatermarkMetric]*executor.Watermark) } - // Use a heap here, so we don't need to use - as value, just use - if e.ThrottleDownWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)] == nil { - e.ThrottleDownWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)] = &executor.WaterLine{} + // Use a heap here, so we don't need to use - as value, just use + if e.ThrottleDownWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)] == nil { + e.ThrottleDownWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)] = &executor.Watermark{} } - heap.Push(e.ThrottleDownWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)], ensurance.MetricRule.Value) + heap.Push(e.ThrottleDownWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)], ensurance.MetricRule.Value) } } } - for waterLineMetric, waterlines := range e.ThrottleDownWaterLine { - klog.V(6).Infof("ThrottleDownWaterLine info: metric: %s, value: %#v", waterLineMetric, waterlines) + for watermarkMetric, watermarks := range e.ThrottleDownWatermark { + klog.V(6).Infof("ThrottleDownWatermark info: metric: %s, value: %#v", watermarkMetric, watermarks) } - if enableSchedule && ac.Restored { - for _, ensurance := range ac.Nep.Spec.ObjectiveEnsurances { - if ensurance.Name == ac.ObjectiveEnsuranceName { - if e.ThrottleUpWaterLine == nil { - e.ThrottleUpWaterLine = make(map[executor.WaterLineMetric]*executor.WaterLine) + if ac.Restored { + for _, ensurance := range ac.NodeQOS.Spec.Rules { + if ensurance.Name == ac.RuleName { + if e.ThrottleUpWatermark == nil { + e.ThrottleUpWatermark = make(map[executor.WatermarkMetric]*executor.Watermark) } - if e.ThrottleUpWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)] == nil { - e.ThrottleUpWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)] = &executor.WaterLine{} + if e.ThrottleUpWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)] == nil { + e.ThrottleUpWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)] = &executor.Watermark{} } - heap.Push(e.ThrottleUpWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)], ensurance.MetricRule.Value) + heap.Push(e.ThrottleUpWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)], ensurance.MetricRule.Value) } } } - for waterLineMetric, waterlines := range e.ThrottleUpWaterLine { - klog.V(6).Infof("ThrottleUpWaterLine info: metric: %s, value: %#v", waterLineMetric, waterlines) + for watermarkMetric, watermarks := range e.ThrottleUpWatermark { + klog.V(6).Infof("ThrottleUpWatermark info: metric: %s, value: %#v", watermarkMetric, watermarks) } } -func combineEvictWaterLine(e *executor.EvictExecutor, ac ecache.ActionContext) { +func combineEvictWatermark(e *executor.EvictExecutor, ac ecache.ActionContext) { if !ac.Triggered { return } - for _, ensurance := range ac.Nep.Spec.ObjectiveEnsurances { - if ensurance.Name == ac.ObjectiveEnsuranceName { - if e.EvictWaterLine == nil { - e.EvictWaterLine = make(map[executor.WaterLineMetric]*executor.WaterLine) + for _, ensurance := range ac.NodeQOS.Spec.Rules { + if ensurance.Name == ac.RuleName { + if e.EvictWatermark == nil { + e.EvictWatermark = make(map[executor.WatermarkMetric]*executor.Watermark) } - if e.EvictWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)] == nil { - e.EvictWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)] = &executor.WaterLine{} + if e.EvictWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)] == nil { + e.EvictWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)] = &executor.Watermark{} } - heap.Push(e.EvictWaterLine[executor.WaterLineMetric(ensurance.MetricRule.Name)], ensurance.MetricRule.Value) + heap.Push(e.EvictWatermark[executor.WatermarkMetric(ensurance.MetricRule.Name)], ensurance.MetricRule.Value) } } - for waterLineMetric, waterlines := range e.EvictWaterLine { - klog.V(6).Infof("EvictWaterLine info: metric: %s, value: %#v", waterLineMetric, waterlines) + for watermarkMetric, watermarks := range e.EvictWatermark { + klog.V(6).Infof("EvictWatermark info: metric: %s, value: %#v", watermarkMetric, watermarks) } } diff --git a/pkg/ensurance/analyzer/podqos-fetcher.go b/pkg/ensurance/analyzer/podqos-fetcher.go new file mode 100644 index 000000000..22d47e4a7 --- /dev/null +++ b/pkg/ensurance/analyzer/podqos-fetcher.go @@ -0,0 +1,245 @@ +package analyzer + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/klog/v2" +) + +type ObjectIdentity struct { + Namespace string + APIVersion string + Kind string + Name string + Labels map[string]string +} + +func objRefKey(kind, apiVersion, namespace, name string) string { + return fmt.Sprintf("%s#%s#%s#%s", kind, apiVersion, namespace, name) +} + +func labelMatch(labelSelector metav1.LabelSelector, matchLabels map[string]string) bool { + for k, v := range labelSelector.MatchLabels { + if matchLabels[k] != v { + return false + } + } + + for _, expr := range labelSelector.MatchExpressions { + switch expr.Operator { + case metav1.LabelSelectorOpExists: + if _, exists := matchLabels[expr.Key]; !exists { + return false + } + case metav1.LabelSelectorOpDoesNotExist: + if _, exists := matchLabels[expr.Key]; exists { + return false + } + case metav1.LabelSelectorOpIn: + if v, exists := matchLabels[expr.Key]; !exists { + return false + } else { + var found bool + for i := range expr.Values { + if expr.Values[i] == v { + found = true + break + } + } + if !found { + return false + } + } + case metav1.LabelSelectorOpNotIn: + if v, exists := matchLabels[expr.Key]; exists { + for i := range expr.Values { + if expr.Values[i] == v { + return false + } + } + } + } + } + + return true +} + +func match(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool { + + if podQOS.Spec.ScopeSelector == nil && + podQOS.Spec.LabelSelector.MatchLabels == nil && + podQOS.Spec.LabelSelector.MatchExpressions == nil { + return false + } + + if !reflect.DeepEqual(podQOS.Spec.LabelSelector, metav1.LabelSelector{}) { + matchLabels := map[string]string{} + for k, v := range pod.Labels { + matchLabels[k] = v + } + if !labelMatch(podQOS.Spec.LabelSelector, matchLabels) { + return false + } + } + + if podQOS.Spec.ScopeSelector == nil { + return true + } + + // AND of the selectors + var nameSpaceSelectors, prioritySelectors, qosClassSelectors []ensuranceapi.ScopedResourceSelectorRequirement + for _, ss := range podQOS.Spec.ScopeSelector.MatchExpressions { + if ss.ScopeName == ensuranceapi.NamespaceSelectors { + nameSpaceSelectors = append(nameSpaceSelectors, ss) + } + if ss.ScopeName == ensuranceapi.PrioritySelectors { + prioritySelectors = append(prioritySelectors, ss) + } + if ss.ScopeName == ensuranceapi.QOSClassSelector { + qosClassSelectors = append(qosClassSelectors, ss) + } + } + + // namespace selector must be satisfied + for _, nss := range nameSpaceSelectors { + match, err := podMatchesNameSpaceSelector(pod, nss) + if err != nil { + klog.Errorf("Error on matching scope %s: %v", podQOS.Name, err) + return false + } + if !match { + klog.V(6).Infof("PodQOS %s namespace selector not match pod %s/%s", podQOS.Name, pod.Namespace, pod.Name) + return false + } + } + + var priorityTotalMatch = true + for _, selector := range prioritySelectors { + var priorityMatch bool + switch selector.Operator { + case v1.ScopeSelectorOpIn: + for _, vaules := range selector.Values { + priority := strings.Split(vaules, "-") + // In format of 1000 + if len(priority) == 1 { + p, err := strconv.Atoi(priority[0]) + if err == nil && int(*pod.Spec.Priority) == p { + priorityMatch = true + } + if err != nil { + klog.Errorf("%s can't transfer to int", priority[0]) + } + } + //In format of 1000-3000 + if len(priority) == 2 { + priStart, err1 := strconv.Atoi(priority[0]) + priEnd, err2 := strconv.Atoi(priority[1]) + if err1 == nil && err2 == nil && priEnd >= priStart && (int(*pod.Spec.Priority) <= priEnd) && (int(*pod.Spec.Priority) >= priStart) { + priorityMatch = true + } + } + } + case v1.ScopeSelectorOpNotIn: + for _, vaules := range selector.Values { + priority := strings.Split(vaules, "-") + // In format of 1000 + priorityMatch = true + if len(priority) == 1 { + p, err := strconv.Atoi(priority[0]) + if err == nil && int(*pod.Spec.Priority) == p { + priorityMatch = false + } + if err != nil { + klog.Errorf("%s can't transfer to int", priority[0]) + } + } + //In format of 1000-3000 + if len(priority) == 2 { + priStart, err1 := strconv.Atoi(priority[0]) + priEnd, err2 := strconv.Atoi(priority[1]) + if err1 == nil && err2 == nil && priEnd >= priStart && (int(*pod.Spec.Priority) <= priEnd) && (int(*pod.Spec.Priority) >= priStart) { + priorityMatch = false + } + } + } + } + priorityTotalMatch = priorityTotalMatch && priorityMatch + if priorityMatch == false { + break + } + } + if !priorityTotalMatch { + return false + } + + var qosClassMatch = true + for _, qos := range qosClassSelectors { + match, err := podMatchesqosClassSelector(pod, qos) + if err != nil { + klog.Errorf("Error on matching scope %s: %v", podQOS.Name, err) + qosClassMatch = false + } + if !match { + klog.V(6).Infof("PodQOS %s qosclass selector not match pod %s/%s", podQOS.Name, pod.Namespace, pod.Name) + qosClassMatch = false + } + } + if !qosClassMatch { + return false + } + + return true +} + +func podMatchesNameSpaceSelector(pod *v1.Pod, selector ensuranceapi.ScopedResourceSelectorRequirement) (bool, error) { + labelSelector, err := scopedResourceSelectorRequirementsAsSelector(selector) + if err != nil { + return false, fmt.Errorf("failed to parse and convert selector: %v", err) + } + m := map[string]string{string(selector.ScopeName): pod.Namespace} + if labelSelector.Matches(labels.Set(m)) { + return true, nil + } + return false, nil +} + +// scopedResourceSelectorRequirementsAsSelector converts the ScopedResourceSelectorRequirement api type into a struct that implements +// labels.Selector. +func scopedResourceSelectorRequirementsAsSelector(nss ensuranceapi.ScopedResourceSelectorRequirement) (labels.Selector, error) { + selector := labels.NewSelector() + var op selection.Operator + switch nss.Operator { + case v1.ScopeSelectorOpIn: + op = selection.In + case v1.ScopeSelectorOpNotIn: + op = selection.NotIn + default: + return nil, fmt.Errorf("%q is not a valid scope selector operator", nss.Operator) + } + r, err := labels.NewRequirement(string(nss.ScopeName), op, nss.Values) + if err != nil { + return nil, err + } + selector = selector.Add(*r) + return selector, nil +} + +func podMatchesqosClassSelector(pod *v1.Pod, selector ensuranceapi.ScopedResourceSelectorRequirement) (bool, error) { + labelSelector, err := scopedResourceSelectorRequirementsAsSelector(selector) + if err != nil { + return false, fmt.Errorf("failed to parse and convert selector: %v", err) + } + m := map[string]string{string(selector.ScopeName): string(pod.Status.QOSClass)} + if labelSelector.Matches(labels.Set(m)) { + return true, nil + } + return false, nil +} diff --git a/pkg/ensurance/cache/detection.go b/pkg/ensurance/cache/detection.go index aee1b71bf..bc401e552 100644 --- a/pkg/ensurance/cache/detection.go +++ b/pkg/ensurance/cache/detection.go @@ -5,14 +5,12 @@ import ( "sync" "time" - "k8s.io/apimachinery/pkg/types" - ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" ) type ActionContext struct { // the Objective Ensurance name - ObjectiveEnsuranceName string + RuleName string // strategy for the action Strategy ensuranceapi.AvoidanceActionStrategy // if the policy triggered action @@ -22,12 +20,9 @@ type ActionContext struct { // action name ActionName string // node qos ensurance policy - Nep *ensuranceapi.NodeQOSEnsurancePolicy + NodeQOS *ensuranceapi.NodeQOS // time for detection Time time.Time - // the influenced pod list - // node detection the pod list is empty - BeInfluencedPods []types.NamespacedName } type ActionContextCache struct { @@ -85,7 +80,7 @@ func (s *ActionContextCache) ListDetections() []ActionContext { } func GenerateDetectionKey(c ActionContext) string { - return strings.Join([]string{"node", c.Nep.Name, c.ObjectiveEnsuranceName}, ".") + return strings.Join([]string{"node", c.NodeQOS.Name, c.RuleName}, ".") } type DetectionStatus struct { diff --git a/pkg/ensurance/cache/nep.go b/pkg/ensurance/cache/nep.go deleted file mode 100644 index 8e27d8bc0..000000000 --- a/pkg/ensurance/cache/nep.go +++ /dev/null @@ -1,64 +0,0 @@ -package cache - -import ( - "sync" - - ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" -) - -type NodeQOSEnsurancePolicyCache struct { - mu sync.Mutex - nepMap map[string]*ensuranceapi.NodeQOSEnsurancePolicy -} - -func (s *NodeQOSEnsurancePolicyCache) Init() { - s.nepMap = make(map[string]*ensuranceapi.NodeQOSEnsurancePolicy) -} - -// ListKeys implements the interface required by DeltaFIFO to list the keys we -// already know about. -func (s *NodeQOSEnsurancePolicyCache) ListKeys() []string { - s.mu.Lock() - defer s.mu.Unlock() - keys := make([]string, 0, len(s.nepMap)) - for k := range s.nepMap { - keys = append(keys, k) - } - return keys -} - -func (s *NodeQOSEnsurancePolicyCache) Get(name string) (*ensuranceapi.NodeQOSEnsurancePolicy, bool) { - s.mu.Lock() - defer s.mu.Unlock() - nep, ok := s.nepMap[name] - return nep, ok -} - -func (s *NodeQOSEnsurancePolicyCache) Exist(name string) bool { - s.mu.Lock() - defer s.mu.Unlock() - _, ok := s.nepMap[name] - return ok -} - -func (s *NodeQOSEnsurancePolicyCache) GetOrCreate(nep *ensuranceapi.NodeQOSEnsurancePolicy) *ensuranceapi.NodeQOSEnsurancePolicy { - s.mu.Lock() - defer s.mu.Unlock() - cacheNep, ok := s.nepMap[nep.Name] - if !ok { - s.nepMap[nep.Name] = nep - } - return cacheNep -} - -func (s *NodeQOSEnsurancePolicyCache) Set(nep *ensuranceapi.NodeQOSEnsurancePolicy) { - s.mu.Lock() - defer s.mu.Unlock() - s.nepMap[nep.Name] = nep -} - -func (s *NodeQOSEnsurancePolicyCache) Delete(name string) { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.nepMap, name) -} diff --git a/pkg/ensurance/cache/nodeqos.go b/pkg/ensurance/cache/nodeqos.go new file mode 100644 index 000000000..ace1c1ead --- /dev/null +++ b/pkg/ensurance/cache/nodeqos.go @@ -0,0 +1,64 @@ +package cache + +import ( + "sync" + + ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" +) + +type NodeQOSCache struct { + mu sync.Mutex + nodeQOSMap map[string]*ensuranceapi.NodeQOS +} + +func (s *NodeQOSCache) Init() { + s.nodeQOSMap = make(map[string]*ensuranceapi.NodeQOS) +} + +// ListKeys implements the interface required by DeltaFIFO to list the keys we +// already know about. +func (s *NodeQOSCache) ListKeys() []string { + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]string, 0, len(s.nodeQOSMap)) + for k := range s.nodeQOSMap { + keys = append(keys, k) + } + return keys +} + +func (s *NodeQOSCache) Get(name string) (*ensuranceapi.NodeQOS, bool) { + s.mu.Lock() + defer s.mu.Unlock() + nodeQOS, ok := s.nodeQOSMap[name] + return nodeQOS, ok +} + +func (s *NodeQOSCache) Exist(name string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.nodeQOSMap[name] + return ok +} + +func (s *NodeQOSCache) GetOrCreate(nodeQOS *ensuranceapi.NodeQOS) *ensuranceapi.NodeQOS { + s.mu.Lock() + defer s.mu.Unlock() + cache, ok := s.nodeQOSMap[nodeQOS.Name] + if !ok { + s.nodeQOSMap[nodeQOS.Name] = nodeQOS + } + return cache +} + +func (s *NodeQOSCache) Set(nodeQOS *ensuranceapi.NodeQOS) { + s.mu.Lock() + defer s.mu.Unlock() + s.nodeQOSMap[nodeQOS.Name] = nodeQOS +} + +func (s *NodeQOSCache) Delete(name string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.nodeQOSMap, name) +} diff --git a/pkg/ensurance/collector/collector.go b/pkg/ensurance/collector/collector.go index 754bd8347..131bf4ce5 100644 --- a/pkg/ensurance/collector/collector.go +++ b/pkg/ensurance/collector/collector.go @@ -24,7 +24,7 @@ import ( type StateCollector struct { nodeName string - nepLister ensuranceListers.NodeQOSEnsurancePolicyLister + nodeQOSLister ensuranceListers.NodeQOSLister podLister corelisters.PodLister nodeLister corelisters.NodeLister healthCheck *metrics.HealthCheck @@ -40,7 +40,7 @@ type StateCollector struct { rw sync.RWMutex } -func NewStateCollector(nodeName string, nepLister ensuranceListers.NodeQOSEnsurancePolicyLister, podLister corelisters.PodLister, +func NewStateCollector(nodeName string, nodeQOSLister ensuranceListers.NodeQOSLister, podLister corelisters.PodLister, nodeLister corelisters.NodeLister, ifaces []string, healthCheck *metrics.HealthCheck, collectInterval time.Duration, exclusiveCPUSet func() cpuset.CPUSet, manager cadvisor.Manager) *StateCollector { analyzerChann := make(chan map[string][]common.TimeSeries) nodeResourceChann := make(chan map[string][]common.TimeSeries) @@ -48,7 +48,7 @@ func NewStateCollector(nodeName string, nepLister ensuranceListers.NodeQOSEnsura State := make(map[string][]common.TimeSeries) return &StateCollector{ nodeName: nodeName, - nepLister: nepLister, + nodeQOSLister: nodeQOSLister, podLister: podLister, nodeLister: nodeLister, healthCheck: healthCheck, @@ -69,6 +69,7 @@ func (s *StateCollector) Name() string { } func (s *StateCollector) Run(stop <-chan struct{}) { + klog.Infof("Starting state collector.") s.UpdateCollectors() go func() { updateTicker := time.NewTicker(s.collectInterval) @@ -151,9 +152,9 @@ func (s *StateCollector) Collect() { } func (s *StateCollector) UpdateCollectors() { - allNeps, err := s.nepLister.List(labels.Everything()) + allNodeQOSs, err := s.nodeQOSLister.List(labels.Everything()) if err != nil { - klog.Warningf("Failed to list NodeQOSEnsurancePolicy, err %v", err) + klog.Warningf("Failed to list NodeQOS, err %v", err) } node, err := s.nodeLister.Get(s.nodeName) if err != nil { @@ -161,13 +162,13 @@ func (s *StateCollector) UpdateCollectors() { return } var nodeLocal bool - for _, n := range allNeps { + for _, n := range allNodeQOSs { if matched, err := utils.LabelSelectorMatched(node.Labels, n.Spec.Selector); err != nil || !matched { continue } if n.Spec.NodeQualityProbe.NodeLocalGet == nil { - klog.V(4).Infof("Probe type of NEP %s/%s is not node local, continue", n.Namespace, n.Name) + klog.V(4).Infof("Probe type of NodeQOS %s/%s is not node local, continue", n.Namespace, n.Name) continue } @@ -184,7 +185,7 @@ func (s *StateCollector) UpdateCollectors() { break } - // if node resource controller is enabled, it indicates local metrics need to be collected no matter nep is defined or not + // if node resource controller is enabled, it indicates local metrics need to be collected no matter nodeqos is defined or not if nodeResourceGate := utilfeature.DefaultFeatureGate.Enabled(features.CraneNodeResource); nodeResourceGate { if _, exists := s.collectors.Load(types.NodeLocalCollectorType); !exists { nc := nodelocal.NewNodeLocal(s.ifaces, s.exclusiveCPUSet) diff --git a/pkg/ensurance/collector/noderesource/noderesource.go b/pkg/ensurance/collector/noderesource/noderesource.go index 1f508dce8..14b318b5e 100644 --- a/pkg/ensurance/collector/noderesource/noderesource.go +++ b/pkg/ensurance/collector/noderesource/noderesource.go @@ -54,7 +54,7 @@ func (n *NodeResource) Collect() (map[string][]common.TimeSeries, error) { } } } - klog.V(4).Infof("allExtCpu: %d, distributeExtCpu: %d", allExtCpu, distributeExtCpu) + klog.V(4).Infof("Allocatable Elastic CPU: %d, allocated Elastic CPU: %d", allExtCpu, distributeExtCpu) return map[string][]common.TimeSeries{string(types.MetricNameExtCpuTotalDistribute): {{Samples: []common.Sample{{Value: (float64(distributeExtCpu) / float64(allExtCpu)) * 100, Timestamp: time.Now().Unix()}}}}}, nil } diff --git a/pkg/ensurance/collector/types/types.go b/pkg/ensurance/collector/types/types.go index f56b72dd1..42f930550 100644 --- a/pkg/ensurance/collector/types/types.go +++ b/pkg/ensurance/collector/types/types.go @@ -68,6 +68,7 @@ func GetCgroupPath(p *v1.Pod, cgroupDriver string) string { return "" } } + func GetCgroupName(p *v1.Pod) cm.CgroupName { switch p.Status.QOSClass { case v1.PodQOSGuaranteed: diff --git a/pkg/ensurance/executor/cpu_usage.go b/pkg/ensurance/executor/cpu_usage.go index a521907ff..65bd1f0cf 100644 --- a/pkg/ensurance/executor/cpu_usage.go +++ b/pkg/ensurance/executor/cpu_usage.go @@ -7,7 +7,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" "github.com/gocrane/crane/pkg/ensurance/executor/sort" cruntime "github.com/gocrane/crane/pkg/ensurance/runtime" "github.com/gocrane/crane/pkg/metrics" @@ -15,59 +15,59 @@ import ( ) func init() { - registerMetricMap(cpu_usage) + registerMetricMap(cpuUsage) } -var cpu_usage = metric{ +var cpuUsage = metric{ Name: CpuUsage, ActionPriority: 5, - SortAble: true, - SortFunc: sort.CpuUsageSorter, + Sortable: true, + SortFunc: sort.CpuUsageSort, - ThrottleAble: true, + Throttleable: true, ThrottleQuantified: true, ThrottleFunc: throttleOnePodCpu, RestoreFunc: restoreOnePodCpu, - EvictAble: true, + Evictable: true, EvictQuantified: true, - EvictFunc: evictOnePodCpu, + EvictFunc: evictPod, } func throttleOnePodCpu(ctx *ExecuteContext, index int, ThrottleDownPods ThrottlePods, totalReleasedResource *ReleaseResource) (errPodKeys []string, released ReleaseResource) { - pod, err := ctx.PodLister.Pods(ThrottleDownPods[index].PodKey.Namespace).Get(ThrottleDownPods[index].PodKey.Name) + pod, err := ctx.PodLister.Pods(ThrottleDownPods[index].Key.Namespace).Get(ThrottleDownPods[index].Key.Name) if err != nil { - errPodKeys = append(errPodKeys, fmt.Sprintf("pod %s not found", ThrottleDownPods[index].PodKey.String())) + errPodKeys = append(errPodKeys, fmt.Sprintf("pod %s not found", ThrottleDownPods[index].Key.String())) return } // Throttle for CPU metrics - klog.V(6).Infof("index %d, containerusage is %#v", index, ThrottleDownPods[index].ContainerCPUUsages) + klog.V(6).Infof("index %d, ContainerCPUUsages is %#v", index, ThrottleDownPods[index].ContainerCPUUsages) for _, v := range ThrottleDownPods[index].ContainerCPUUsages { - // pause container to skip + // skip pause container if v.ContainerName == "" { continue } - klog.V(4).Infof("ThrottleExecutor begin to avoid container %s/%s", klog.KObj(pod), v.ContainerName) + klog.V(4).Infof("Begin to avoid container %s/%s", klog.KObj(pod), v.ContainerName) containerCPUQuota, err := podinfo.GetUsageById(ThrottleDownPods[index].ContainerCPUQuotas, v.ContainerId) if err != nil { - errPodKeys = append(errPodKeys, err.Error(), ThrottleDownPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, err.Error(), ThrottleDownPods[index].Key.String()) continue } containerCPUPeriod, err := podinfo.GetUsageById(ThrottleDownPods[index].ContainerCPUPeriods, v.ContainerId) if err != nil { - errPodKeys = append(errPodKeys, err.Error(), ThrottleDownPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, err.Error(), ThrottleDownPods[index].Key.String()) continue } container, err := utils.GetPodContainerByName(pod, v.ContainerName) if err != nil { - errPodKeys = append(errPodKeys, err.Error(), ThrottleDownPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, err.Error(), ThrottleDownPods[index].Key.String()) continue } @@ -96,14 +96,14 @@ func throttleOnePodCpu(ctx *ExecuteContext, index int, ThrottleDownPods Throttle if !utils.AlmostEqual(containerCPUQuotaNew*containerCPUPeriod.Value, containerCPUQuota.Value) { err = cruntime.UpdateContainerResources(ctx.RuntimeClient, v.ContainerId, cruntime.UpdateOptions{CPUQuota: int64(containerCPUQuotaNew * containerCPUPeriod.Value)}) if err != nil { - errPodKeys = append(errPodKeys, fmt.Sprintf("failed to updateResource for %s/%s, error: %v", ThrottleDownPods[index].PodKey.String(), v.ContainerName, err)) + errPodKeys = append(errPodKeys, fmt.Sprintf("failed to updateResource for %s/%s, error: %v", ThrottleDownPods[index].Key.String(), v.ContainerName, err)) continue } else { klog.V(4).Infof("ThrottleExecutor avoid pod %s, container %s, set cpu quota %.2f.", klog.KObj(pod), v.ContainerName, containerCPUQuotaNew*containerCPUPeriod.Value) - released = ConstructCpuUsageRelease(ThrottleDownPods[index], containerCPUQuotaNew, v.Value) - klog.V(6).Infof("For pod %s, container %s, release %f cpu usage", ThrottleDownPods[index].PodKey.String(), container.Name, released[CpuUsage]) + released = releaseCPUUsage(ThrottleDownPods[index], containerCPUQuotaNew, v.Value) + klog.V(6).Infof("For pod %s, container %s, release %f cpu usage", ThrottleDownPods[index].Key.String(), container.Name, released[CpuUsage]) totalReleasedResource.Add(released) } @@ -113,9 +113,9 @@ func throttleOnePodCpu(ctx *ExecuteContext, index int, ThrottleDownPods Throttle } func restoreOnePodCpu(ctx *ExecuteContext, index int, ThrottleUpPods ThrottlePods, totalReleasedResource *ReleaseResource) (errPodKeys []string, released ReleaseResource) { - pod, err := ctx.PodLister.Pods(ThrottleUpPods[index].PodKey.Namespace).Get(ThrottleUpPods[index].PodKey.Name) + pod, err := ctx.PodLister.Pods(ThrottleUpPods[index].Key.Namespace).Get(ThrottleUpPods[index].Key.Name) if err != nil { - errPodKeys = append(errPodKeys, "not found ", ThrottleUpPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, "not found ", ThrottleUpPods[index].Key.String()) return } @@ -130,19 +130,19 @@ func restoreOnePodCpu(ctx *ExecuteContext, index int, ThrottleUpPods ThrottlePod containerCPUQuota, err := podinfo.GetUsageById(ThrottleUpPods[index].ContainerCPUQuotas, v.ContainerId) if err != nil { - errPodKeys = append(errPodKeys, err.Error(), ThrottleUpPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, err.Error(), ThrottleUpPods[index].Key.String()) continue } containerCPUPeriod, err := podinfo.GetUsageById(ThrottleUpPods[index].ContainerCPUPeriods, v.ContainerId) if err != nil { - errPodKeys = append(errPodKeys, err.Error(), ThrottleUpPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, err.Error(), ThrottleUpPods[index].Key.String()) continue } container, err := utils.GetPodContainerByName(pod, v.ContainerName) if err != nil { - errPodKeys = append(errPodKeys, err.Error(), ThrottleUpPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, err.Error(), ThrottleUpPods[index].Key.String()) continue } @@ -175,20 +175,20 @@ func restoreOnePodCpu(ctx *ExecuteContext, index int, ThrottleUpPods ThrottlePod if utils.AlmostEqual(containerCPUQuotaNew, -1) { err = cruntime.UpdateContainerResources(ctx.RuntimeClient, v.ContainerId, cruntime.UpdateOptions{CPUQuota: int64(-1)}) if err != nil { - errPodKeys = append(errPodKeys, fmt.Sprintf("Failed to updateResource, err %s", err.Error()), ThrottleUpPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, fmt.Sprintf("Failed to updateResource, err %s", err.Error()), ThrottleUpPods[index].Key.String()) continue } } else { err = cruntime.UpdateContainerResources(ctx.RuntimeClient, v.ContainerId, cruntime.UpdateOptions{CPUQuota: int64(containerCPUQuotaNew * containerCPUPeriod.Value)}) if err != nil { klog.Errorf("Failed to updateResource, err %s", err.Error()) - errPodKeys = append(errPodKeys, fmt.Sprintf("Failed to updateResource, err %s", err.Error()), ThrottleUpPods[index].PodKey.String()) + errPodKeys = append(errPodKeys, fmt.Sprintf("Failed to updateResource, err %s", err.Error()), ThrottleUpPods[index].Key.String()) continue } klog.V(4).Infof("ThrottleExecutor restore pod %s, container %s, set cpu quota %.2f, .", klog.KObj(pod), v.ContainerName, containerCPUQuotaNew*containerCPUPeriod.Value) - released = ConstructCpuUsageRelease(ThrottleUpPods[index], containerCPUQuotaNew, v.Value) - klog.V(6).Infof("For pod %s, container %s, restore %f cpu usage", ThrottleUpPods[index].PodKey, container.Name, released[CpuUsage]) + released = releaseCPUUsage(ThrottleUpPods[index], containerCPUQuotaNew, v.Value) + klog.V(6).Infof("For pod %s, container %s, restore %f cpu usage", ThrottleUpPods[index].Key, container.Name, released[CpuUsage]) totalReleasedResource.Add(released) } @@ -198,43 +198,42 @@ func restoreOnePodCpu(ctx *ExecuteContext, index int, ThrottleUpPods ThrottlePod return } -func evictOnePodCpu(wg *sync.WaitGroup, ctx *ExecuteContext, index int, totalReleasedResource *ReleaseResource, EvictPods EvictPods) (errPodKeys []string, released ReleaseResource) { +func evictPod(wg *sync.WaitGroup, ctx *ExecuteContext, index int, totalReleasedResource *ReleaseResource, EvictPods EvictPods) (errPodKeys []string, released ReleaseResource) { wg.Add(1) // Calculate release resources - released = ConstructCpuUsageRelease(EvictPods[index], 0.0, 0.0) + released = releaseCPUUsage(EvictPods[index], 0.0, 0.0) totalReleasedResource.Add(released) go func(evictPod podinfo.PodContext) { defer wg.Done() - pod, err := ctx.PodLister.Pods(evictPod.PodKey.Namespace).Get(evictPod.PodKey.Name) + pod, err := ctx.PodLister.Pods(evictPod.Key.Namespace).Get(evictPod.Key.Name) if err != nil { - errPodKeys = append(errPodKeys, "not found ", evictPod.PodKey.String()) + errPodKeys = append(errPodKeys, "not found ", evictPod.Key.String()) return } - + klog.Warningf("Evicting pod %v", evictPod.Key) err = utils.EvictPodWithGracePeriod(ctx.Client, pod, evictPod.DeletionGracePeriodSeconds) if err != nil { - errPodKeys = append(errPodKeys, "evict failed ", evictPod.PodKey.String()) - klog.Warningf("Failed to evict pod %s: %v", evictPod.PodKey.String(), err) + errPodKeys = append(errPodKeys, "evict failed ", evictPod.Key.String()) + klog.Warningf("Failed to evict pod %s: %v", evictPod.Key.String(), err) return } - metrics.ExecutorEvictCountsInc() - klog.V(4).Infof("Pod %s is evicted", klog.KObj(pod)) + klog.Warningf("Pod %s is evicted", klog.KObj(pod)) }(EvictPods[index]) return } -func ConstructCpuUsageRelease(pod podinfo.PodContext, containerCPUQuotaNew, currentContainerCpuUsage float64) ReleaseResource { - if pod.PodType == podinfo.Evict { +func releaseCPUUsage(pod podinfo.PodContext, containerCPUQuotaNew, currentContainerCpuUsage float64) ReleaseResource { + if pod.ActionType == podinfo.Evict { return ReleaseResource{ CpuUsage: pod.PodCPUUsage * CpuQuotaCoefficient, } } - if pod.PodType == podinfo.ThrottleDown { + if pod.ActionType == podinfo.ThrottleDown { reduction := (currentContainerCpuUsage - containerCPUQuotaNew) * CpuQuotaCoefficient if reduction > 0 { return ReleaseResource{ @@ -243,7 +242,7 @@ func ConstructCpuUsageRelease(pod podinfo.PodContext, containerCPUQuotaNew, curr } return ReleaseResource{} } - if pod.PodType == podinfo.ThrottleUp { + if pod.ActionType == podinfo.ThrottleUp { reduction := (containerCPUQuotaNew - currentContainerCpuUsage) * CpuQuotaCoefficient if reduction > 0 { return ReleaseResource{ diff --git a/pkg/ensurance/executor/evict.go b/pkg/ensurance/executor/evict.go index b73203d64..436da8114 100644 --- a/pkg/ensurance/executor/evict.go +++ b/pkg/ensurance/executor/evict.go @@ -9,7 +9,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" execsort "github.com/gocrane/crane/pkg/ensurance/executor/sort" "github.com/gocrane/crane/pkg/known" "github.com/gocrane/crane/pkg/metrics" @@ -17,15 +17,15 @@ import ( type EvictExecutor struct { EvictPods EvictPods - // All metrics(not only can be quantified metrics) metioned in triggerd NodeQOSEnsurancePolicy and their corresponding waterlines - EvictWaterLine WaterLines + // All metrics(not only can be quantified metrics) metioned in triggerd NodeQOS and their corresponding watermarks + EvictWatermark Watermarks } type EvictPods []podinfo.PodContext func (e EvictPods) Find(key types.NamespacedName) int { for i, v := range e { - if v.PodKey == key { + if v.Key == key { return i } } @@ -53,40 +53,40 @@ func (e *EvictExecutor) Avoid(ctx *ExecuteContext) error { totalReleased := ReleaseResource{} /* The step to evict: - 1. If EvictWaterLine has metrics that can't be quantified, select a evictable metric which has the highest action priority, use its EvictFunc to evict all selected pods, then return - 2. Get the gaps between current usage and waterlines + 1. If EvictWatermark has metrics that can't be quantified, select a evictable metric which has the highest action priority, use its EvictFunc to evict all selected pods, then return + 2. Get the gaps between current usage and watermarks 2.1 If there is a metric that can't get current usage, select a evictable metric which has the highest action priority, use its EvictFunc to evict all selected pods, then return 2.2 Traverse metrics that can be quantified, if there is gap for the metric, then sort candidate pods by its SortFunc if exists, otherwise use GeneralSorter by default. - Then evict sorted pods one by one util there is no gap to waterline + Then evict sorted pods one by one util there is no gap to watermark */ - metricsEvictQuantified, MetricsNotEvcitQuantified := e.EvictWaterLine.DivideMetricsByEvictQuantified() + quantified, notQuantified := e.EvictWatermark.DivideMetricsByEvictQuantified() // There is a metric that can't be EvictQuantified, so evict all selected pods - if len(MetricsNotEvcitQuantified) != 0 { + if len(notQuantified) != 0 { klog.V(6).Info("There is a metric that can't be EvcitQuantified") - highestPriorityMetric := e.EvictWaterLine.GetHighestPriorityEvictAbleMetric() + highestPriorityMetric := e.EvictWatermark.GetHighestPriorityEvictableMetric() if highestPriorityMetric != "" { klog.V(6).Infof("The highestPriorityMetric is %s", highestPriorityMetric) errPodKeys = e.evictPods(ctx, &totalReleased, highestPriorityMetric) } } else { - _, _, ctx.EvictGapToWaterLines = buildGapToWaterLine(ctx.stateMap, ThrottleExecutor{}, *e, ctx.executeExcessPercent) + ctx.ToBeEvict = calculateGaps(ctx.stateMap, nil, e, ctx.executeExcessPercent) - if ctx.EvictGapToWaterLines.HasUsageMissedMetric() { + if ctx.ToBeEvict.HasUsageMissedMetric() { klog.V(6).Infof("There is a metric usage missed") - highestPriorityMetric := e.EvictWaterLine.GetHighestPriorityEvictAbleMetric() + highestPriorityMetric := e.EvictWatermark.GetHighestPriorityEvictableMetric() if highestPriorityMetric != "" { errPodKeys = e.evictPods(ctx, &totalReleased, highestPriorityMetric) } } else { - // The metrics in EvictGapToWaterLines are can be EvictQuantified and has current usage, then evict precisely + // The metrics in ToBeEvict are can be EvictQuantified and has current usage, then evict precisely var released ReleaseResource wg := sync.WaitGroup{} - for _, m := range metricsEvictQuantified { - klog.V(6).Infof("Evict precisely on metric %s", m) - if metricMap[m].SortAble { + for _, m := range quantified { + klog.V(6).Infof("Evict precisely on metric %s, and current gaps are %+v", m, ctx.ToBeEvict) + if metricMap[m].Sortable { metricMap[m].SortFunc(e.EvictPods) } else { execsort.GeneralSorter(e.EvictPods) @@ -94,19 +94,17 @@ func (e *EvictExecutor) Avoid(ctx *ExecuteContext) error { klog.V(6).Info("After sort, the sequence to evict is ") for _, pc := range e.EvictPods { - klog.V(6).Info(pc.PodKey.String()) + klog.V(6).Info(pc.Key.String()) } - - for !ctx.EvictGapToWaterLines.TargetGapsRemoved(m) { - klog.V(6).Infof("For metric %s, there is still gap to waterlines: %f", m, ctx.EvictGapToWaterLines[m]) - if podinfo.HasNoExecutedPod(e.EvictPods) { - index := podinfo.GetFirstNoExecutedPod(e.EvictPods) + for !ctx.ToBeEvict.TargetGapsRemoved(m) { + klog.V(2).Infof("For metric %s, there is more gap to watermarks: %f of %s", m, ctx.ToBeEvict[m], m) + if podinfo.ContainsPendingPod(e.EvictPods) { + index := podinfo.GetFirstPendingPod(e.EvictPods) errKeys, released = metricMap[m].EvictFunc(&wg, ctx, index, &totalReleased, e.EvictPods) errPodKeys = append(errPodKeys, errKeys...) - klog.V(6).Infof("Evict pods %s, released %f resource", e.EvictPods[index].PodKey, released[m]) - - e.EvictPods[index].HasBeenActioned = true - ctx.EvictGapToWaterLines[m] -= released[m] + klog.Warningf("Evicted pods %s, released %f of %s", e.EvictPods[index].Key, released[m], m) + e.EvictPods[index].Executed = true + ctx.ToBeEvict[m] -= released[m] } else { klog.V(6).Info("There is no pod that can be evicted") break @@ -128,7 +126,7 @@ func (e *EvictExecutor) Restore(ctx *ExecuteContext) error { return nil } -func (e *EvictExecutor) evictPods(ctx *ExecuteContext, totalReleasedResource *ReleaseResource, m WaterLineMetric) (errPodKeys []string) { +func (e *EvictExecutor) evictPods(ctx *ExecuteContext, totalReleasedResource *ReleaseResource, m WatermarkMetric) (errPodKeys []string) { wg := sync.WaitGroup{} for i := range e.EvictPods { errKeys, _ := metricMap[m].EvictFunc(&wg, ctx, i, totalReleasedResource, e.EvictPods) diff --git a/pkg/ensurance/executor/interface.go b/pkg/ensurance/executor/interface.go index b51016271..3e0330385 100644 --- a/pkg/ensurance/executor/interface.go +++ b/pkg/ensurance/executor/interface.go @@ -29,13 +29,13 @@ type ExecuteContext struct { RuntimeClient pb.RuntimeServiceClient RuntimeConn *grpc.ClientConn - // Gap for metrics EvictAble/ThrottleAble - // Key is the metric name, value is (actual used)-(the lowest waterline for NodeQOSEnsurancePolicies which use throttleDown action) - ThrottoleDownGapToWaterLines GapToWaterLines - // Key is the metric name, value is (actual used)-(the lowest waterline for NodeQOSEnsurancePolicies which use throttleUp action) - ThrottoleUpGapToWaterLines GapToWaterLines - // key is the metric name, value is (actual used)-(the lowest waterline for NodeQOSEnsurancePolicies which use evict action) - EvictGapToWaterLines GapToWaterLines + // Gap for metrics Evictable/ThrottleAble + // Key is the metric name, value is (actual used)-(the lowest watermark for NodeQOSEnsurancePolicies which use throttleDown action) + ToBeThrottleDown Gaps + // Key is the metric name, value is (actual used)-(the lowest watermark for NodeQOSEnsurancePolicies which use throttleUp action) + ToBeThrottleUp Gaps + // key is the metric name, value is (actual used)-(the lowest watermark for NodeQOSEnsurancePolicies which use evict action) + ToBeEvict Gaps stateMap map[string][]common.TimeSeries diff --git a/pkg/ensurance/executor/metric.go b/pkg/ensurance/executor/metric.go index 50f5e1b66..002f98ff7 100644 --- a/pkg/ensurance/executor/metric.go +++ b/pkg/ensurance/executor/metric.go @@ -3,12 +3,12 @@ package executor import ( "sync" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" ) type metric struct { // Should be consistent with metrics in collector/types/types.go - Name WaterLineMetric + Name WatermarkMetric // ActionPriority describe the priority of the metric, used to choose the highest priority metric which can be throttlable or evictable // when there is MetricsNotThrottleQuantified in executor process; @@ -16,40 +16,22 @@ type metric struct { // Some incompressible metric such as memory usage can be given a higher priority ActionPriority int - SortAble bool + Sortable bool SortFunc func(pods []podinfo.PodContext) - ThrottleAble bool + Throttleable bool ThrottleQuantified bool ThrottleFunc func(ctx *ExecuteContext, index int, ThrottleDownPods ThrottlePods, totalReleasedResource *ReleaseResource) (errPodKeys []string, released ReleaseResource) RestoreFunc func(ctx *ExecuteContext, index int, ThrottleUpPods ThrottlePods, totalReleasedResource *ReleaseResource) (errPodKeys []string, released ReleaseResource) - EvictAble bool + Evictable bool EvictQuantified bool // If use goroutine to evcit, make sure to calculate release resources outside the goroutine EvictFunc func(wg *sync.WaitGroup, ctx *ExecuteContext, index int, totalReleasedResource *ReleaseResource, EvictPods EvictPods) (errPodKeys []string, released ReleaseResource) } -var metricMap = make(map[WaterLineMetric]metric) +var metricMap = make(map[WatermarkMetric]metric) func registerMetricMap(m metric) { metricMap[m.Name] = m } - -func GetThrottleAbleMetricName() (throttleAbleMetricList []WaterLineMetric) { - for _, m := range metricMap { - if m.ThrottleAble { - throttleAbleMetricList = append(throttleAbleMetricList, m.Name) - } - } - return -} - -func GetEvictAbleMetricName() (evictAbleMetricList []WaterLineMetric) { - for _, m := range metricMap { - if m.EvictAble { - evictAbleMetricList = append(evictAbleMetricList, m.Name) - } - } - return -} diff --git a/pkg/ensurance/executor/pod-info/pod_info.go b/pkg/ensurance/executor/pod-info/pod_info.go deleted file mode 100644 index 067181e86..000000000 --- a/pkg/ensurance/executor/pod-info/pod_info.go +++ /dev/null @@ -1,244 +0,0 @@ -package pod_info - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - corelisters "k8s.io/client-go/listers/core/v1" - "k8s.io/klog/v2" - - ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" - "github.com/gocrane/crane/pkg/common" - stypes "github.com/gocrane/crane/pkg/ensurance/collector/types" - "github.com/gocrane/crane/pkg/utils" -) - -type ClassAndPriority struct { - PodQOSClass v1.PodQOSClass - PriorityClassValue int32 -} - -type PodType string - -const ( - ThrottleDown PodType = "ThrottleDown" - ThrottleUp PodType = "ThrottleUp" - Evict PodType = "Evict" -) - -type ContainerUsage struct { - ContainerName string - ContainerId string - Value float64 -} - -func GetUsageById(usages []ContainerUsage, containerId string) (ContainerUsage, error) { - for _, v := range usages { - if v.ContainerId == containerId { - return v, nil - } - } - - return ContainerUsage{}, fmt.Errorf("containerUsage not found") -} - -func GetPodUsage(metricName string, stateMap map[string][]common.TimeSeries, pod *v1.Pod) (float64, []ContainerUsage) { - var podUsage = 0.0 - var containerUsages []ContainerUsage - var podMaps = map[string]string{common.LabelNamePodName: pod.Name, common.LabelNamePodNamespace: pod.Namespace, common.LabelNamePodUid: string(pod.UID)} - state, ok := stateMap[metricName] - if !ok { - return podUsage, containerUsages - } - for _, vv := range state { - var labelMaps = common.Labels2Maps(vv.Labels) - if utils.ContainMaps(labelMaps, podMaps) { - if labelMaps[common.LabelNameContainerId] == "" { - podUsage = vv.Samples[0].Value - } else { - containerUsages = append(containerUsages, ContainerUsage{ContainerId: labelMaps[common.LabelNameContainerId], - ContainerName: labelMaps[common.LabelNameContainerName], Value: vv.Samples[0].Value}) - } - } - } - - return podUsage, containerUsages -} - -type CPURatio struct { - //the min of cpu ratio for pods - MinCPURatio uint64 `json:"minCPURatio,omitempty"` - - //the step of cpu share and limit for once down-size (1-100) - StepCPURatio uint64 `json:"stepCPURatio,omitempty"` -} - -type MemoryThrottleExecutor struct { - // to force gc the page cache of low level pods - ForceGC bool `json:"forceGC,omitempty"` -} - -type PodContext struct { - PodKey types.NamespacedName - ClassAndPriority ClassAndPriority - PodCPUUsage float64 - ContainerCPUUsages []ContainerUsage - PodCPUShare float64 - ContainerCPUShares []ContainerUsage - PodCPUQuota float64 - ContainerCPUQuotas []ContainerUsage - PodCPUPeriod float64 - ContainerCPUPeriods []ContainerUsage - ExtCpuBeUsed bool - ExtCpuLimit int64 - ExtCpuRequest int64 - StartTime *metav1.Time - - PodType PodType - - CPUThrottle CPURatio - MemoryThrottle MemoryThrottleExecutor - - DeletionGracePeriodSeconds *int32 - - HasBeenActioned bool -} - -func HasNoExecutedPod(pods []PodContext) bool { - for _, p := range pods { - if p.HasBeenActioned == false { - return true - } - } - return false -} - -func GetFirstNoExecutedPod(pods []PodContext) int { - for index, p := range pods { - if p.HasBeenActioned == false { - return index - } - } - return -1 -} - -func BuildPodBasicInfo(pod *v1.Pod, stateMap map[string][]common.TimeSeries, action *ensuranceapi.AvoidanceAction, podType PodType) PodContext { - var podContext PodContext - - podContext.ClassAndPriority = ClassAndPriority{PodQOSClass: pod.Status.QOSClass, PriorityClassValue: utils.GetInt32withDefault(pod.Spec.Priority, 0)} - podContext.PodKey = types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} - - podContext.PodCPUUsage, podContext.ContainerCPUUsages = GetPodUsage(string(stypes.MetricNameContainerCpuTotalUsage), stateMap, pod) - podContext.PodCPUShare, podContext.ContainerCPUShares = GetPodUsage(string(stypes.MetricNameContainerCpuLimit), stateMap, pod) - podContext.PodCPUQuota, podContext.ContainerCPUQuotas = GetPodUsage(string(stypes.MetricNameContainerCpuQuota), stateMap, pod) - podContext.PodCPUPeriod, podContext.ContainerCPUPeriods = GetPodUsage(string(stypes.MetricNameContainerCpuPeriod), stateMap, pod) - podContext.ExtCpuBeUsed, podContext.ExtCpuLimit, podContext.ExtCpuRequest = utils.ExtResourceAllocated(pod, v1.ResourceCPU) - podContext.StartTime = pod.Status.StartTime - - if action.Spec.Throttle != nil { - podContext.CPUThrottle.MinCPURatio = uint64(action.Spec.Throttle.CPUThrottle.MinCPURatio) - podContext.CPUThrottle.StepCPURatio = uint64(action.Spec.Throttle.CPUThrottle.StepCPURatio) - } - - podContext.PodType = podType - - return podContext -} - -func CompareClassAndPriority(a, b ClassAndPriority) int32 { - qosClassCmp := comparePodQosClass(a.PodQOSClass, b.PodQOSClass) - if qosClassCmp != 0 { - return qosClassCmp - } - if a.PriorityClassValue == b.PriorityClassValue { - return 0 - } else if a.PriorityClassValue < b.PriorityClassValue { - return -1 - } - return 1 -} - -func (s ClassAndPriority) Less(i ClassAndPriority) bool { - if comparePodQosClass(s.PodQOSClass, i.PodQOSClass) == 1 { - return false - } - - if comparePodQosClass(s.PodQOSClass, i.PodQOSClass) == -1 { - return true - } - - return s.PriorityClassValue < i.PriorityClassValue -} - -func (s ClassAndPriority) Greater(i ClassAndPriority) bool { - if comparePodQosClass(s.PodQOSClass, i.PodQOSClass) == 1 { - return true - } - - if comparePodQosClass(s.PodQOSClass, i.PodQOSClass) == -1 { - return false - } - - return s.PriorityClassValue > i.PriorityClassValue -} - -func GetMaxQOSPriority(podLister corelisters.PodLister, podTypes []types.NamespacedName) (types.NamespacedName, ClassAndPriority) { - - var podType types.NamespacedName - var scheduledQOSPriority ClassAndPriority - - for _, podNamespace := range podTypes { - if pod, err := podLister.Pods(podNamespace.Namespace).Get(podNamespace.Name); err != nil { - klog.V(6).Infof("Warning: getMaxQOSPriority get pod %s not found", podNamespace.String()) - continue - } else { - var priority = ClassAndPriority{PodQOSClass: pod.Status.QOSClass, PriorityClassValue: utils.GetInt32withDefault(pod.Spec.Priority, 0) - 1} - if priority.Greater(scheduledQOSPriority) { - scheduledQOSPriority = priority - podType = podNamespace - } - } - } - - return podType, scheduledQOSPriority -} - -// We defined guaranteed is the highest qos class, burstable is the middle level -// bestEffort is the lowest -// if a qos class is greater than b, return 1 -// if a qos class is less than b, return -1 -// if a qos class equal with b , return 0 -func comparePodQosClass(a v1.PodQOSClass, b v1.PodQOSClass) int32 { - switch b { - case v1.PodQOSGuaranteed: - if a == v1.PodQOSGuaranteed { - return 0 - } else { - return -1 - } - case v1.PodQOSBurstable: - if a == v1.PodQOSGuaranteed { - return 1 - } else if a == v1.PodQOSBurstable { - return 0 - } else { - return -1 - } - case v1.PodQOSBestEffort: - if (a == v1.PodQOSGuaranteed) || (a == v1.PodQOSBurstable) { - return 1 - } else if a == v1.PodQOSBestEffort { - return 0 - } else { - return -1 - } - default: - if (a == v1.PodQOSGuaranteed) || (a == v1.PodQOSBurstable) || (a == v1.PodQOSBestEffort) { - return 1 - } else { - return 0 - } - } -} diff --git a/pkg/ensurance/executor/podinfo/pod_info.go b/pkg/ensurance/executor/podinfo/pod_info.go new file mode 100644 index 000000000..928b0d1c2 --- /dev/null +++ b/pkg/ensurance/executor/podinfo/pod_info.go @@ -0,0 +1,138 @@ +package podinfo + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" + "github.com/gocrane/crane/pkg/common" + stypes "github.com/gocrane/crane/pkg/ensurance/collector/types" + "github.com/gocrane/crane/pkg/utils" +) + +type ClassAndPriority struct { + PodQOSClass v1.PodQOSClass + PriorityClassValue int32 +} + +type ActionType string + +const ( + ThrottleDown ActionType = "ThrottleDown" + ThrottleUp ActionType = "ThrottleUp" + Evict ActionType = "Evict" +) + +type ContainerState struct { + ContainerName string + ContainerId string + Value float64 +} + +func GetUsageById(usages []ContainerState, containerId string) (ContainerState, error) { + for _, v := range usages { + if v.ContainerId == containerId { + return v, nil + } + } + + return ContainerState{}, fmt.Errorf("containerUsage not found") +} + +func GetPodUsage(metricName string, stateMap map[string][]common.TimeSeries, pod *v1.Pod) (float64, []ContainerState) { + var podUsage = 0.0 + var containerUsages []ContainerState + var podMaps = map[string]string{common.LabelNamePodName: pod.Name, common.LabelNamePodNamespace: pod.Namespace, common.LabelNamePodUid: string(pod.UID)} + state, ok := stateMap[metricName] + if !ok { + return podUsage, containerUsages + } + for _, vv := range state { + var labelMaps = common.Labels2Maps(vv.Labels) + if utils.ContainMaps(labelMaps, podMaps) { + if labelMaps[common.LabelNameContainerId] == "" { + podUsage = vv.Samples[0].Value + } else { + containerUsages = append(containerUsages, ContainerState{ContainerId: labelMaps[common.LabelNameContainerId], + ContainerName: labelMaps[common.LabelNameContainerName], Value: vv.Samples[0].Value}) + } + } + } + + return podUsage, containerUsages +} + +type CPURatio struct { + //the min of cpu ratio for pods + MinCPURatio uint64 `json:"minCPURatio,omitempty"` + + //the step of cpu share and limit for once down-size (1-100) + StepCPURatio uint64 `json:"stepCPURatio,omitempty"` +} + +type MemoryThrottleExecutor struct { + // to force gc the page cache of low level pods + ForceGC bool `json:"forceGC,omitempty"` +} + +type PodContext struct { + Key types.NamespacedName + QOSClass v1.PodQOSClass + Priority int32 + StartTime *metav1.Time + DeletionGracePeriodSeconds *int32 + + ElasticCPU int64 + PodCPUUsage, PodCPUShare, PodCPUQuota, PodCPUPeriod float64 + ContainerCPUUsages, ContainerCPUShares, ContainerCPUQuotas, ContainerCPUPeriods []ContainerState + + ActionType ActionType + CPUThrottle CPURatio + Executed bool +} + +func ContainsPendingPod(pods []PodContext) bool { + for _, p := range pods { + if p.Executed == false { + return true + } + } + return false +} + +func GetFirstPendingPod(pods []PodContext) int { + for index, p := range pods { + if p.Executed == false { + return index + } + } + return -1 +} + +func BuildPodActionContext(pod *v1.Pod, stateMap map[string][]common.TimeSeries, action *ensuranceapi.AvoidanceAction, actionType ActionType) PodContext { + var podContext PodContext + + podContext.QOSClass = pod.Status.QOSClass + podContext.Priority = utils.GetInt32withDefault(pod.Spec.Priority, 0) + + podContext.Key = types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} + + podContext.PodCPUUsage, podContext.ContainerCPUUsages = GetPodUsage(string(stypes.MetricNameContainerCpuTotalUsage), stateMap, pod) + podContext.PodCPUShare, podContext.ContainerCPUShares = GetPodUsage(string(stypes.MetricNameContainerCpuLimit), stateMap, pod) + podContext.PodCPUQuota, podContext.ContainerCPUQuotas = GetPodUsage(string(stypes.MetricNameContainerCpuQuota), stateMap, pod) + podContext.PodCPUPeriod, podContext.ContainerCPUPeriods = GetPodUsage(string(stypes.MetricNameContainerCpuPeriod), stateMap, pod) + podContext.ElasticCPU = utils.GetElasticResourceLimit(pod, v1.ResourceCPU) + podContext.StartTime = pod.Status.StartTime + + if action.Spec.Throttle != nil { + podContext.CPUThrottle.MinCPURatio = uint64(action.Spec.Throttle.CPUThrottle.MinCPURatio) + podContext.CPUThrottle.StepCPURatio = uint64(action.Spec.Throttle.CPUThrottle.StepCPURatio) + } + + podContext.ActionType = actionType + + return podContext +} diff --git a/pkg/ensurance/executor/release_resource.go b/pkg/ensurance/executor/release_resource.go index fe245b5ce..2d4066f0f 100644 --- a/pkg/ensurance/executor/release_resource.go +++ b/pkg/ensurance/executor/release_resource.go @@ -1,6 +1,6 @@ package executor -type ReleaseResource map[WaterLineMetric]float64 +type ReleaseResource map[WatermarkMetric]float64 func (r ReleaseResource) Add(new ReleaseResource) { for metric, value := range new { diff --git a/pkg/ensurance/executor/schedule.go b/pkg/ensurance/executor/schedule.go index 663ad227e..0e91d89f5 100644 --- a/pkg/ensurance/executor/schedule.go +++ b/pkg/ensurance/executor/schedule.go @@ -6,7 +6,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" "github.com/gocrane/crane/pkg/known" "github.com/gocrane/crane/pkg/metrics" "github.com/gocrane/crane/pkg/utils" @@ -17,8 +16,7 @@ const ( ) type ScheduleExecutor struct { - DisableClassAndPriority *podinfo.ClassAndPriority - RestoreClassAndPriority *podinfo.ClassAndPriority + ToBeDisable, ToBeRestore bool } func (b *ScheduleExecutor) Avoid(ctx *ExecuteContext) error { @@ -26,9 +24,9 @@ func (b *ScheduleExecutor) Avoid(ctx *ExecuteContext) error { metrics.UpdateLastTimeWithSubComponent(string(known.ModuleActionExecutor), string(metrics.SubComponentSchedule), metrics.StepAvoid, start) defer metrics.UpdateDurationFromStartWithSubComponent(string(known.ModuleActionExecutor), string(metrics.SubComponentSchedule), metrics.StepAvoid, start) - klog.V(6).Info("DisableScheduledExecutor avoid, %v", *b) + klog.V(4).Infof("ScheduleExecutor, ToBeDisable: %v, ToBeRestore: %v", b.ToBeDisable, b.ToBeRestore) - if b.DisableClassAndPriority == nil { + if !b.ToBeDisable { metrics.UpdateExecutorStatus(metrics.SubComponentSchedule, metrics.StepAvoid, 0) return nil } @@ -56,9 +54,9 @@ func (b *ScheduleExecutor) Restore(ctx *ExecuteContext) error { metrics.UpdateLastTimeWithSubComponent(string(known.ModuleActionExecutor), string(metrics.SubComponentSchedule), metrics.StepRestore, start) defer metrics.UpdateDurationFromStartWithSubComponent(string(known.ModuleActionExecutor), string(metrics.SubComponentSchedule), metrics.StepRestore, start) - klog.V(6).Info("DisableScheduledExecutor restore, %v", *b) + klog.V(4).Infof("ScheduleExecutor, ToBeDisable: %v, ToBeRestore: %v", b.ToBeDisable, b.ToBeRestore) - if b.RestoreClassAndPriority == nil { + if !b.ToBeRestore { metrics.UpdateExecutorStatus(metrics.SubComponentSchedule, metrics.StepRestore, 0.0) return nil } diff --git a/pkg/ensurance/executor/sort/cpu_usage_sort.go b/pkg/ensurance/executor/sort/cpu_usage_sort.go index 08acb1f99..ef840db64 100644 --- a/pkg/ensurance/executor/sort/cpu_usage_sort.go +++ b/pkg/ensurance/executor/sort/cpu_usage_sort.go @@ -1,47 +1,37 @@ package sort import ( - v1 "k8s.io/api/core/v1" - - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" "github.com/gocrane/crane/pkg/utils" ) -func CpuUsageSorter(pods []podinfo.PodContext) { - orderedBy(classAndPriority, cpuUsage, extCpuUsage, runningTime).Sort(pods) +func CpuUsageSort(pods []podinfo.PodContext) { + // todo, need ut to make sure all cases + orderedBy(UseElasticCPU, ComparePriority, ComparePodQOSClass, CompareCPUUsage, CompareElasticCPU, CompareRunningTime).Sort(pods) } -// extCpuUsage compares the partition of extcpu usage to extcpu limit -func extCpuUsage(p1, p2 podinfo.PodContext) int32 { +// CompareElasticCPU compares the partition of extcpu usage to extcpu limit +func CompareElasticCPU(p1, p2 podinfo.PodContext) int32 { // if both pod don't use ext resource, then return - if p1.ExtCpuBeUsed == false && p2.ExtCpuBeUsed == false { + if p1.ElasticCPU == 0 && p2.ElasticCPU == 0 { return 0 } - p1Ratio := p1.PodCPUUsage / float64(p1.ExtCpuLimit) - p2Ratio := p2.PodCPUUsage / float64(p2.ExtCpuLimit) + p1Ratio := p1.PodCPUUsage / float64(p1.ElasticCPU) + p2Ratio := p2.PodCPUUsage / float64(p2.ElasticCPU) return utils.CmpFloat(p1Ratio, p2Ratio) } -// cpuUsage compares the partition extcpu usage of extcpu limit -func cpuUsage(p1, p2 podinfo.PodContext) int32 { - var p1usage, p2usage float64 - // if both pod is PodQOSBestEffort, then compare the absolute usage;otherwise, cmpare the ratio compared with PodCPUQuota - if p1.ClassAndPriority.PodQOSClass == v1.PodQOSBestEffort && p2.ClassAndPriority.PodQOSClass == v1.PodQOSBestEffort { - p1usage = p1.PodCPUUsage - p2usage = p2.PodCPUUsage - } else { - p1usage = p1.PodCPUUsage * p1.PodCPUPeriod / p1.PodCPUQuota - p2usage = p2.PodCPUUsage * p2.PodCPUPeriod / p2.PodCPUQuota - } - return utils.CmpFloat(p1usage, p2usage) +// CompareCPUUsage compares the partition cpu usage of cpu limit +func CompareCPUUsage(p1, p2 podinfo.PodContext) int32 { + return utils.CmpFloat(p2.PodCPUUsage, p1.PodCPUUsage) } -// extCpuBeUsed compares pod by using ext resource whether -func extCpuBeUsed(p1, p2 podinfo.PodContext) int32 { - use1 := utils.Bool2Uint(p1.ExtCpuBeUsed) - use2 := utils.Bool2Uint(p2.ExtCpuBeUsed) +// UseElasticCPU compares pod by using ext resource whether +func UseElasticCPU(p1, p2 podinfo.PodContext) int32 { + use1 := utils.Bool2Uint(p1.ElasticCPU != 0) + use2 := utils.Bool2Uint(p2.ElasticCPU != 0) - return int32(use1 - use2) + return int32(use2 - use1) } diff --git a/pkg/ensurance/executor/sort/cpu_usage_sort_test.go b/pkg/ensurance/executor/sort/cpu_usage_sort_test.go new file mode 100644 index 000000000..be113c1bd --- /dev/null +++ b/pkg/ensurance/executor/sort/cpu_usage_sort_test.go @@ -0,0 +1,82 @@ +package sort + +import ( + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" +) + +func TestCpuUsageSorter(t *testing.T) { + now := metav1.NewTime(time.Unix(1000, 0).UTC()) + later := metav1.NewTime(time.Unix(2000, 0).UTC()) + // orderedBy(UseElasticCPU, ComparePodQOSClass, ComparePriority, CompareCPUUsage, CompareElasticCPU, CompareRunningTime).Sort(pods) + pods := []podinfo.PodContext{ + { + Key: types.NamespacedName{Name: "elastic-cpu-2"}, + ElasticCPU: 2, + QOSClass: v1.PodQOSBestEffort, + }, + { + Key: types.NamespacedName{Name: "elastic-cpu-4"}, + ElasticCPU: 4, + QOSClass: v1.PodQOSBestEffort, + }, + { + Key: types.NamespacedName{Name: "cpu-1"}, + PodCPUUsage: 1, + QOSClass: v1.PodQOSGuaranteed, + }, + { + Key: types.NamespacedName{Name: "cpu-2"}, + PodCPUUsage: 2, + QOSClass: v1.PodQOSBurstable, + }, + { + Key: types.NamespacedName{Name: "guarantee-1"}, + PodCPUUsage: 1, + QOSClass: v1.PodQOSGuaranteed, + }, + { + Key: types.NamespacedName{Name: "burstable-2"}, + PodCPUUsage: 1, + QOSClass: v1.PodQOSBurstable, + }, + { + Key: types.NamespacedName{Name: "prioirty-2"}, + Priority: 2, + PodCPUUsage: 1, + QOSClass: v1.PodQOSBurstable, + }, + { + Key: types.NamespacedName{Name: "prioirty-2-2"}, + Priority: 2, + PodCPUUsage: 2, + QOSClass: v1.PodQOSBurstable, + }, + { + Key: types.NamespacedName{Name: "priority-1"}, + Priority: 1, + QOSClass: v1.PodQOSBurstable, + }, + { + Key: types.NamespacedName{Name: "time-1"}, + StartTime: &now, + QOSClass: v1.PodQOSGuaranteed, + }, + { + Key: types.NamespacedName{Name: "time-2"}, + StartTime: &later, + QOSClass: v1.PodQOSGuaranteed, + }, + } + CpuUsageSort(pods) + t.Logf("sorted pods:") + for _, p := range pods { + t.Logf("key %s, useElasticCPU %v, qosClass %s, priority %d, usage %f, elasticCPUUsage %d, startTime %v", p.Key, (p.ElasticCPU != 0), p.QOSClass, p.Priority, p.PodCPUUsage, p.ElasticCPU, p.StartTime) + } +} diff --git a/pkg/ensurance/executor/sort/general_sort.go b/pkg/ensurance/executor/sort/general_sort.go index 1a509ce92..32dd0ae94 100644 --- a/pkg/ensurance/executor/sort/general_sort.go +++ b/pkg/ensurance/executor/sort/general_sort.go @@ -1,7 +1,7 @@ package sort -import podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" +import "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" func GeneralSorter(pods []podinfo.PodContext) { - orderedBy(classAndPriority, runningTime).Sort(pods) + orderedBy(ComparePriority, ComparePodQOSClass, CompareRunningTime).Sort(pods) } diff --git a/pkg/ensurance/executor/sort/mem_metrics_sort.go b/pkg/ensurance/executor/sort/mem_metrics_sort.go index 9fa01e742..9d1ae74eb 100644 --- a/pkg/ensurance/executor/sort/mem_metrics_sort.go +++ b/pkg/ensurance/executor/sort/mem_metrics_sort.go @@ -1,9 +1,9 @@ package sort -import podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" +import "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" // Todo: Memory metrics related sort func need to be filled func MemMetricsSorter(pods []podinfo.PodContext) { - orderedBy(classAndPriority, runningTime).Sort(pods) + orderedBy(ComparePriority, ComparePodQOSClass, CompareRunningTime).Sort(pods) } diff --git a/pkg/ensurance/executor/sort/sort.go b/pkg/ensurance/executor/sort/sort.go index 3c5d1a0a8..a2392aec7 100644 --- a/pkg/ensurance/executor/sort/sort.go +++ b/pkg/ensurance/executor/sort/sort.go @@ -3,22 +3,23 @@ package sort import ( "sort" + v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" ) -// RankFunc sorts the pods type RankFunc func(pods []podinfo.PodContext) var sortFunc = map[string]func(p1, p2 podinfo.PodContext) int32{ - "ExtCpuBeUsed": extCpuBeUsed, - "ClassAndPriority": classAndPriority, - "ExtCpuUsage": extCpuUsage, - "CpuUsage": cpuUsage, - "RunningTime": runningTime, + "UseElasticResource": UseElasticCPU, + "PodQOSClass": ComparePodQOSClass, + "ExtCpuUsage": CompareElasticCPU, + "CpuUsage": CompareCPUUsage, + "RunningTime": CompareRunningTime, } +// RankFuncConstruct is a sample for future extends, keep it even it is not called func RankFuncConstruct(customize []string) RankFunc { if len(customize) == 0 { klog.Fatal("If customize sort func is defined, it can't be empty.") @@ -33,14 +34,14 @@ func RankFuncConstruct(customize []string) RankFunc { rankFunc = orderedBy(cmp...).Sort } } else { - rankFunc = CpuUsageSorter + rankFunc = CpuUsageSort } return rankFunc } -// runningTime compares pods by pod's start time -func runningTime(p1, p2 podinfo.PodContext) int32 { +// CompareRunningTime compares pods by pod's start time +func CompareRunningTime(p1, p2 podinfo.PodContext) int32 { t1 := p1.StartTime t2 := p2.StartTime @@ -56,9 +57,53 @@ func runningTime(p1, p2 podinfo.PodContext) int32 { return -1 } -// classAndPriority compares pods by pod's ClassAndPriority -func classAndPriority(p1, p2 podinfo.PodContext) int32 { - return podinfo.CompareClassAndPriority(p1.ClassAndPriority, p2.ClassAndPriority) +// ComparePodQOSClass compares pods by pod's QOSClass +func ComparePodQOSClass(p1, p2 podinfo.PodContext) int32 { + return ComparePodQosClass(p1.QOSClass, p2.QOSClass) +} + +func ComparePriority(p1, p2 podinfo.PodContext) int32 { + if p1.Priority == p2.Priority { + return 0 + } else if p1.Priority < p2.Priority { + return -1 + } + return 1 +} + +// ComparePodQosClass compares Pod QOSClass +// Guaranteed > Burstable > BestEffort +func ComparePodQosClass(a v1.PodQOSClass, b v1.PodQOSClass) int32 { + switch b { + case v1.PodQOSGuaranteed: + if a == v1.PodQOSGuaranteed { + return 0 + } else { + return -1 + } + case v1.PodQOSBurstable: + if a == v1.PodQOSGuaranteed { + return 1 + } else if a == v1.PodQOSBurstable { + return 0 + } else { + return -1 + } + case v1.PodQOSBestEffort: + if (a == v1.PodQOSGuaranteed) || (a == v1.PodQOSBurstable) { + return 1 + } else if a == v1.PodQOSBestEffort { + return 0 + } else { + return -1 + } + default: + if (a == v1.PodQOSGuaranteed) || (a == v1.PodQOSBurstable) || (a == v1.PodQOSBestEffort) { + return 1 + } else { + return 0 + } + } } // Cmp compares p1 and p2 and returns: diff --git a/pkg/ensurance/executor/throttle.go b/pkg/ensurance/executor/throttle.go index fd55248fd..b324b00de 100644 --- a/pkg/ensurance/executor/throttle.go +++ b/pkg/ensurance/executor/throttle.go @@ -8,7 +8,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" execsort "github.com/gocrane/crane/pkg/ensurance/executor/sort" "github.com/gocrane/crane/pkg/known" "github.com/gocrane/crane/pkg/metrics" @@ -23,16 +23,16 @@ const ( type ThrottleExecutor struct { ThrottleDownPods ThrottlePods ThrottleUpPods ThrottlePods - // All metrics(not only metrics that can be quantified) metioned in triggerd NodeQOSEnsurancePolicy and their corresponding waterlines - ThrottleDownWaterLine WaterLines - ThrottleUpWaterLine WaterLines + // All metrics(not only metrics that can be quantified) metioned in triggerd NodeQOS and their corresponding watermarks + ThrottleDownWatermark Watermarks + ThrottleUpWatermark Watermarks } type ThrottlePods []podinfo.PodContext func (t ThrottlePods) Find(podTypes types.NamespacedName) int { for i, v := range t { - if v.PodKey == podTypes { + if v.Key == podTypes { return i } } @@ -68,38 +68,39 @@ func (t *ThrottleExecutor) Avoid(ctx *ExecuteContext) error { totalReleased := ReleaseResource{} /* The step to throttle: - 1. If ThrottleDownWaterLine has metrics that can't be quantified, select a throttleable metric which has the highest action priority, use its throttlefunc to throttle all ThrottleDownPods, then return - 2. Get the gaps between current usage and waterlines + 1. If ThrottleDownWatermark has metrics that can't be quantified, select a throttleable metric which has the highest action priority, use its throttlefunc to throttle all ThrottleDownPods, then return + 2. Get the gaps between current usage and watermarks 2.1 If there is a metric that can't get current usage, select a throttleable metric which has the highest action priority, use its throttlefunc to throttle all ThrottleDownPods, then return 2.2 Traverse metrics that can be quantified, if there is a gap for the metric, then sort candidate pods by its SortFunc if exists, otherwise use GeneralSorter by default. - Then throttle sorted pods one by one util there is no gap to waterline + Then throttle sorted pods one by one util there is no gap to watermark */ - metricsThrottleQuantified, MetricsNotThrottleQuantified := t.ThrottleDownWaterLine.DivideMetricsByThrottleQuantified() + metricsThrottleQuantified, MetricsNotThrottleQuantified := t.ThrottleDownWatermark.DivideMetricsByThrottleQuantified() // There is a metric that can't be ThrottleQuantified, so throttle all selected pods if len(MetricsNotThrottleQuantified) != 0 { klog.V(6).Info("ThrottleDown: There is a metric that can't be ThrottleQuantified") - highestPriorityMetric := t.ThrottleDownWaterLine.GetHighestPriorityThrottleAbleMetric() + highestPriorityMetric := t.ThrottleDownWatermark.GetHighestPriorityThrottleAbleMetric() if highestPriorityMetric != "" { klog.V(6).Infof("The highestPriorityMetric is %s", highestPriorityMetric) errPodKeys = t.throttlePods(ctx, &totalReleased, highestPriorityMetric) } } else { - ctx.ThrottoleDownGapToWaterLines, _, _ = buildGapToWaterLine(ctx.stateMap, *t, EvictExecutor{}, ctx.executeExcessPercent) + ctx.ToBeThrottleDown = calculateGaps(ctx.stateMap, t, nil, ctx.executeExcessPercent) - if ctx.ThrottoleDownGapToWaterLines.HasUsageMissedMetric() { + if ctx.ToBeThrottleDown.HasUsageMissedMetric() { klog.V(6).Info("There is a metric usage missed") - highestPriorityMetric := t.ThrottleDownWaterLine.GetHighestPriorityThrottleAbleMetric() + // todo remove highest priority + highestPriorityMetric := t.ThrottleDownWatermark.GetHighestPriorityThrottleAbleMetric() if highestPriorityMetric != "" { errPodKeys = t.throttlePods(ctx, &totalReleased, highestPriorityMetric) } } else { - // The metrics in ThrottoleDownGapToWaterLines are all in WaterLineMetricsCanBeQuantified and has current usage, then throttle precisely + // The metrics in ToBeThrottleDown are all in WatermarkMetricsCanBeQuantified and has current usage, then throttle precisely var released ReleaseResource for _, m := range metricsThrottleQuantified { klog.V(6).Infof("ThrottleDown precisely on metric %s", m) - if metricMap[m].SortAble { + if metricMap[m].Sortable { metricMap[m].SortFunc(t.ThrottleDownPods) } else { execsort.GeneralSorter(t.ThrottleDownPods) @@ -107,17 +108,17 @@ func (t *ThrottleExecutor) Avoid(ctx *ExecuteContext) error { klog.V(6).Info("After sort, the sequence to throttle is ") for _, pc := range t.ThrottleDownPods { - klog.V(6).Info(pc.PodKey.String(), pc.ContainerCPUUsages) + klog.V(6).Info(pc.Key.String(), pc.ContainerCPUUsages) } - for index := 0; !ctx.ThrottoleDownGapToWaterLines.TargetGapsRemoved(m) && index < len(t.ThrottleDownPods); index++ { - klog.V(6).Infof("For metric %s, there is still gap to waterlines: %f", m, ctx.ThrottoleDownGapToWaterLines[m]) + for index := 0; !ctx.ToBeThrottleDown.TargetGapsRemoved(m) && index < len(t.ThrottleDownPods); index++ { + klog.V(6).Infof("For metric %s, there is still gap to watermarks: %f", m, ctx.ToBeThrottleDown[m]) errKeys, released = metricMap[m].ThrottleFunc(ctx, index, t.ThrottleDownPods, &totalReleased) - klog.V(6).Infof("ThrottleDown pods %s, released %f resource", t.ThrottleDownPods[index].PodKey, released[m]) + klog.V(6).Infof("ThrottleDown pods %s, released %f resource", t.ThrottleDownPods[index].Key, released[m]) errPodKeys = append(errPodKeys, errKeys...) - ctx.ThrottoleDownGapToWaterLines[m] -= released[m] + ctx.ToBeThrottleDown[m] -= released[m] } } } @@ -130,7 +131,7 @@ func (t *ThrottleExecutor) Avoid(ctx *ExecuteContext) error { return nil } -func (t *ThrottleExecutor) throttlePods(ctx *ExecuteContext, totalReleasedResource *ReleaseResource, m WaterLineMetric) (errPodKeys []string) { +func (t *ThrottleExecutor) throttlePods(ctx *ExecuteContext, totalReleasedResource *ReleaseResource, m WatermarkMetric) (errPodKeys []string) { for i := range t.ThrottleDownPods { errKeys, _ := metricMap[m].ThrottleFunc(ctx, i, t.ThrottleDownPods, totalReleasedResource) errPodKeys = append(errPodKeys, errKeys...) @@ -143,7 +144,7 @@ func (t *ThrottleExecutor) Restore(ctx *ExecuteContext) error { metrics.UpdateLastTimeWithSubComponent(string(known.ModuleActionExecutor), string(metrics.SubComponentThrottle), metrics.StepRestore, start) defer metrics.UpdateDurationFromStartWithSubComponent(string(known.ModuleActionExecutor), string(metrics.SubComponentThrottle), metrics.StepRestore, start) - klog.V(6).Info("ThrottleExecutor restore, %v", *t) + klog.V(6).Infof("ThrottleExecutor restore, %v", *t) if len(t.ThrottleUpPods) == 0 { metrics.UpdateExecutorStatus(metrics.SubComponentThrottle, metrics.StepRestore, 0) @@ -158,38 +159,38 @@ func (t *ThrottleExecutor) Restore(ctx *ExecuteContext) error { totalReleased := ReleaseResource{} /* The step to restore: - 1. If ThrottleUpWaterLine has metrics that can't be quantified, select a throttleable metric which has the highest action priority, use its RestoreFunc to restore all ThrottleUpPods, then return - 2. Get the gaps between current usage and waterlines + 1. If ThrottleUpWatermark has metrics that can't be quantified, select a throttleable metric which has the highest action priority, use its RestoreFunc to restore all ThrottleUpPods, then return + 2. Get the gaps between current usage and watermarks 2.1 If there is a metric that can't get current usage, select a throttleable metric which has the highest action priority, use its RestoreFunc to restore all ThrottleUpPods, then return 2.2 Traverse metrics that can be quantified, if there is a gap for the metric, then sort candidate pods by its SortFunc if exists, otherwise use GeneralSorter by default. - Then restore sorted pods one by one util there is no gap to waterline + Then restore sorted pods one by one util there is no gap to watermark */ - metricsThrottleQuantified, MetricsNotThrottleQuantified := t.ThrottleUpWaterLine.DivideMetricsByThrottleQuantified() + metricsThrottleQuantified, MetricsNotThrottleQuantified := t.ThrottleUpWatermark.DivideMetricsByThrottleQuantified() // There is a metric that can't be ThrottleQuantified, so restore all selected pods if len(MetricsNotThrottleQuantified) != 0 { klog.V(6).Info("ThrottleUp: There is a metric that can't be ThrottleQuantified") - highestPrioriyMetric := t.ThrottleUpWaterLine.GetHighestPriorityThrottleAbleMetric() + highestPrioriyMetric := t.ThrottleUpWatermark.GetHighestPriorityThrottleAbleMetric() if highestPrioriyMetric != "" { klog.V(6).Infof("The highestPrioriyMetric is %s", highestPrioriyMetric) errPodKeys = t.restorePods(ctx, &totalReleased, highestPrioriyMetric) } } else { - _, ctx.ThrottoleUpGapToWaterLines, _ = buildGapToWaterLine(ctx.stateMap, *t, EvictExecutor{}, ctx.executeExcessPercent) + ctx.ToBeThrottleUp = calculateGaps(ctx.stateMap, t, nil, ctx.executeExcessPercent) - if ctx.ThrottoleUpGapToWaterLines.HasUsageMissedMetric() { + if ctx.ToBeThrottleUp.HasUsageMissedMetric() { klog.V(6).Info("There is a metric usage missed") - highestPrioriyMetric := t.ThrottleUpWaterLine.GetHighestPriorityThrottleAbleMetric() + highestPrioriyMetric := t.ThrottleUpWatermark.GetHighestPriorityThrottleAbleMetric() if highestPrioriyMetric != "" { errPodKeys = t.restorePods(ctx, &totalReleased, highestPrioriyMetric) } } else { - // The metrics in ThrottoleUpGapToWaterLines are all in WaterLineMetricsCanBeQuantified and has current usage, then throttle precisely + // The metrics in ToBeThrottleUp are all in WatermarkMetricsCanBeQuantified and has current usage, then throttle precisely var released ReleaseResource for _, m := range metricsThrottleQuantified { klog.V(6).Infof("ThrottleUp precisely on metric %s", m) - if metricMap[m].SortAble { + if metricMap[m].Sortable { metricMap[m].SortFunc(t.ThrottleUpPods) } else { execsort.GeneralSorter(t.ThrottleUpPods) @@ -198,17 +199,17 @@ func (t *ThrottleExecutor) Restore(ctx *ExecuteContext) error { klog.V(6).Info("After sort, the sequence to throttle is ") for _, pc := range t.ThrottleUpPods { - klog.V(6).Info(pc.PodKey.String()) + klog.V(6).Info(pc.Key.String()) } - for index := 0; !ctx.ThrottoleUpGapToWaterLines.TargetGapsRemoved(m) && index < len(t.ThrottleUpPods); index++ { - klog.V(6).Infof("For metric %s, there is still gap to waterlines: %f", m, ctx.ThrottoleUpGapToWaterLines[m]) + for index := 0; !ctx.ToBeThrottleUp.TargetGapsRemoved(m) && index < len(t.ThrottleUpPods); index++ { + klog.V(6).Infof("For metric %s, there is still gap to watermarks: %f", m, ctx.ToBeThrottleUp[m]) errKeys, released = metricMap[m].RestoreFunc(ctx, index, t.ThrottleUpPods, &totalReleased) - klog.V(6).Infof("ThrottleUp pods %s, released %f resource", t.ThrottleUpPods[index].PodKey, released[m]) + klog.V(6).Infof("ThrottleUp pods %s, released %f resource", t.ThrottleUpPods[index].Key, released[m]) errPodKeys = append(errPodKeys, errKeys...) - ctx.ThrottoleUpGapToWaterLines[m] -= released[m] + ctx.ToBeThrottleUp[m] -= released[m] } } } @@ -221,7 +222,7 @@ func (t *ThrottleExecutor) Restore(ctx *ExecuteContext) error { return nil } -func (t *ThrottleExecutor) restorePods(ctx *ExecuteContext, totalReleasedResource *ReleaseResource, m WaterLineMetric) (errPodKeys []string) { +func (t *ThrottleExecutor) restorePods(ctx *ExecuteContext, totalReleasedResource *ReleaseResource, m WatermarkMetric) (errPodKeys []string) { for i := range t.ThrottleUpPods { errKeys, _ := metricMap[m].RestoreFunc(ctx, i, t.ThrottleDownPods, totalReleasedResource) errPodKeys = append(errPodKeys, errKeys...) diff --git a/pkg/ensurance/executor/waterline.go b/pkg/ensurance/executor/waterline.go deleted file mode 100644 index 56dfe29db..000000000 --- a/pkg/ensurance/executor/waterline.go +++ /dev/null @@ -1,244 +0,0 @@ -package executor - -import ( - "math" - "reflect" - - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/klog/v2" - - "github.com/gocrane/crane/pkg/common" - "github.com/gocrane/crane/pkg/ensurance/collector/types" -) - -// Metrics that can be measured for waterLine -// Should be consistent with metrics in collector/types/types.go -type WaterLineMetric string - -// Be consistent with metrics in collector/types/types.go -const ( - CpuUsage = WaterLineMetric(types.MetricNameCpuTotalUsage) - MemUsage = WaterLineMetric(types.MetricNameMemoryTotalUsage) -) - -const ( - // We can't get current use, so can't do actions precisely, just evict every evictedPod - MissedCurrentUsage float64 = math.MaxFloat64 -) - -// An WaterLine is a min-heap of Quantity. The values come from each objectiveEnsurance.metricRule.value -type WaterLine []resource.Quantity - -func (w WaterLine) Len() int { - return len(w) -} - -func (w WaterLine) Swap(i, j int) { - w[i], w[j] = w[j], w[i] -} - -func (w *WaterLine) Push(x interface{}) { - *w = append(*w, x.(resource.Quantity)) -} - -func (w *WaterLine) Pop() interface{} { - old := *w - n := len(old) - x := old[n-1] - *w = old[0 : n-1] - return x -} - -func (w *WaterLine) PopSmallest() *resource.Quantity { - wl := *w - return &wl[0] -} - -func (w WaterLine) Less(i, j int) bool { - cmp := w[i].Cmp(w[j]) - if cmp == -1 { - return true - } - return false -} - -func (w WaterLine) String() string { - str := "" - for i := 0; i < w.Len(); i++ { - str += w[i].String() - str += " " - } - return str -} - -// WaterLines 's key is the metric name, value is waterline which get from each objectiveEnsurance.metricRule.value -type WaterLines map[WaterLineMetric]*WaterLine - -// DivideMetricsByThrottleQuantified divide metrics by whether metrics can be throttleQuantified -func (e WaterLines) DivideMetricsByThrottleQuantified() (MetricsThrottleQuantified []WaterLineMetric, MetricsNotThrottleQuantified []WaterLineMetric) { - for m := range e { - if metricMap[m].ThrottleQuantified == true { - MetricsThrottleQuantified = append(MetricsThrottleQuantified, m) - } else { - MetricsNotThrottleQuantified = append(MetricsNotThrottleQuantified, m) - } - } - return -} - -// DivideMetricsByEvictQuantified divide metrics in waterlines into can be EvictQuantified and can not be EvictQuantified -func (e WaterLines) DivideMetricsByEvictQuantified() (metricsEvictQuantified []WaterLineMetric, metricsNotEvictQuantified []WaterLineMetric) { - for m := range e { - if metricMap[m].EvictQuantified == true { - metricsEvictQuantified = append(metricsEvictQuantified, m) - } else { - metricsNotEvictQuantified = append(metricsNotEvictQuantified, m) - } - } - return -} - -// GetHighestPriorityThrottleAbleMetric get the highest priority in metrics from waterlines -func (e WaterLines) GetHighestPriorityThrottleAbleMetric() (highestPrioriyMetric WaterLineMetric) { - highestActionPriority := 0 - for m := range e { - if metricMap[m].ThrottleAble == true { - if metricMap[m].ActionPriority >= highestActionPriority { - highestPrioriyMetric = m - highestActionPriority = metricMap[m].ActionPriority - } - } - } - return -} - -// GetHighestPriorityEvictAbleMetric get the highest priority in metrics that can be EvictAble -func (e WaterLines) GetHighestPriorityEvictAbleMetric() (highestPrioriyMetric WaterLineMetric) { - highestActionPriority := 0 - for m := range e { - if metricMap[m].EvictAble == true { - if metricMap[m].ActionPriority >= highestActionPriority { - highestPrioriyMetric = m - highestActionPriority = metricMap[m].ActionPriority - } - } - } - return -} - -// GapToWaterLines's key is metric name, value is the difference between usage and the smallest waterline -type GapToWaterLines map[WaterLineMetric]float64 - -// Only calculate gap for metrics that can be quantified -func buildGapToWaterLine(stateMap map[string][]common.TimeSeries, - throttleExecutor ThrottleExecutor, evictExecutor EvictExecutor, executeExcessPercent float64) ( - throttleDownGapToWaterLines, throttleUpGapToWaterLines, eviceGapToWaterLines GapToWaterLines) { - - throttleDownGapToWaterLines, throttleUpGapToWaterLines, eviceGapToWaterLines = make(map[WaterLineMetric]float64), make(map[WaterLineMetric]float64), make(map[WaterLineMetric]float64) - - if !reflect.DeepEqual(evictExecutor, EvictExecutor{}) { - // Traverse EvictAbleMetric but not evictExecutor.EvictWaterLine can make it easier when users use the wrong metric name in NEP, cause this limit metrics - // must come from EvictAbleMetrics - for _, m := range GetEvictAbleMetricName() { - // Get the series for each metric - series, ok := stateMap[string(m)] - if !ok { - klog.Warningf("BuildEvictWaterLineGap: Evict Metric %s not found from collector stateMap", string(m)) - // Can't get current usage, so can not do actions precisely, just evict every evictedPod; - eviceGapToWaterLines[m] = MissedCurrentUsage - continue - } - - // Find the biggest used value - var maxUsed float64 - if series[0].Samples[0].Value > maxUsed { - maxUsed = series[0].Samples[0].Value - } - - // Get the waterLine for each metric in WaterLineMetricsCanBeQuantified - evictWaterLine, evictExist := evictExecutor.EvictWaterLine[m] - - // If metric not exist in EvictWaterLine, eviceGapToWaterLines of metric will can't be calculated - if !evictExist { - delete(eviceGapToWaterLines, m) - } else { - klog.V(6).Infof("BuildEvictWaterLineGap: For metrics %s, maxUsed is %f, waterline is %f", m, maxUsed, float64(evictWaterLine.PopSmallest().Value())) - eviceGapToWaterLines[m] = (1 + executeExcessPercent) * (maxUsed - float64(evictWaterLine.PopSmallest().Value())) - } - } - } - - if !reflect.DeepEqual(throttleExecutor, ThrottleExecutor{}) { - // Traverse ThrottleAbleMetricName but not throttleExecutor.ThrottleDownWaterLine can make it easier when users use the wrong metric name in NEP, cause this limit metrics - // must come from ThrottleAbleMetrics - for _, m := range GetThrottleAbleMetricName() { - // Get the series for each metric - series, ok := stateMap[string(m)] - if !ok { - klog.Warningf("BuildThrottleWaterLineGap: Metric %s not found from collector stateMap", string(m)) - // Can't get current usage, so can not do actions precisely, just evict every evictedPod; - throttleDownGapToWaterLines[m] = MissedCurrentUsage - throttleUpGapToWaterLines[m] = MissedCurrentUsage - continue - } - - // Find the biggest used value - var maxUsed float64 - if series[0].Samples[0].Value > maxUsed { - maxUsed = series[0].Samples[0].Value - } - - // Get the waterLine for each metric in WaterLineMetricsCanBeQuantified - throttleDownWaterLine, throttleDownExist := throttleExecutor.ThrottleDownWaterLine[m] - throttleUpWaterLine, throttleUpExist := throttleExecutor.ThrottleUpWaterLine[m] - - // If a metric does not exist in ThrottleDownWaterLine, throttleDownGapToWaterLines of this metric will can't be calculated - if !throttleDownExist { - delete(throttleDownGapToWaterLines, m) - } else { - klog.V(6).Infof("BuildThrottleDownWaterLineGap: For metrics %s, maxUsed is %f, waterline is %f", m, maxUsed, float64(throttleDownWaterLine.PopSmallest().Value())) - throttleDownGapToWaterLines[m] = (1 + executeExcessPercent) * (maxUsed - float64(throttleDownWaterLine.PopSmallest().Value())) - } - - // If metric not exist in ThrottleUpWaterLine, throttleUpGapToWaterLines of metric will can't be calculated - if !throttleUpExist { - delete(throttleUpGapToWaterLines, m) - } else { - klog.V(6).Infof("BuildThrottleUpWaterLineGap: For metrics %s, maxUsed is %f, waterline is %f", m, maxUsed, float64(throttleUpWaterLine.PopSmallest().Value())) - // Attention: different with throttleDown and evict, use waterline - used - throttleUpGapToWaterLines[m] = (1 + executeExcessPercent) * (float64(throttleUpWaterLine.PopSmallest().Value()) - maxUsed) - } - } - } - return -} - -// Whether no gaps in GapToWaterLines -func (g GapToWaterLines) GapsAllRemoved() bool { - for _, v := range g { - if v > 0 { - return false - } - } - return true -} - -// For a specified metric in GapToWaterLines, whether there still has gap -func (g GapToWaterLines) TargetGapsRemoved(metric WaterLineMetric) bool { - val, ok := g[metric] - if !ok || val <= 0 { - return true - } - return false -} - -// Whether there is a metric that can't get usage in GapToWaterLines -func (g GapToWaterLines) HasUsageMissedMetric() bool { - for m, v := range g { - if v == MissedCurrentUsage { - klog.V(6).Infof("Metric %s usage missed", m) - return true - } - } - return false -} diff --git a/pkg/ensurance/executor/watermark.go b/pkg/ensurance/executor/watermark.go new file mode 100644 index 000000000..a42008bfe --- /dev/null +++ b/pkg/ensurance/executor/watermark.go @@ -0,0 +1,243 @@ +package executor + +import ( + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog/v2" + "math" + + "github.com/gocrane/crane/pkg/common" + "github.com/gocrane/crane/pkg/ensurance/collector/types" +) + +// WatermarkMetric defines metrics that can be measured for watermark +// Should be consistent with metrics in collector/types/types.go +type WatermarkMetric string + +// Be consistent with metrics in collector/types/types.go +const ( + CpuUsage = WatermarkMetric(types.MetricNameCpuTotalUsage) + MemUsage = WatermarkMetric(types.MetricNameMemoryTotalUsage) +) + +const ( + // We can't get current use, so can't do actions precisely, just evict every evictedPod + maxFloat float64 = math.MaxFloat64 +) + +// An Watermark is a min-heap of Quantity. The values come from each objectiveEnsurance.metricRule.value +type Watermark []resource.Quantity + +func (w Watermark) Len() int { + return len(w) +} + +func (w Watermark) Swap(i, j int) { + w[i], w[j] = w[j], w[i] +} + +func (w *Watermark) Push(x interface{}) { + *w = append(*w, x.(resource.Quantity)) +} + +func (w *Watermark) Pop() interface{} { + old := *w + n := len(old) + x := old[n-1] + *w = old[0 : n-1] + return x +} + +func (w *Watermark) PopSmallest() *resource.Quantity { + wl := *w + return &wl[0] +} + +func (w Watermark) Less(i, j int) bool { + cmp := w[i].Cmp(w[j]) + if cmp == -1 { + return true + } + return false +} + +func (w Watermark) String() string { + str := "" + for i := 0; i < w.Len(); i++ { + str += w[i].String() + str += " " + } + return str +} + +// Watermarks 's key is the metric name, value is watermark which get from each objectiveEnsurance.metricRule.value +type Watermarks map[WatermarkMetric]*Watermark + +// DivideMetricsByThrottleQuantified divide metrics by whether metrics can be throttleQuantified +func (e Watermarks) DivideMetricsByThrottleQuantified() (MetricsThrottleQuantified []WatermarkMetric, MetricsNotThrottleQuantified []WatermarkMetric) { + for m := range e { + if metricMap[m].ThrottleQuantified == true { + MetricsThrottleQuantified = append(MetricsThrottleQuantified, m) + } else { + MetricsNotThrottleQuantified = append(MetricsNotThrottleQuantified, m) + } + } + return +} + +// DivideMetricsByEvictQuantified divide metrics in watermarks into can be EvictQuantified and can not be EvictQuantified +func (e Watermarks) DivideMetricsByEvictQuantified() (quantified []WatermarkMetric, notQuantified []WatermarkMetric) { + for m := range e { + if metricMap[m].EvictQuantified == true { + quantified = append(quantified, m) + } else { + notQuantified = append(notQuantified, m) + } + } + return +} + +// GetHighestPriorityThrottleAbleMetric get the highest priority in metrics from watermarks +func (e Watermarks) GetHighestPriorityThrottleAbleMetric() (highestPrioriyMetric WatermarkMetric) { + highestActionPriority := 0 + for m := range e { + if metricMap[m].Throttleable == true { + if metricMap[m].ActionPriority >= highestActionPriority { + highestPrioriyMetric = m + highestActionPriority = metricMap[m].ActionPriority + } + } + } + return +} + +// GetHighestPriorityEvictableMetric get the highest priority in metrics that can be Evictable +func (e Watermarks) GetHighestPriorityEvictableMetric() (highestPrioriyMetric WatermarkMetric) { + highestActionPriority := 0 + for m := range e { + if metricMap[m].Evictable == true { + if metricMap[m].ActionPriority >= highestActionPriority { + highestPrioriyMetric = m + highestActionPriority = metricMap[m].ActionPriority + } + } + } + return +} + +// Gaps key is metric name, value is the difference between usage and the smallest watermark +type Gaps map[WatermarkMetric]float64 + +// Only calculate gap for metrics that can be quantified +func calculateGaps(stateMap map[string][]common.TimeSeries, + throttleExecutor *ThrottleExecutor, evictExecutor *EvictExecutor, executeExcessPercent float64) Gaps { + result := map[WatermarkMetric]float64{} + //throttleDownGapToWatermarks, throttleUpGapToWatermarks, eviceGapToWatermarks = make(map[WatermarkMetric]float64), make(map[WatermarkMetric]float64), make(map[WatermarkMetric]float64) + + if evictExecutor != nil { + // Traverse EvictableMetric but not evictExecutor.EvictWatermark can make it easier when users use the wrong metric name in NodeQOS, cause this limit metrics + // must come from EvictableMetrics + for _, m := range metricMap { + if !m.Evictable { + continue + } + // Get the series for each metric + series, ok := stateMap[string(m.Name)] + if !ok { + klog.Warningf("BuildEvictWatermarkGap: Evict Metric %s not found from collector stateMap", string(m.Name)) + // Can't get current usage, so can not do actions precisely, just evict every evictedPod; + result[m.Name] = maxFloat + continue + } + + // Find the biggest used value + var maxUsed float64 + if series[0].Samples[0].Value > maxUsed { + maxUsed = series[0].Samples[0].Value + } + + // Get the watermark for each metric cannot be quantified + evictWatermark, evictExist := evictExecutor.EvictWatermark[m.Name] + // If metric not exist in EvictWatermark, gap can't be calculated + if !evictExist { + delete(result, m.Name) + } else { + klog.V(6).Infof("BuildEvictWatermarkGap: For metrics %+v, maxUsed is %f, watermark is %f", m, maxUsed, float64(evictWatermark.PopSmallest().Value())) + result[m.Name] = (1 + executeExcessPercent) * (maxUsed - float64(evictWatermark.PopSmallest().Value())) + } + } + } else if throttleExecutor != nil { + // Traverse ThrottleAbleMetricName but not throttleExecutor.ThrottleDownWatermark can make it easier when users use the wrong metric name in NEP, cause this limit metrics + // must come from ThrottleAbleMetrics + for _, m := range metricMap { + if !m.Throttleable { + continue + } + // Get the series for each metric + series, ok := stateMap[string(m.Name)] + if !ok { + klog.Warningf("BuildThrottleWatermarkGap: Metric %s not found from collector stateMap", string(m.Name)) + // Can't get current usage, so can not do actions precisely, just evict every evictedPod; + result[m.Name] = maxFloat + continue + } + + // Find the biggest used value + var maxUsed float64 + if series[0].Samples[0].Value > maxUsed { + maxUsed = series[0].Samples[0].Value + } + + // Get the watermark for each metric in WatermarkMetricsCanBeQuantified + throttleDownWatermark, throttleDownExist := throttleExecutor.ThrottleDownWatermark[m.Name] + throttleUpWatermark, throttleUpExist := throttleExecutor.ThrottleUpWatermark[m.Name] + + // If a metric does not exist in ThrottleDownWatermark, throttleDownGapToWatermarks of this metric will can't be calculated + if !throttleDownExist { + delete(result, m.Name) + } else { + klog.V(6).Infof("BuildThrottleDownWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleDownWatermark.PopSmallest().Value())) + result[m.Name] = (1 + executeExcessPercent) * (maxUsed - float64(throttleDownWatermark.PopSmallest().Value())) + } + + // If metric not exist in ThrottleUpWatermark, throttleUpGapToWatermarks of metric will can't be calculated + if !throttleUpExist { + delete(result, m.Name) + } else { + klog.V(6).Infof("BuildThrottleUpWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleUpWatermark.PopSmallest().Value())) + // Attention: different with throttleDown and evict, use watermark - used + result[m.Name] = (1 + executeExcessPercent) * (float64(throttleUpWatermark.PopSmallest().Value()) - maxUsed) + } + } + } + return result +} + +// Whether no gaps in Gaps +func (g Gaps) GapsAllRemoved() bool { + for _, v := range g { + if v > 0 { + return false + } + } + return true +} + +// For a specified metric in Gaps, whether there still has gap +func (g Gaps) TargetGapsRemoved(metric WatermarkMetric) bool { + val, ok := g[metric] + if !ok || val <= 0 { + return true + } + return false +} + +// Whether there is a metric that can't get usage in Gaps +func (g Gaps) HasUsageMissedMetric() bool { + for m, v := range g { + if v == maxFloat { + klog.V(6).Infof("Metric %s usage missed", m) + return true + } + } + return false +} diff --git a/pkg/ensurance/executor/waterline_test.go b/pkg/ensurance/executor/watermark_test.go similarity index 93% rename from pkg/ensurance/executor/waterline_test.go rename to pkg/ensurance/executor/watermark_test.go index 970560cb6..d4f7682d5 100644 --- a/pkg/ensurance/executor/waterline_test.go +++ b/pkg/ensurance/executor/watermark_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func (w WaterLine) verify(t *testing.T, i int) { +func (w Watermark) verify(t *testing.T, i int) { t.Helper() n := w.Len() j1 := 2*i + 1 @@ -33,7 +33,7 @@ func (w WaterLine) verify(t *testing.T, i int) { // TestPopSmallest make sure that we can get the smallest value func TestPopSmallest(t *testing.T) { - h := WaterLine{} + h := Watermark{} for i := 20; i > 0; i-- { heap.Push(&h, resource.MustParse(strconv.Itoa(i)+"m")) diff --git a/pkg/known/label.go b/pkg/known/label.go index 8aec55e3f..8695d5130 100644 --- a/pkg/known/label.go +++ b/pkg/known/label.go @@ -6,8 +6,8 @@ const ( ) const ( - EnsuranceAnalyzedPressureTaintKey = "ensurance.crane.io/analyzed-pressure" - EnsuranceAnalyzedPressureConditionKey = "analyzed-pressure" + EnsuranceAnalyzedPressureTaintKey = "interference.crane.io" + EnsuranceAnalyzedPressureConditionKey = "interference-identified" ) const ( diff --git a/pkg/known/types.go b/pkg/known/types.go index b93d88071..8a9ad35b7 100644 --- a/pkg/known/types.go +++ b/pkg/known/types.go @@ -3,7 +3,7 @@ package known type Module string const ( - ModuleAnormalyAnalyzer Module = "AnormalyAnalyzer" + ModuleAnomalyAnalyzer Module = "AnomalyAnalyzer" ModuleStateCollector Module = "StateCollector" ModuleActionExecutor Module = "ActionExecutor" ModuleNodeResourceManager Module = "ModuleNodeResourceManager" diff --git a/pkg/metrics/ensuarance.go b/pkg/metrics/ensuarance.go index 2d0b662e2..e008c5f38 100644 --- a/pkg/metrics/ensuarance.go +++ b/pkg/metrics/ensuarance.go @@ -119,7 +119,7 @@ var ( Namespace: CraneNamespace, Subsystem: CraneAgentSubsystem, Name: AnalyzerStatus, - Help: "Status of anormaly analyzer.", + Help: "Status of anomaly analyzer.", StabilityLevel: k8smetrics.ALPHA, }, []string{"key", "type"}, ) @@ -129,7 +129,7 @@ var ( Namespace: CraneNamespace, Subsystem: CraneAgentSubsystem, Name: AnalyzerStatusTotal, - Help: "The times of nep rule triggered/restored.", + Help: "The times of NodeQOS rule triggered/restored.", StabilityLevel: k8smetrics.ALPHA, }, []string{"key", "type"}, ) diff --git a/pkg/resource/node_resource_manager.go b/pkg/resource/node_resource_manager.go index 09e67f97d..2995ccb9f 100644 --- a/pkg/resource/node_resource_manager.go +++ b/pkg/resource/node_resource_manager.go @@ -130,7 +130,6 @@ func (o *NodeResourceManager) Run(stop <-chan struct{}) { case state := <-o.stateChann: o.state = state o.lastStateTime = time.Now() - case <-tspUpdateTicker.C: start := time.Now() metrics.UpdateLastTime(string(known.ModuleNodeResourceManager), metrics.StepUpdateNodeResource, start) o.UpdateNodeResource() @@ -158,10 +157,10 @@ func (o *NodeResourceManager) UpdateNodeResource() { // Update Node status extend-resource info // TODO fix: strategic merge patch kubernetes if _, err := o.client.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{}); err != nil { - klog.Errorf("Failed to update node %s's status extend-resource, %v", nodeCopy.Name, err) + klog.Errorf("Failed to update node %s extended resource, %v", nodeCopy.Name, err) return } - klog.V(4).Infof("Update Node %s Extend Resource Success", node.Name) + klog.V(2).Infof("Update node %s extended resource successfully", node.Name) o.recorder.Event(node, v1.EventTypeNormal, "UpdateNode", generateUpdateEventMessage(resourcesFrom)) } } @@ -331,7 +330,7 @@ func (o *NodeResourceManager) GetCpuCoreCanNotBeReclaimedFromLocal() float64 { klog.V(4).Infof("Can't get %s from NodeResourceManager local state", types.MetricNameExclusiveCPUIdle) } - klog.V(6).Infof("nodeCpuUsageTotal: %s, exclusiveCPUIdle: %s, extResContainerCpuUsageTotal: %s", nodeCpuUsageTotal, exclusiveCPUIdle, extResContainerCpuUsageTotal) + klog.V(6).Infof("nodeCpuUsageTotal: %f, exclusiveCPUIdle: %f, extResContainerCpuUsageTotal: %f", nodeCpuUsageTotal, exclusiveCPUIdle, extResContainerCpuUsageTotal) // 1. Exclusive tethered CPU cannot be reclaimed even if the free part is free, so add the exclusive CPUIdle to the CanNotBeReclaimed CPU // 2. The CPU used by extRes-container needs to be reclaimed, otherwise it will be double-counted due to the allotted mechanism of k8s, so the extResContainerCpuUsageTotal is subtracted from the CanNotBeReclaimedCpu diff --git a/pkg/resource/pod_resource_manger.go b/pkg/resource/pod_resource_manger.go index 03dd07d00..d694b8b85 100644 --- a/pkg/resource/pod_resource_manger.go +++ b/pkg/resource/pod_resource_manger.go @@ -20,7 +20,7 @@ import ( "github.com/gocrane/crane/pkg/ensurance/collector/cadvisor" stypes "github.com/gocrane/crane/pkg/ensurance/collector/types" "github.com/gocrane/crane/pkg/ensurance/executor" - podinfo "github.com/gocrane/crane/pkg/ensurance/executor/pod-info" + podinfo "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" cgrpc "github.com/gocrane/crane/pkg/ensurance/grpc" cruntime "github.com/gocrane/crane/pkg/ensurance/runtime" "github.com/gocrane/crane/pkg/known" diff --git a/pkg/utils/node.go b/pkg/utils/node.go index c42defa5f..1a959a9b9 100644 --- a/pkg/utils/node.go +++ b/pkg/utils/node.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "k8s.io/klog/v2" "golang.org/x/net/context" v1 "k8s.io/api/core/v1" @@ -9,7 +10,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" corelisters "k8s.io/client-go/listers/core/v1" - "k8s.io/klog/v2" ) const defaultRetryTimes = 3 @@ -25,6 +25,7 @@ func UpdateNodeConditionsStatues(client clientset.Interface, nodeLister corelist updateNode, needUpdate := updateNodeConditions(node, condition) if needUpdate { + klog.Warningf("Updating node condition %v", condition) if updateNode, err = client.CoreV1().Nodes().UpdateStatus(context.Background(), updateNode, metav1.UpdateOptions{}); err != nil { if errors.IsConflict(err) { continue @@ -84,7 +85,7 @@ func UpdateNodeTaints(client clientset.Interface, nodeLister corelisters.NodeLis return updateNode, nil } - return nil, fmt.Errorf("update node failed, conflict too more times") + return nil, fmt.Errorf("failed to update node taints after %d retries", GetUint64withDefault(retry, defaultRetryTimes)) } func updateNodeTaints(node *v1.Node, taint v1.Taint) (*v1.Node, bool) { @@ -107,8 +108,6 @@ func updateNodeTaints(node *v1.Node, taint v1.Taint) (*v1.Node, bool) { } func RemoveNodeTaints(client clientset.Interface, nodeLister corelisters.NodeLister, nodeName string, taint v1.Taint, retry *uint64) (*v1.Node, error) { - klog.V(4).Infof("RemoveNodeTaints %v", taint) - for i := uint64(0); i < GetUint64withDefault(retry, defaultRetryTimes); i++ { node, err := nodeLister.Get(nodeName) if err != nil { @@ -117,6 +116,7 @@ func RemoveNodeTaints(client clientset.Interface, nodeLister corelisters.NodeLis updateNode, needUpdate := removeNodeTaints(node, taint) if needUpdate { + klog.V(4).Infof("Removing node taint %v", taint) if updateNode, err = client.CoreV1().Nodes().Update(context.Background(), updateNode, metav1.UpdateOptions{}); err != nil { if errors.IsConflict(err) { continue @@ -136,19 +136,19 @@ func removeNodeTaints(node *v1.Node, taint v1.Taint) (*v1.Node, bool) { updatedNode := node.DeepCopy() - var bFound = false + var foundTaint = false var taints []v1.Taint for _, t := range updatedNode.Spec.Taints { - if t.Key == taint.Key { - bFound = true + if t.Key == taint.Key && t.Effect == taint.Effect { + foundTaint = true } else { taints = append(taints, t) } } // found the taint, remove it - if bFound { + if foundTaint { updatedNode.Spec.Taints = taints return updatedNode, true } diff --git a/pkg/utils/pod.go b/pkg/utils/pod.go index 457328f2e..a6505e204 100644 --- a/pkg/utils/pod.go +++ b/pkg/utils/pod.go @@ -216,21 +216,15 @@ func GetContainerIdFromPod(pod *v1.Pod, containerName string) string { return "" } -// Whether pod uses ext resource ext-resource.node.gocrane.io, and value it uses -func ExtResourceAllocated(pod *v1.Pod, resName v1.ResourceName) (hasExtResource bool, extCpuLimit, extCpuRequest int64) { +// GetElasticResourceLimit sum all containers resources limit for gocrane.io/resource +// As extended resource is not over committable resource, so request = limit +func GetElasticResourceLimit(pod *v1.Pod, resName v1.ResourceName) (amount int64) { resPrefix := fmt.Sprintf(ExtResourcePrefixFormat, resName) for i := range pod.Spec.Containers { container := pod.Spec.Containers[i] for res, val := range container.Resources.Limits { if strings.HasPrefix(res.String(), resPrefix) { - hasExtResource = true - extCpuLimit += val.MilliValue() - } - } - for res, val := range container.Resources.Requests { - if strings.HasPrefix(res.String(), resPrefix) { - hasExtResource = true - extCpuRequest += val.MilliValue() + amount += val.MilliValue() } } } diff --git a/pkg/webhooks/ensurance/validating.go b/pkg/webhooks/ensurance/validating.go index a7d39dd3a..723a892fb 100644 --- a/pkg/webhooks/ensurance/validating.go +++ b/pkg/webhooks/ensurance/validating.go @@ -20,33 +20,33 @@ import ( "github.com/gocrane/crane/pkg/known" ) -type NepValidationAdmission struct { +type NodeQOSValidationAdmission struct { } type ActionValidationAdmission struct { } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (p *NepValidationAdmission) ValidateCreate(ctx context.Context, req runtime.Object) error { +func (p *NodeQOSValidationAdmission) ValidateCreate(ctx context.Context, req runtime.Object) error { - nep, ok := req.(*ensuranceapi.NodeQOSEnsurancePolicy) + nodeQOS, ok := req.(*ensuranceapi.NodeQOS) if !ok { - return fmt.Errorf("req can not convert to NodeQOSEnsurancePolicy") + return fmt.Errorf("req can not convert to NodeQOS") } - allErrs := genericvalidation.ValidateObjectMeta(&nep.ObjectMeta, false, genericvalidation.NameIsDNS1035Label, field.NewPath("metadata")) + allErrs := genericvalidation.ValidateObjectMeta(&nodeQOS.ObjectMeta, false, genericvalidation.NameIsDNS1035Label, field.NewPath("metadata")) - if nep.Spec.Selector != nil { - allErrs = append(allErrs, metavalidation.ValidateLabelSelector(nep.Spec.Selector, field.NewPath("spec").Child("selector"))...) + if nodeQOS.Spec.Selector != nil { + allErrs = append(allErrs, metavalidation.ValidateLabelSelector(nodeQOS.Spec.Selector, field.NewPath("spec").Child("selector"))...) } - allErrs = append(allErrs, validateNodeQualityProbe(nep.Spec.NodeQualityProbe, field.NewPath("nodeQualityProbe"))...) + allErrs = append(allErrs, validateNodeQualityProbe(nodeQOS.Spec.NodeQualityProbe, field.NewPath("nodeQualityProbe"))...) var httpGetEnable bool - if nep.Spec.NodeQualityProbe.HTTPGet != nil { + if nodeQOS.Spec.NodeQualityProbe.HTTPGet != nil { httpGetEnable = true } - allErrs = append(allErrs, validateObjectiveEnsurances(nep.Spec.ObjectiveEnsurances, field.NewPath("objectiveEnsurances"), httpGetEnable)...) + allErrs = append(allErrs, validateObjectiveEnsurances(nodeQOS.Spec.Rules, field.NewPath("objectiveEnsurances"), httpGetEnable)...) if len(allErrs) != 0 { return allErrs.ToAggregate() @@ -98,7 +98,7 @@ func validateHTTPGetAction(http *corev1.HTTPGetAction, fldPath *field.Path) fiel return allErrs } -func validateObjectiveEnsurances(objects []ensuranceapi.ObjectiveEnsurance, fldPath *field.Path, httpGetEnable bool) field.ErrorList { +func validateObjectiveEnsurances(objects []ensuranceapi.Rule, fldPath *field.Path, httpGetEnable bool) field.ErrorList { allErrs := field.ErrorList{} if len(objects) == 0 { @@ -181,19 +181,19 @@ func validateMetricRule(rule *ensuranceapi.MetricRule, fldPath *field.Path, http } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (p *NepValidationAdmission) ValidateUpdate(ctx context.Context, old, new runtime.Object) error { +func (p *NodeQOSValidationAdmission) ValidateUpdate(ctx context.Context, old, new runtime.Object) error { - nepOld, ok := old.(*ensuranceapi.NodeQOSEnsurancePolicy) + oldNodeQOS, ok := old.(*ensuranceapi.NodeQOS) if !ok { - return fmt.Errorf("old can not convert to NodeQOSEnsurancePolicy") + return fmt.Errorf("old can not convert to NodeQOS") } - nep, ok := old.(*ensuranceapi.NodeQOSEnsurancePolicy) + newNodeQOS, ok := old.(*ensuranceapi.NodeQOS) if !ok { - return fmt.Errorf("new can not convert to NodeQOSEnsurancePolicy") + return fmt.Errorf("new can not convert to NodeQOS") } - allErrs := genericvalidation.ValidateObjectMetaUpdate(&nep.ObjectMeta, &nepOld.ObjectMeta, field.NewPath("metadata")) + allErrs := genericvalidation.ValidateObjectMetaUpdate(&newNodeQOS.ObjectMeta, &oldNodeQOS.ObjectMeta, field.NewPath("metadata")) if len(allErrs) != 0 { return allErrs.ToAggregate() @@ -203,7 +203,7 @@ func (p *NepValidationAdmission) ValidateUpdate(ctx context.Context, old, new ru } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (p *NepValidationAdmission) ValidateDelete(ctx context.Context, req runtime.Object) error { +func (p *NodeQOSValidationAdmission) ValidateDelete(ctx context.Context, req runtime.Object) error { return nil } @@ -275,17 +275,17 @@ func validateEvictionAction(eviction *ensuranceapi.EvictionAction, fldPath *fiel // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type func (p *ActionValidationAdmission) ValidateUpdate(ctx context.Context, old, new runtime.Object) error { - nepOld, ok := old.(*ensuranceapi.AvoidanceAction) + oldNodeQOS, ok := old.(*ensuranceapi.AvoidanceAction) if !ok { return fmt.Errorf("old can not convert to AvoidanceAction") } - nep, ok := old.(*ensuranceapi.AvoidanceAction) + newNodeQOS, ok := old.(*ensuranceapi.AvoidanceAction) if !ok { return fmt.Errorf("new can not convert to AvoidanceAction") } - allErrs := genericvalidation.ValidateObjectMetaUpdate(&nep.ObjectMeta, &nepOld.ObjectMeta, field.NewPath("metadata")) + allErrs := genericvalidation.ValidateObjectMetaUpdate(&newNodeQOS.ObjectMeta, &oldNodeQOS.ObjectMeta, field.NewPath("metadata")) if len(allErrs) != 0 { return allErrs.ToAggregate() diff --git a/pkg/webhooks/webhook.go b/pkg/webhooks/webhook.go index f813ff15e..095b5d2ab 100644 --- a/pkg/webhooks/webhook.go +++ b/pkg/webhooks/webhook.go @@ -66,13 +66,13 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE } if nodeResourceEnabled || clusterNodePredictionEnabled { - nepValidationAdmission := ensurance.NepValidationAdmission{} + nodeQOSValidationAdmission := ensurance.NodeQOSValidationAdmission{} err := ctrl.NewWebhookManagedBy(mgr). - For(&ensuranceapi.NodeQOSEnsurancePolicy{}). - WithValidator(&nepValidationAdmission). + For(&ensuranceapi.NodeQOS{}). + WithValidator(&nodeQOSValidationAdmission). Complete() if err != nil { - klog.Errorf("Failed to setup NodeQOSEnsurancePolicy webhook: %v", err) + klog.Errorf("Failed to setup NodeQOS webhook: %v", err) return err }