From 42c13eb4faac32eaa8518a5f7fc767291a1649bb Mon Sep 17 00:00:00 2001 From: "ying.ding" Date: Thu, 22 Dec 2022 16:51:53 +0800 Subject: [PATCH 1/8] add support for prom-adapter overrides --- cmd/craned/app/manager.go | 2 +- pkg/controller/ehpa/predict.go | 8 +- pkg/prometheus-adapter/config_fetcher.go | 14 +- pkg/prometheus-adapter/expression.go | 43 ++++- .../effective-hpa-with-prometheus-adapter.md | 164 +++++++++--------- 5 files changed, 128 insertions(+), 103 deletions(-) diff --git a/cmd/craned/app/manager.go b/cmd/craned/app/manager.go index 8489f71e2..124c30149 100644 --- a/cmd/craned/app/manager.go +++ b/cmd/craned/app/manager.go @@ -303,7 +303,7 @@ func initControllers(oomRecorder oom.Recorder, mgr ctrl.Manager, opts *options.O AdapterConfig: opts.DataSourcePromConfig.AdapterConfig, } prometheus_adapter.SetExtensionLabels(opts.DataSourcePromConfig.AdapterExtensionLabels) - go pac.Reload() + go pac.Reload(mgr.GetRESTMapper()) } if err := (&ehpa.SubstituteController{ diff --git a/pkg/controller/ehpa/predict.go b/pkg/controller/ehpa/predict.go index f9fee9eb5..829cc2300 100644 --- a/pkg/controller/ehpa/predict.go +++ b/pkg/controller/ehpa/predict.go @@ -165,7 +165,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect if len(mrs.MetricRulesResource) > 0 { metricRule = prometheus_adapter.MatchMetricRule(mrs.MetricRulesResource, metricName) if metricRule == nil { - klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricMatches[%s] SeriesName[%s] MetricName[%s]", metricRule.MetricMatches, metricRule.SeriesName, metricName) + klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName) } else { klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) matchLabels = append(matchLabels, fmt.Sprintf("pod=~\"%s\"", utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind))) @@ -175,7 +175,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect if len(mrs.MetricRulesCustomer) > 0 { metricRule = prometheus_adapter.MatchMetricRule(mrs.MetricRulesCustomer, metricName) if metricRule == nil { - klog.Errorf("Got MetricRulesCustomer prometheus-adapter-customer Failed MetricMatches[%s] SeriesName[%s] MetricName[%s]", metricRule.MetricMatches, metricRule.SeriesName, metricName) + klog.Errorf("Got MetricRulesCustomer prometheus-adapter-customer Failed MetricName[%s]", metricName) } else { klog.V(4).Infof("Got MetricRulesCustomer prometheus-adapter-customer MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) if metric.Pods.Metric.Selector != nil { @@ -189,7 +189,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect if len(mrs.MetricRulesExternal) > 0 { metricRule = prometheus_adapter.MatchMetricRule(mrs.MetricRulesExternal, metricName) if metricRule == nil { - klog.Errorf("Got MetricRulesExternal prometheus-adapter-external Failed MetricMatches[%s] SeriesName[%s] MetricName[%s]", metricRule.MetricMatches, metricRule.SeriesName, metricName) + klog.Errorf("Got MetricRulesExternal prometheus-adapter-external Failed MetricName[%s]", metricName) } else { klog.V(4).Infof("Got MetricRulesExternal prometheus-adapter-external MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) if metric.External.Metric.Selector != nil { @@ -206,7 +206,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect var err error expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, append(mrs.ExtensionLabels, matchLabels...)) if err != nil { - klog.Errorf("Got promSelector prometheus-adapter %v", err) + klog.Errorf("Got promSelector prometheus-adapter %v %v", metricRule, err) } else { klog.V(4).Infof("Got expressionQuery [%s] from prometheus-adapter ", expressionQuery) } diff --git a/pkg/prometheus-adapter/config_fetcher.go b/pkg/prometheus-adapter/config_fetcher.go index a49841e08..346d3d4a4 100644 --- a/pkg/prometheus-adapter/config_fetcher.go +++ b/pkg/prometheus-adapter/config_fetcher.go @@ -57,7 +57,7 @@ func (pc *PrometheusAdapterConfigFetcher) Reconcile(ctx context.Context, req ctr klog.Errorf("Got metricsDiscoveryConfig failed[%s] %v", pc.AdapterConfigMapName, err) } - err = FlushRules(*cfg) + err = FlushRules(*cfg, pc.RestMapper) if err != nil { klog.Errorf("Flush rules failed %v", err) } @@ -95,7 +95,7 @@ func (paCm *PrometheusAdapterConfigChangedPredicate) Update(e event.UpdateEvent) } // if set promAdapterConfig, daemon reload by config fsnotify -func (pc *PrometheusAdapterConfigFetcher) Reload() { +func (pc *PrometheusAdapterConfigFetcher) Reload(mapper meta.RESTMapper) { watcher, err := fsnotify.NewWatcher() if err != nil { klog.Error(err) @@ -120,7 +120,7 @@ func (pc *PrometheusAdapterConfigFetcher) Reload() { if err != nil { klog.Errorf("Got metricsDiscoveryConfig failed[%s] %v", pc.AdapterConfig, err) } else { - err = FlushRules(*metricsDiscoveryConfig) + err = FlushRules(*metricsDiscoveryConfig, mapper) if err != nil { klog.Errorf("Flush rules failed %v", err) } @@ -134,16 +134,16 @@ func (pc *PrometheusAdapterConfigFetcher) Reload() { } } -func FlushRules(metricsDiscoveryConfig config.MetricsDiscoveryConfig) error { - err := ParsingResourceRules(metricsDiscoveryConfig) +func FlushRules(metricsDiscoveryConfig config.MetricsDiscoveryConfig, mapper meta.RESTMapper) error { + err := ParsingResourceRules(metricsDiscoveryConfig, mapper) if err != nil { return err } - err = ParsingRules(metricsDiscoveryConfig) + err = ParsingRules(metricsDiscoveryConfig, mapper) if err != nil { return err } - err = ParsingExternalRules(metricsDiscoveryConfig) + err = ParsingExternalRules(metricsDiscoveryConfig, mapper) if err != nil { return err } diff --git a/pkg/prometheus-adapter/expression.go b/pkg/prometheus-adapter/expression.go index cd6b8b72e..ea580980d 100644 --- a/pkg/prometheus-adapter/expression.go +++ b/pkg/prometheus-adapter/expression.go @@ -7,7 +7,9 @@ import ( "strings" "text/template" + "k8s.io/apimachinery/pkg/api/meta" "sigs.k8s.io/prometheus-adapter/pkg/config" + "sigs.k8s.io/prometheus-adapter/pkg/naming" ) // MetricMatches for Prometheus-adapter-config @@ -41,6 +43,7 @@ type MetricRules struct { type MetricRule struct { MetricMatches string SeriesName string + ResConverter naming.ResourceConverter Template *template.Template Namespaced bool LabelMatchers []string @@ -57,27 +60,27 @@ func GetMetricRules() *MetricRules { } // ParsingResourceRules from config.MetricsDiscoveryConfig -func ParsingResourceRules(mc config.MetricsDiscoveryConfig) (err error) { - metricRules.MetricRulesResource, err = GetMetricRulesFromResourceRules(*mc.ResourceRules) +func ParsingResourceRules(mc config.MetricsDiscoveryConfig, mapper meta.RESTMapper) (err error) { + metricRules.MetricRulesResource, err = GetMetricRulesFromResourceRules(*mc.ResourceRules, mapper) return err } // ParsingRules from config.MetricsDiscoveryConfig -func ParsingRules(mc config.MetricsDiscoveryConfig) (err error) { +func ParsingRules(mc config.MetricsDiscoveryConfig, mapper meta.RESTMapper) (err error) { if mc.Rules == nil { return fmt.Errorf("Rules is nil") } else { - metricRules.MetricRulesCustomer, err = GetMetricRulesFromDiscoveryRule(mc.Rules) + metricRules.MetricRulesCustomer, err = GetMetricRulesFromDiscoveryRule(mc.Rules, mapper) } return err } // ParsingExternalRules from config.MetricsDiscoveryConfig -func ParsingExternalRules(mc config.MetricsDiscoveryConfig) (err error) { +func ParsingExternalRules(mc config.MetricsDiscoveryConfig, mapper meta.RESTMapper) (err error) { if mc.ExternalRules == nil { return fmt.Errorf("ExternalRules is nil") } else { - metricRules.MetricRulesExternal, err = GetMetricRulesFromDiscoveryRule(mc.ExternalRules) + metricRules.MetricRulesExternal, err = GetMetricRulesFromDiscoveryRule(mc.ExternalRules, mapper) } return err } @@ -92,11 +95,16 @@ func SetExtensionLabels(extensionLabels string) { } // GetMetricRuleResourceFromRules produces a MetricNamer for each rule in the given config. -func GetMetricRulesFromResourceRules(cfg config.ResourceRules) ([]MetricRule, error) { +func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTMapper) ([]MetricRule, error) { var metricRules []MetricRule // get cpu MetricsQuery if cfg.CPU.ContainerQuery != "" { + resConverter, err := naming.NewResourceConverter(cfg.CPU.Resources.Template, cfg.CPU.Resources.Overrides, mapper) + if err != nil { + return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", "resource.cpu", err) + } + reg, err := regexp.Compile(`\s*by\s*\(<<.GroupBy>>\)\s*$`) if err != nil { return nil, fmt.Errorf("unable to match <.GroupBy>") @@ -110,12 +118,18 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules) ([]MetricRule, er metricRules = append(metricRules, MetricRule{ MetricMatches: "cpu", + ResConverter: resConverter, Template: templ, Namespaced: true, }) } // get cpu MetricsQuery if cfg.Memory.ContainerQuery != "" { + resConverter, err := naming.NewResourceConverter(cfg.Memory.Resources.Template, cfg.Memory.Resources.Overrides, mapper) + if err != nil { + return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", "resource.memory", err) + } + reg, err := regexp.Compile(`\s*by\s*\(<<.GroupBy>>\)\s*$`) if err != nil { return nil, fmt.Errorf("unable to match <.GroupBy>") @@ -129,6 +143,7 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules) ([]MetricRule, er metricRules = append(metricRules, MetricRule{ MetricMatches: "memory", + ResConverter: resConverter, Template: templ, Namespaced: true, }) @@ -138,10 +153,15 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules) ([]MetricRule, er } // GetMetricRuleFromRules produces a MetricNamer for each rule in the given config. -func GetMetricRulesFromDiscoveryRule(cfg []config.DiscoveryRule) ([]MetricRule, error) { +func GetMetricRulesFromDiscoveryRule(cfg []config.DiscoveryRule, mapper meta.RESTMapper) ([]MetricRule, error) { metricRules := make([]MetricRule, len(cfg)) for i, rule := range cfg { + resConverter, err := naming.NewResourceConverter(rule.Resources.Template, rule.Resources.Overrides, mapper) + if err != nil { + return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", rule.SeriesQuery, err) + } + // queries are namespaced by default unless the rule specifically disables it namespaced := true if rule.Resources.Namespaced != nil { @@ -178,6 +198,7 @@ func GetMetricRulesFromDiscoveryRule(cfg []config.DiscoveryRule) ([]MetricRule, MetricMatches: metricMatches, SeriesName: seriesName, LabelMatchers: labelMatchers, + ResConverter: resConverter, Template: templ, Namespaced: namespaced, } @@ -203,7 +224,11 @@ func (mr *MetricRule) QueryForSeries(namespace string, exprs []string) (expressi } if mr.Namespaced && namespace != "" { - exprs = append(exprs, fmt.Sprintf("namespace=\"%s\"", namespace)) + namespaceLbl, err := mr.ResConverter.LabelForResource(naming.NsGroupResource) + if err != nil { + return "", err + } + exprs = append(exprs, fmt.Sprintf("%s=\"%s\"", namespaceLbl, namespace)) } args := &QueryTemplateArgs{ diff --git a/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md b/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md index 0006edc41..0f4fd3ab6 100644 --- a/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md +++ b/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md @@ -2,7 +2,7 @@ title: "基于 Effective HPA 实现自定义指标的智能弹性实践" weight: 10 description: > - Effective HPA 的最佳实践. +Effective HPA 的最佳实践. --- Kubernetes HPA 支持了丰富的弹性扩展能力，Kubernetes 平台开发者部署服务实现自定义 Metric 的服务，Kubernetes 用户配置多项内置的资源指标或者自定义 Metric 指标实现自定义水平弹性。 @@ -179,24 +179,24 @@ craned通过读取prometheus-adapter配置，实现查询表达式模板的自 通过ConfigFile加载 - prometheus-adapter-config=/prometheus-adapter.cfg -全局扩展标签 +全局扩展标签 - 通过该参数可实现查询表达式labelMatchers的全局扩展，实现指标的分类，多个标签以","分隔 +通过该参数可实现查询表达式labelMatchers的全局扩展，实现指标的分类，多个标签以","分隔 -- prometheus-adapter-extension-labels=cluster="prod",container!="" +- prometheus-adapter-extension-labels=cluster="prod",container!="" ```yaml - spec: - containers: - - args: +spec: + containers: + - args: - --prometheus-adapter-configmap-namespace=monitoring - --prometheus-adapter-configmap-name=prometheus-adapter-config - - --prometheus-adapter-configmap-key=config.yaml + - --prometheus-adapter-configmap-key=config - --prometheus-adapter-extension-labels=cluster="prod",container!="" ... - command: - - /craned +command: + - /craned ``` @@ -226,16 +226,16 @@ spec: app: sample-app spec: containers: - - image: luxas/autoscale-demo:v0.1.2 - name: metrics-provider - resources: - limits: - cpu: 500m - requests: - cpu: 200m - ports: - - name: http - containerPort: 8080 + - image: luxas/autoscale-demo:v0.1.2 + name: metrics-provider + resources: + limits: + cpu: 500m + requests: + cpu: 200m + ports: + - name: http + containerPort: 8080 ``` sample-app.service.yaml @@ -249,10 +249,10 @@ metadata: name: sample-app spec: ports: - - name: http - port: 80 - protocol: TCP - targetPort: 8080 + - name: http + port: 80 + protocol: TCP + targetPort: 8080 selector: app: sample-app type: ClusterIP @@ -282,22 +282,22 @@ kubectl edit configmap -n crane-system prometheus-server ```yaml - job_name: sample-app kubernetes_sd_configs: - - role: pod + - role: pod relabel_configs: - - action: keep - regex: default;sample-app-(.+) - source_labels: - - __meta_kubernetes_namespace - - __meta_kubernetes_pod_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: pod + - action: keep + regex: default;sample-app-(.+) + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod ``` 此时，您可以在 Prometheus 查询 psql：sum(rate(http_requests_total[5m])) by (pod) @@ -364,19 +364,19 @@ spec: scaleStrategy: Auto # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Manual". # Metrics contains the specifications for which to use to calculate the desired replica count. metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 50 - - type: Pods - pods: - metric: - name: http_requests - target: - type: AverageValue - averageValue: 500m + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + - type: Pods + pods: + metric: + name: http_requests + target: + type: AverageValue + averageValue: 500m # Prediction defines configurations for predict resources. # If unspecified, defaults don't enable prediction. prediction: @@ -395,13 +395,13 @@ kubectl create -f sample-app-hpa.yaml 通过Prometheus-adapter增加模板配置 ```yaml rules: - - seriesQuery: 'http_requests_total{pod!=""}' - name: - matches: "(.*)_total$" - as: "${1}" - resources: - namespaced: true - metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)' + - seriesQuery: 'http_requests_total{pod!=""}' + name: + matches: "(.*)_total$" + as: "${1}" + resources: + namespaced: true + metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)' ``` 查看 TimeSeriesPrediction 状态，如果应用运行时间较短，可能会无法预测： @@ -489,29 +489,29 @@ metadata: spec: maxReplicas: 10 metrics: - - pods: - metric: - name: http_requests - target: - averageValue: 500m - type: AverageValue - type: Pods - - pods: - metric: - name: pods.http_requests - selector: - matchLabels: - autoscaling.crane.io/effective-hpa-uid: 1322c5ac-a1c6-4c71-98d6-e85d07b22da0 - target: - averageValue: 500m - type: AverageValue - type: Pods - - resource: - name: cpu - target: - averageUtilization: 50 - type: Utilization - type: Resource + - pods: + metric: + name: http_requests + target: + averageValue: 500m + type: AverageValue + type: Pods + - pods: + metric: + name: pods.http_requests + selector: + matchLabels: + autoscaling.crane.io/effective-hpa-uid: 1322c5ac-a1c6-4c71-98d6-e85d07b22da0 + target: + averageValue: 500m + type: AverageValue + type: Pods + - resource: + name: cpu + target: + averageUtilization: 50 + type: Utilization + type: Resource minReplicas: 1 scaleTargetRef: apiVersion: apps/v1 From f5ae999311389b06459775ae006656992709dbfb Mon Sep 17 00:00:00 2001 From: "ying.ding" Date: Thu, 22 Dec 2022 16:56:17 +0800 Subject: [PATCH 2/8] add support for prom-adapter overrides --- cmd/craned/app/manager.go | 3 ++- pkg/prometheus-adapter/config_fetcher.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/craned/app/manager.go b/cmd/craned/app/manager.go index 124c30149..dacedf511 100644 --- a/cmd/craned/app/manager.go +++ b/cmd/craned/app/manager.go @@ -300,10 +300,11 @@ func initControllers(oomRecorder oom.Recorder, mgr ctrl.Manager, opts *options.O } else if opts.DataSourcePromConfig.AdapterConfig != "" { // PrometheusAdapterConfigFetcher pac := &prometheus_adapter.PrometheusAdapterConfigFetcher{ + RestMapper: mgr.GetRESTMapper(), AdapterConfig: opts.DataSourcePromConfig.AdapterConfig, } prometheus_adapter.SetExtensionLabels(opts.DataSourcePromConfig.AdapterExtensionLabels) - go pac.Reload(mgr.GetRESTMapper()) + go pac.Reload() } if err := (&ehpa.SubstituteController{ diff --git a/pkg/prometheus-adapter/config_fetcher.go b/pkg/prometheus-adapter/config_fetcher.go index 346d3d4a4..14b465934 100644 --- a/pkg/prometheus-adapter/config_fetcher.go +++ b/pkg/prometheus-adapter/config_fetcher.go @@ -95,7 +95,7 @@ func (paCm *PrometheusAdapterConfigChangedPredicate) Update(e event.UpdateEvent) } // if set promAdapterConfig, daemon reload by config fsnotify -func (pc *PrometheusAdapterConfigFetcher) Reload(mapper meta.RESTMapper) { +func (pc *PrometheusAdapterConfigFetcher) Reload() { watcher, err := fsnotify.NewWatcher() if err != nil { klog.Error(err) @@ -120,7 +120,7 @@ func (pc *PrometheusAdapterConfigFetcher) Reload(mapper meta.RESTMapper) { if err != nil { klog.Errorf("Got metricsDiscoveryConfig failed[%s] %v", pc.AdapterConfig, err) } else { - err = FlushRules(*metricsDiscoveryConfig, mapper) + err = FlushRules(*metricsDiscoveryConfig, pc.RestMapper) if err != nil { klog.Errorf("Flush rules failed %v", err) } From f594fce1a4b0f45853b3a8977aa3cad09f96df3a Mon Sep 17 00:00:00 2001 From: "ying.ding" Date: Thu, 22 Dec 2022 17:12:36 +0800 Subject: [PATCH 3/8] add support for prom-adapter overrides --- .../effective-hpa-with-prometheus-adapter.md | 160 +++++++++--------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md b/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md index 0f4fd3ab6..2300e2300 100644 --- a/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md +++ b/site/content/zh/docs/Best Practices/effective-hpa-with-prometheus-adapter.md @@ -2,7 +2,7 @@ title: "基于 Effective HPA 实现自定义指标的智能弹性实践" weight: 10 description: > -Effective HPA 的最佳实践. + Effective HPA 的最佳实践. --- Kubernetes HPA 支持了丰富的弹性扩展能力，Kubernetes 平台开发者部署服务实现自定义 Metric 的服务，Kubernetes 用户配置多项内置的资源指标或者自定义 Metric 指标实现自定义水平弹性。 @@ -179,24 +179,24 @@ craned通过读取prometheus-adapter配置，实现查询表达式模板的自 通过ConfigFile加载 - prometheus-adapter-config=/prometheus-adapter.cfg -全局扩展标签 +全局扩展标签 -通过该参数可实现查询表达式labelMatchers的全局扩展，实现指标的分类，多个标签以","分隔 + 通过该参数可实现查询表达式labelMatchers的全局扩展，实现指标的分类，多个标签以","分隔 - prometheus-adapter-extension-labels=cluster="prod",container!="" ```yaml -spec: - containers: - - args: + spec: + containers: + - args: - --prometheus-adapter-configmap-namespace=monitoring - --prometheus-adapter-configmap-name=prometheus-adapter-config - --prometheus-adapter-configmap-key=config - --prometheus-adapter-extension-labels=cluster="prod",container!="" ... -command: - - /craned + command: + - /craned ``` @@ -226,16 +226,16 @@ spec: app: sample-app spec: containers: - - image: luxas/autoscale-demo:v0.1.2 - name: metrics-provider - resources: - limits: - cpu: 500m - requests: - cpu: 200m - ports: - - name: http - containerPort: 8080 + - image: luxas/autoscale-demo:v0.1.2 + name: metrics-provider + resources: + limits: + cpu: 500m + requests: + cpu: 200m + ports: + - name: http + containerPort: 8080 ``` sample-app.service.yaml @@ -249,10 +249,10 @@ metadata: name: sample-app spec: ports: - - name: http - port: 80 - protocol: TCP - targetPort: 8080 + - name: http + port: 80 + protocol: TCP + targetPort: 8080 selector: app: sample-app type: ClusterIP @@ -282,22 +282,22 @@ kubectl edit configmap -n crane-system prometheus-server ```yaml - job_name: sample-app kubernetes_sd_configs: - - role: pod + - role: pod relabel_configs: - - action: keep - regex: default;sample-app-(.+) - source_labels: - - __meta_kubernetes_namespace - - __meta_kubernetes_pod_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: pod + - action: keep + regex: default;sample-app-(.+) + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod ``` 此时，您可以在 Prometheus 查询 psql：sum(rate(http_requests_total[5m])) by (pod) @@ -364,19 +364,19 @@ spec: scaleStrategy: Auto # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Manual". # Metrics contains the specifications for which to use to calculate the desired replica count. metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 50 - - type: Pods - pods: - metric: - name: http_requests - target: - type: AverageValue - averageValue: 500m + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + - type: Pods + pods: + metric: + name: http_requests + target: + type: AverageValue + averageValue: 500m # Prediction defines configurations for predict resources. # If unspecified, defaults don't enable prediction. prediction: @@ -395,13 +395,13 @@ kubectl create -f sample-app-hpa.yaml 通过Prometheus-adapter增加模板配置 ```yaml rules: - - seriesQuery: 'http_requests_total{pod!=""}' - name: - matches: "(.*)_total$" - as: "${1}" - resources: - namespaced: true - metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)' + - seriesQuery: 'http_requests_total{pod!=""}' + name: + matches: "(.*)_total$" + as: "${1}" + resources: + namespaced: true + metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)' ``` 查看 TimeSeriesPrediction 状态，如果应用运行时间较短，可能会无法预测： @@ -489,29 +489,29 @@ metadata: spec: maxReplicas: 10 metrics: - - pods: - metric: - name: http_requests - target: - averageValue: 500m - type: AverageValue - type: Pods - - pods: - metric: - name: pods.http_requests - selector: - matchLabels: - autoscaling.crane.io/effective-hpa-uid: 1322c5ac-a1c6-4c71-98d6-e85d07b22da0 - target: - averageValue: 500m - type: AverageValue - type: Pods - - resource: - name: cpu - target: - averageUtilization: 50 - type: Utilization - type: Resource + - pods: + metric: + name: http_requests + target: + averageValue: 500m + type: AverageValue + type: Pods + - pods: + metric: + name: pods.http_requests + selector: + matchLabels: + autoscaling.crane.io/effective-hpa-uid: 1322c5ac-a1c6-4c71-98d6-e85d07b22da0 + target: + averageValue: 500m + type: AverageValue + type: Pods + - resource: + name: cpu + target: + averageUtilization: 50 + type: Utilization + type: Resource minReplicas: 1 scaleTargetRef: apiVersion: apps/v1 From 9e5771da096d2462fa900bfeebf4e550190aaace Mon Sep 17 00:00:00 2001 From: "ying.ding" Date: Thu, 22 Dec 2022 18:23:53 +0800 Subject: [PATCH 4/8] Overrides for pod and ut --- pkg/controller/ehpa/predict.go | 8 ++-- pkg/prometheus-adapter/expression.go | 11 +++++- pkg/prometheus-adapter/expression_test.go | 46 +++++++++++++++++++---- 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/pkg/controller/ehpa/predict.go b/pkg/controller/ehpa/predict.go index 829cc2300..d77db793b 100644 --- a/pkg/controller/ehpa/predict.go +++ b/pkg/controller/ehpa/predict.go @@ -3,7 +3,6 @@ package ehpa import ( "context" "fmt" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -159,6 +158,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect // Supreme priority: annotation expressionQuery := utils.GetExpressionQueryAnnotation(metricIdentifier, ehpa.Annotations) if expressionQuery == "" { + var nameReg string // get metricRule from prometheus-adapter switch metric.Type { case autoscalingv2.ResourceMetricSourceType: @@ -168,7 +168,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName) } else { klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) - matchLabels = append(matchLabels, fmt.Sprintf("pod=~\"%s\"", utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind))) + nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind) } } case autoscalingv2.PodsMetricSourceType: @@ -178,6 +178,8 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect klog.Errorf("Got MetricRulesCustomer prometheus-adapter-customer Failed MetricName[%s]", metricName) } else { klog.V(4).Infof("Got MetricRulesCustomer prometheus-adapter-customer MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) + nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind) + if metric.Pods.Metric.Selector != nil { for _, i := range utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels) { matchLabels = append(matchLabels, i) @@ -204,7 +206,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect if metricRule != nil { // Second priority: get default expressionQuery var err error - expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, append(mrs.ExtensionLabels, matchLabels...)) + expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, nameReg, append(mrs.ExtensionLabels, matchLabels...)) if err != nil { klog.Errorf("Got promSelector prometheus-adapter %v %v", metricRule, err) } else { diff --git a/pkg/prometheus-adapter/expression.go b/pkg/prometheus-adapter/expression.go index ea580980d..f2dc438d8 100644 --- a/pkg/prometheus-adapter/expression.go +++ b/pkg/prometheus-adapter/expression.go @@ -3,6 +3,7 @@ package prometheus_adapter import ( "bytes" "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" "regexp" "strings" "text/template" @@ -218,7 +219,7 @@ func MatchMetricRule(mrs []MetricRule, metricName string) *MetricRule { } // get MetrycsQuery by naming.MetricsQuery.BuildExternal from prometheus-adapter -func (mr *MetricRule) QueryForSeries(namespace string, exprs []string) (expressionQuery string, err error) { +func (mr *MetricRule) QueryForSeries(namespace string, nameReg string, exprs []string) (expressionQuery string, err error) { if mr.LabelMatchers != nil { exprs = append(mr.LabelMatchers, exprs...) } @@ -231,6 +232,14 @@ func (mr *MetricRule) QueryForSeries(namespace string, exprs []string) (expressi exprs = append(exprs, fmt.Sprintf("%s=\"%s\"", namespaceLbl, namespace)) } + if nameReg != "" { + resourceLbl, err := mr.ResConverter.LabelForResource(schema.GroupResource{Resource: "pods"}) + if err != nil { + return "", err + } + exprs = append(exprs, fmt.Sprintf("%s=~\"%s\"", resourceLbl, nameReg)) + } + args := &QueryTemplateArgs{ Series: mr.SeriesName, LabelMatchers: strings.Join(exprs, ","), diff --git a/pkg/prometheus-adapter/expression_test.go b/pkg/prometheus-adapter/expression_test.go index 9b3f8ce9f..83cf5da18 100644 --- a/pkg/prometheus-adapter/expression_test.go +++ b/pkg/prometheus-adapter/expression_test.go @@ -3,6 +3,9 @@ package prometheus_adapter import ( "testing" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/prometheus-adapter/pkg/config" ) @@ -14,25 +17,38 @@ func TestQueryForSeriesResource(t *testing.T) { CPU: config.ResourceRule{ ContainerQuery: containerQuery, Resources: config.ResourceMapping{ - Overrides: map[string]config.GroupResource{}, + Overrides: map[string]config.GroupResource{ + "pod_namespace": {Resource: "namespace"}, + "pod_name": {Resource: "pod"}, + }, Namespaced: &namespaced, }, ContainerLabel: "container", }, } + mp := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + mp.Add(corev1.SchemeGroupVersion.WithKind("Pod"), meta.RESTScopeNamespace) + mp.Add(corev1.SchemeGroupVersion.WithKind("Namespace"), meta.RESTScopeRoot) + test := struct { description string resource config.ResourceRules + mapper meta.RESTMapper + namespace string + nameReg string expect string }{ description: "get expressionQuery For SeriesResource", resource: *cfg, - expect: "sum(rate(container_cpu_usage_seconds_total{namespace=\"test\"}[3m]))", + mapper: mp, + namespace: "test", + nameReg: "test-.*", + expect: "sum(rate(container_cpu_usage_seconds_total{pod_namespace=\"test\",pod_name=~\"test-.*\"}[3m]))", } - metricRules, _ := GetMetricRulesFromResourceRules(test.resource) - requests, err := metricRules[0].QueryForSeries("test", []string{}) + metricRules, _ := GetMetricRulesFromResourceRules(test.resource, test.mapper) + requests, err := metricRules[0].QueryForSeries(test.namespace, test.nameReg, []string{}) if err != nil { t.Errorf("Failed to QueryForSeriesResource: %v", err) } @@ -42,7 +58,7 @@ func TestQueryForSeriesResource(t *testing.T) { } func TestQueryForSeriesRules(t *testing.T) { - seriesQuery := `nginx_concurrent_utilization{pod_namespace!="",pod_name!=""}` + seriesQuery := `nginx_concurrent_utilization{namespace!="",pod!=""}` metricsQuery := `sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)` namespaced := true @@ -50,22 +66,36 @@ func TestQueryForSeriesRules(t *testing.T) { SeriesQuery: seriesQuery, MetricsQuery: metricsQuery, Resources: config.ResourceMapping{ + Overrides: map[string]config.GroupResource{ + "pod_namespace": {Resource: "namespace"}, + "pod_name": {Resource: "pod"}, + }, Namespaced: &namespaced, }, } + mp := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) + mp.Add(corev1.SchemeGroupVersion.WithKind("Pod"), meta.RESTScopeNamespace) + mp.Add(corev1.SchemeGroupVersion.WithKind("Namespace"), meta.RESTScopeRoot) + test := struct { description string resource []config.DiscoveryRule + mapper meta.RESTMapper + namespace string + nameReg string expect string }{ description: "get expressionQuery For SeriesRules", resource: []config.DiscoveryRule{discoveryRule}, - expect: "sum(nginx_concurrent_utilization{pod_namespace!=\"\",pod_name!=\"\",namespace=\"test\"})", + mapper: mp, + namespace: "test", + nameReg: "test-.*", + expect: "sum(nginx_concurrent_utilization{namespace!=\"\",pod!=\"\",pod_namespace=\"test\",pod_name=~\"test-.*\"})", } - metricRules, _ := GetMetricRulesFromDiscoveryRule(test.resource) - requests, err := metricRules[0].QueryForSeries("test", []string{}) + metricRules, _ := GetMetricRulesFromDiscoveryRule(test.resource, test.mapper) + requests, err := metricRules[0].QueryForSeries(test.namespace, test.nameReg, []string{}) if err != nil { t.Errorf("Failed to QueryForSeriesResource: %v", err) } From 17d5061c42a922c0e069813dcef581465f237c4e Mon Sep 17 00:00:00 2001 From: "ying.ding" Date: Thu, 22 Dec 2022 18:32:27 +0800 Subject: [PATCH 5/8] Overrides for pod and ut --- pkg/querybuilder-providers/prometheus/builder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/querybuilder-providers/prometheus/builder.go b/pkg/querybuilder-providers/prometheus/builder.go index 145e2f18d..11ec1f6cd 100644 --- a/pkg/querybuilder-providers/prometheus/builder.go +++ b/pkg/querybuilder-providers/prometheus/builder.go @@ -71,7 +71,7 @@ func (b *builder) workloadQuery(metric *metricquery.Metric) (query *metricquery. } if queryExpr == "" { - queryExpr, err = metricRule.QueryForSeries(metric.Workload.Namespace, append(mrs.ExtensionLabels, fmt.Sprintf("pod=~\"%s\"", utils.GetPodNameReg(metric.Workload.Name, metric.Workload.Kind)))) + queryExpr, err = metricRule.QueryForSeries(metric.Workload.Namespace, utils.GetPodNameReg(metric.Workload.Name, metric.Workload.Kind), mrs.ExtensionLabels) } return promQuery(&metricquery.PrometheusQuery{Query: queryExpr}), err } @@ -100,7 +100,7 @@ func (b *builder) nodeQuery(metric *metricquery.Metric) (query *metricquery.Quer } if queryExpr == "" { - queryExpr, err = metricRule.QueryForSeries("", append(mrs.ExtensionLabels, fmt.Sprintf("instance=~\"(%s)(:\\d+)?\"", metric.Node.Name))) + queryExpr, err = metricRule.QueryForSeries("", "", append(mrs.ExtensionLabels, fmt.Sprintf("instance=~\"(%s)(:\\d+)?\"", metric.Node.Name))) } return promQuery(&metricquery.PrometheusQuery{Query: queryExpr}), err @@ -130,7 +130,7 @@ func (b *builder) containerQuery(metric *metricquery.Metric) (query *metricquery } if queryExpr == "" { - queryExpr, err = metricRule.QueryForSeries(metric.Container.Namespace, append(mrs.ExtensionLabels, []string{fmt.Sprintf("pod=~\"%s\"", utils.GetPodNameReg(metric.Container.WorkloadName, metric.Container.WorkloadKind)), fmt.Sprintf("container=\"%s\"", metric.Container.Name)}...)) + queryExpr, err = metricRule.QueryForSeries(metric.Container.Namespace, utils.GetPodNameReg(metric.Container.WorkloadName, metric.Container.WorkloadKind), append(mrs.ExtensionLabels, fmt.Sprintf("container=\"%s\"", metric.Container.Name))) } return promQuery(&metricquery.PrometheusQuery{ Query: queryExpr, @@ -161,7 +161,7 @@ func (b *builder) podQuery(metric *metricquery.Metric) (query *metricquery.Query } if queryExpr == "" { - queryExpr, err = metricRule.QueryForSeries(metric.Pod.Namespace, append(mrs.ExtensionLabels, fmt.Sprintf("pod=\"%s\"", metric.Pod.Name))) + queryExpr, err = metricRule.QueryForSeries(metric.Pod.Namespace, "", append(mrs.ExtensionLabels, fmt.Sprintf("pod=\"%s\"", metric.Pod.Name))) } return promQuery(&metricquery.PrometheusQuery{ Query: queryExpr, From ab3018f2789e2bcb14f3ec18aa75159800b462c1 Mon Sep 17 00:00:00 2001 From: "ying.ding" Date: Fri, 23 Dec 2022 09:25:16 +0800 Subject: [PATCH 6/8] golint --- pkg/controller/ehpa/predict.go | 1 + pkg/prometheus-adapter/expression.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/ehpa/predict.go b/pkg/controller/ehpa/predict.go index d77db793b..c1431df23 100644 --- a/pkg/controller/ehpa/predict.go +++ b/pkg/controller/ehpa/predict.go @@ -3,6 +3,7 @@ package ehpa import ( "context" "fmt" + autoscalingv2 "k8s.io/api/autoscaling/v2beta2" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/prometheus-adapter/expression.go b/pkg/prometheus-adapter/expression.go index f2dc438d8..2daded2ac 100644 --- a/pkg/prometheus-adapter/expression.go +++ b/pkg/prometheus-adapter/expression.go @@ -3,12 +3,12 @@ package prometheus_adapter import ( "bytes" "fmt" - "k8s.io/apimachinery/pkg/runtime/schema" "regexp" "strings" "text/template" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/prometheus-adapter/pkg/config" "sigs.k8s.io/prometheus-adapter/pkg/naming" ) From f1da73719aedc1c99a3ae662a24bb34e517f0c93 Mon Sep 17 00:00:00 2001 From: qmhu Date: Wed, 28 Dec 2022 16:41:40 +0800 Subject: [PATCH 7/8] fix the update status error for ehpa --- .../ehpa/effective_hpa_controller.go | 35 ++++++++++++++----- pkg/controller/ehpa/substitute.go | 2 +- .../recommendation_controller.go | 2 +- .../recommendation_rule_controller.go | 4 +-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/pkg/controller/ehpa/effective_hpa_controller.go b/pkg/controller/ehpa/effective_hpa_controller.go index 70c6d6deb..6400d2edf 100644 --- a/pkg/controller/ehpa/effective_hpa_controller.go +++ b/pkg/controller/ehpa/effective_hpa_controller.go @@ -10,10 +10,12 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/discovery" "k8s.io/client-go/scale" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -152,17 +154,34 @@ func (c *EffectiveHPAController) Reconcile(ctx context.Context, req ctrl.Request } func (c *EffectiveHPAController) UpdateStatus(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, newStatus *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) { - if !equality.Semantic.DeepEqual(&ehpa.Status, newStatus) { - ehpa.Status = *newStatus - err := c.Status().Update(ctx, ehpa) - if err != nil { - c.Recorder.Event(ehpa, v1.EventTypeWarning, "FailedUpdateStatus", err.Error()) - klog.Errorf("Failed to update status, ehpa %s error %v", klog.KObj(ehpa), err) - return + ehpaCopy := ehpa.DeepCopy() + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if !equality.Semantic.DeepEqual(&ehpaCopy.Status, newStatus) { + ehpaCopy.Status = *newStatus + err := c.Status().Update(ctx, ehpaCopy) + if err == nil { + return nil + } + + updated := &autoscalingapi.EffectiveHorizontalPodAutoscaler{} + errGet := c.Get(context.TODO(), types.NamespacedName{Namespace: ehpaCopy.Namespace, Name: ehpaCopy.Name}, updated) + if errGet == nil { + ehpaCopy = updated + } + + return err } - klog.Infof("Update EffectiveHorizontalPodAutoscaler status successful, ehpa %s", klog.KObj(ehpa)) + return nil + }) + + if err != nil { + c.Recorder.Event(ehpa, v1.EventTypeWarning, "FailedUpdateStatus", err.Error()) + klog.Errorf("Failed to update status, EffectiveHorizontalPodAutoscaler %s error %v", klog.KObj(ehpa), err) + return } + + klog.V(2).Infof("Update EffectiveHorizontalPodAutoscaler %s status successful ", klog.KObj(ehpa)) } func (c *EffectiveHPAController) SetupWithManager(mgr ctrl.Manager) error { diff --git a/pkg/controller/ehpa/substitute.go b/pkg/controller/ehpa/substitute.go index 9f67586bf..90398e7c6 100644 --- a/pkg/controller/ehpa/substitute.go +++ b/pkg/controller/ehpa/substitute.go @@ -97,7 +97,7 @@ func (c *EffectiveHPAController) UpdateSubstituteIfNeed(ctx context.Context, ehp return nil, err } - klog.Infof("Update Substitute successful, Substitute %s", klog.KObj(substituteExist)) + klog.V(2).Infof("Update Substitute successful, Substitute %s", klog.KObj(substituteExist)) } return substituteExist, nil diff --git a/pkg/controller/recommendation/recommendation_controller.go b/pkg/controller/recommendation/recommendation_controller.go index ade8ca572..7e9e1ceca 100644 --- a/pkg/controller/recommendation/recommendation_controller.go +++ b/pkg/controller/recommendation/recommendation_controller.go @@ -87,7 +87,7 @@ func (c *RecommendationController) UpdateStatus(ctx context.Context, recommendat return err } - klog.Infof("Update Recommendation status successful, Recommendation %s", klog.KObj(recommendation)) + klog.V(2).Infof("Update Recommendation status successful, Recommendation %s", klog.KObj(recommendation)) } return nil diff --git a/pkg/controller/recommendation/recommendation_rule_controller.go b/pkg/controller/recommendation/recommendation_rule_controller.go index 79652c8ce..712d523c1 100644 --- a/pkg/controller/recommendation/recommendation_rule_controller.go +++ b/pkg/controller/recommendation/recommendation_rule_controller.go @@ -414,7 +414,7 @@ func (c *RecommendationRuleController) executeMission(ctx context.Context, wg *s } func (c *RecommendationRuleController) UpdateStatus(ctx context.Context, recommendationRule *analysisv1alph1.RecommendationRule, newStatus *analysisv1alph1.RecommendationRuleStatus) { - klog.Infof("Updating RecommendationRule %s status", klog.KObj(recommendationRule)) + klog.V(2).Infof("Updating RecommendationRule %s status", klog.KObj(recommendationRule)) recommendationRuleCopy := recommendationRule.DeepCopy() err := retry.RetryOnConflict(retry.DefaultRetry, func() error { if !equality.Semantic.DeepEqual(&recommendationRuleCopy.Status, newStatus) { @@ -442,7 +442,7 @@ func (c *RecommendationRuleController) UpdateStatus(ctx context.Context, recomme return } - klog.Infof("Update RecommendationRule status successful, RecommendationRule %s", klog.KObj(recommendationRule)) + klog.V(2).Infof("Update RecommendationRule status successful, RecommendationRule %s", klog.KObj(recommendationRule)) } type ObjectIdentity struct { From e7fa51a7166029684c1bef3e9e6d63dea965f76f Mon Sep 17 00:00:00 2001 From: "ying.ding" Date: Wed, 4 Jan 2023 15:20:58 +0800 Subject: [PATCH 8/8] overrides --- pkg/prometheus-adapter/expression.go | 57 ++++++++++++++++++---------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/pkg/prometheus-adapter/expression.go b/pkg/prometheus-adapter/expression.go index 2daded2ac..6ceac7b71 100644 --- a/pkg/prometheus-adapter/expression.go +++ b/pkg/prometheus-adapter/expression.go @@ -96,14 +96,15 @@ func SetExtensionLabels(extensionLabels string) { } // GetMetricRuleResourceFromRules produces a MetricNamer for each rule in the given config. -func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTMapper) ([]MetricRule, error) { - var metricRules []MetricRule - +func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTMapper) (metricRules []MetricRule, err error) { // get cpu MetricsQuery if cfg.CPU.ContainerQuery != "" { - resConverter, err := naming.NewResourceConverter(cfg.CPU.Resources.Template, cfg.CPU.Resources.Overrides, mapper) - if err != nil { - return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", "resource.cpu", err) + var resConverter naming.ResourceConverter + if cfg.CPU.Resources.Overrides != nil { + resConverter, err = naming.NewResourceConverter(cfg.CPU.Resources.Template, cfg.CPU.Resources.Overrides, mapper) + if err != nil { + return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", "resource.cpu", err) + } } reg, err := regexp.Compile(`\s*by\s*\(<<.GroupBy>>\)\s*$`) @@ -126,11 +127,13 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM } // get cpu MetricsQuery if cfg.Memory.ContainerQuery != "" { - resConverter, err := naming.NewResourceConverter(cfg.Memory.Resources.Template, cfg.Memory.Resources.Overrides, mapper) - if err != nil { - return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", "resource.memory", err) + var resConverter naming.ResourceConverter + if cfg.Memory.Resources.Overrides != nil { + resConverter, err = naming.NewResourceConverter(cfg.Memory.Resources.Template, cfg.Memory.Resources.Overrides, mapper) + if err != nil { + return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", "resource.memory", err) + } } - reg, err := regexp.Compile(`\s*by\s*\(<<.GroupBy>>\)\s*$`) if err != nil { return nil, fmt.Errorf("unable to match <.GroupBy>") @@ -156,11 +159,15 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM // GetMetricRuleFromRules produces a MetricNamer for each rule in the given config. func GetMetricRulesFromDiscoveryRule(cfg []config.DiscoveryRule, mapper meta.RESTMapper) ([]MetricRule, error) { metricRules := make([]MetricRule, len(cfg)) + var err error for i, rule := range cfg { - resConverter, err := naming.NewResourceConverter(rule.Resources.Template, rule.Resources.Overrides, mapper) - if err != nil { - return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", rule.SeriesQuery, err) + var resConverter naming.ResourceConverter + if rule.Resources.Overrides != nil { + resConverter, err = naming.NewResourceConverter(rule.Resources.Template, rule.Resources.Overrides, mapper) + if err != nil { + return nil, fmt.Errorf("unable to construct label-resource converter: %s %v", rule.SeriesQuery, err) + } } // queries are namespaced by default unless the rule specifically disables it @@ -225,19 +232,27 @@ func (mr *MetricRule) QueryForSeries(namespace string, nameReg string, exprs []s } if mr.Namespaced && namespace != "" { - namespaceLbl, err := mr.ResConverter.LabelForResource(naming.NsGroupResource) - if err != nil { - return "", err + if mr.ResConverter != nil { + namespaceLbl, err := mr.ResConverter.LabelForResource(naming.NsGroupResource) + if err != nil { + return "", err + } + exprs = append(exprs, fmt.Sprintf("%s=\"%s\"", namespaceLbl, namespace)) + } else { + exprs = append(exprs, fmt.Sprintf("%s=\"%s\"", "namespace", namespace)) } - exprs = append(exprs, fmt.Sprintf("%s=\"%s\"", namespaceLbl, namespace)) } if nameReg != "" { - resourceLbl, err := mr.ResConverter.LabelForResource(schema.GroupResource{Resource: "pods"}) - if err != nil { - return "", err + if mr.ResConverter != nil { + resourceLbl, err := mr.ResConverter.LabelForResource(schema.GroupResource{Resource: "pods"}) + if err != nil { + return "", err + } + exprs = append(exprs, fmt.Sprintf("%s=~\"%s\"", resourceLbl, nameReg)) + } else { + exprs = append(exprs, fmt.Sprintf("%s=~\"%s\"", "pod", nameReg)) } - exprs = append(exprs, fmt.Sprintf("%s=~\"%s\"", resourceLbl, nameReg)) } args := &QueryTemplateArgs{