From d67aeebb2e8be3c1e155761a2b801ac28d19a78c Mon Sep 17 00:00:00 2001 From: Song Gao Date: Wed, 8 Jul 2020 16:51:16 +0800 Subject: [PATCH] fix error --- docs/api-references/docs.md | 2 +- .../v1alpha1/tidbclusterautoscaler_types.go | 2 +- .../autoscaler/calculate/storage.go | 7 +++-- pkg/autoscaler/autoscaler/calculate/util.go | 16 ----------- pkg/autoscaler/autoscaler/tikv_autoscaler.go | 28 +++++++++++-------- pkg/autoscaler/autoscaler/util.go | 16 +++++++++-- 6 files changed, 37 insertions(+), 34 deletions(-) diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index e9a3973ab09..f251372fb42 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -3671,7 +3671,7 @@ int64

LeastRemainAvailableStoragePercent indicates the least remaining available storage percent compare to the capacity storage. If the available storage is lower than the capacity storage * LeastRemainAvailableStoragePercent, the storage status will become storage pressure and ready to be scaled out. -LeastRemainAvailableStoragePercent should between 10 and 90. If note set, the default value would be 90

+LeastRemainAvailableStoragePercent should between 5 and 90. If note set, the default value would be 10

diff --git a/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go b/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go index 371492c8bb1..8f23d1a7d74 100644 --- a/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go +++ b/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go @@ -147,7 +147,7 @@ type CustomMetrics struct { // LeastRemainAvailableStoragePercent indicates the least remaining available storage percent compare to // the capacity storage. If the available storage is lower than the capacity storage * LeastRemainAvailableStoragePercent, // the storage status will become storage pressure and ready to be scaled out. - // LeastRemainAvailableStoragePercent should between 10 and 90. If note set, the default value would be 90 + // LeastRemainAvailableStoragePercent should between 5 and 90. If note set, the default value would be 10 // +optional LeastRemainAvailableStoragePercent *int64 `json:"leastRemainAvailableStoragePercent,omitempty"` } diff --git a/pkg/autoscaler/autoscaler/calculate/storage.go b/pkg/autoscaler/autoscaler/calculate/storage.go index b34a41c211f..850c312bd72 100644 --- a/pkg/autoscaler/autoscaler/calculate/storage.go +++ b/pkg/autoscaler/autoscaler/calculate/storage.go @@ -15,13 +15,14 @@ package calculate import ( "fmt" + "strconv" + "time" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" promClient "github.com/prometheus/client_golang/api" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" - "strconv" - "time" ) func CalculateWhetherStoragePressure(tac *v1alpha1.TidbClusterAutoScaler, capacitySq, availableSq *SingleQuery, @@ -91,7 +92,7 @@ func CalculateWhetherStoragePressure(tac *v1alpha1.TidbClusterAutoScaler, capaci newStatus = oldStatus } else { newStatus = &v1alpha1.MetricsStatus{ - Name: string(MetricTypeStorage), + Name: string(corev1.ResourceStorage), StorageMetricsStatus: storageMetrics, } } diff --git a/pkg/autoscaler/autoscaler/calculate/util.go b/pkg/autoscaler/autoscaler/calculate/util.go index 25e7c863f96..72d47f8194c 100644 --- a/pkg/autoscaler/autoscaler/calculate/util.go +++ b/pkg/autoscaler/autoscaler/calculate/util.go @@ -22,14 +22,6 @@ import ( corev1 "k8s.io/api/core/v1" ) -// MetricType describe the current Supported Metric Type to calculate the recommended Replicas -type MetricType string - -const ( - MetricTypeCPU MetricType = "cpu" - MetricTypeStorage MetricType = "storage" -) - // currently, we only choose one metrics to be computed. // If there exists several metrics, we tend to choose ResourceMetricSourceType metric func FilterMetrics(metrics []v1alpha1.CustomMetrics, name corev1.ResourceName) []v1alpha1.CustomMetrics { @@ -43,14 +35,6 @@ func FilterMetrics(metrics []v1alpha1.CustomMetrics, name corev1.ResourceName) [ return list } -// genMetricType return the supported MetricType in Operator by kubernetes auto-scaling MetricType -func GenMetricType(tac *v1alpha1.TidbClusterAutoScaler, metric autoscalingv2beta2.MetricSpec) (MetricType, error) { - if metric.Type == autoscalingv2beta2.ResourceMetricSourceType && metric.Resource != nil && metric.Resource.Name == corev1.ResourceCPU { - return MetricTypeCPU, nil - } - return "", fmt.Errorf(InvalidTacMetricConfigureMsg, tac.Namespace, tac.Name) -} - // filterContainer is to filter the specific container from the given statefulset(tidb/tikv) func filterContainer(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.StatefulSet, containerName string) (*corev1.Container, error) { for _, c := range sts.Spec.Template.Spec.Containers { diff --git a/pkg/autoscaler/autoscaler/tikv_autoscaler.go b/pkg/autoscaler/autoscaler/tikv_autoscaler.go index 921a5ad87f4..204d3fbea62 100644 --- a/pkg/autoscaler/autoscaler/tikv_autoscaler.go +++ b/pkg/autoscaler/autoscaler/tikv_autoscaler.go @@ -84,13 +84,13 @@ func calculateTiKVMetrics(tac *v1alpha1.TidbClusterAutoScaler, tc *v1alpha1.Tidb Instances: instances, Quary: fmt.Sprintf(calculate.TikvSumStorageMetricsPattern, tac.Spec.Cluster.Name, "capacity"), } - avaiableSq := &calculate.SingleQuery{ + availableSq := &calculate.SingleQuery{ Endpoint: ep, Timestamp: now, Instances: instances, Quary: fmt.Sprintf(calculate.TikvSumStorageMetricsPattern, tac.Spec.Cluster.Name, "available"), } - return calculateTiKVStorageMetrics(tac, tc, capacitySq, avaiableSq, client, metrics[0]) + return calculateTiKVStorageMetrics(tac, tc, capacitySq, availableSq, client, metrics[0]) } // check CPU @@ -114,22 +114,22 @@ func calculateTiKVStorageMetrics(tac *v1alpha1.TidbClusterAutoScaler, tc *v1alph klog.V(4).Infof("tac[%s/%s]'s tikv won't scale out by storage pressure due to maxReplicas", tac.Namespace, tac.Name) return nil } - storagePressure, err := calculate.CalculateWhetherStoragePressure(tac, capSq, avaSq, client, metric) + intervalSeconds := tac.Spec.TiKV.ScaleOutIntervalSeconds + ableToScale, err := checkTiKVAutoScalingInterval(tac, *intervalSeconds) if err != nil { return err } - if !storagePressure { + if !ableToScale { return nil } - ableToScale, err := checkWhetherAbleToScaleDueToStorage(tac, metric) + storagePressure, err := calculate.CalculateWhetherStoragePressure(tac, capSq, avaSq, client, metric) if err != nil { return err } - if !ableToScale { + if !storagePressure { return nil } - intervalSeconds := tac.Spec.TiKV.ScaleInIntervalSeconds - ableToScale, err = checkTiKVAutoScaling(tac, *intervalSeconds) + ableToScale, err = checkWhetherAbleToScaleDueToStorage(tac, metric) if err != nil { return err } @@ -138,6 +138,7 @@ func calculateTiKVStorageMetrics(tac *v1alpha1.TidbClusterAutoScaler, tc *v1alph } currentReplicas := tc.Spec.TiKV.Replicas targetReplicas := currentReplicas + 1 + emptyStoragePressureStatus(tac) return updateTacIfTiKVScale(tc, tac, targetReplicas) } @@ -154,7 +155,7 @@ func calculateTiKVCPUMetrics(tac *v1alpha1.TidbClusterAutoScaler, tc *v1alpha1.T } currentReplicas := int32(len(sq.Instances)) intervalSeconds := tac.Spec.TiKV.ScaleInIntervalSeconds - ableToScale, err := checkTiKVAutoScaling(tac, *intervalSeconds) + ableToScale, err := checkTiKVAutoScalingInterval(tac, *intervalSeconds) if err != nil { return err } @@ -168,7 +169,10 @@ func calculateTiKVCPUMetrics(tac *v1alpha1.TidbClusterAutoScaler, tc *v1alpha1.T return addAnnotationMarkIfScaleOutDueToCPUMetrics(tc, currentReplicas, targetReplicas, sts) } -func checkTiKVAutoScaling(tac *v1alpha1.TidbClusterAutoScaler, intervalSeconds int32) (bool, error) { +// checkTiKVAutoScalingInterval check the each 2 auto-scaling interval depends on the scaling-in and scaling-out +// Note that for the storage scaling, we will check scale-out interval before we start to scraping metrics, +// and for the cpu scaling, we will check scale-in/scale-out interval after we finish calculating metrics. +func checkTiKVAutoScalingInterval(tac *v1alpha1.TidbClusterAutoScaler, intervalSeconds int32) (bool, error) { if tac.Annotations == nil { tac.Annotations = map[string]string{} } @@ -183,6 +187,8 @@ func checkTiKVAutoScaling(tac *v1alpha1.TidbClusterAutoScaler, intervalSeconds i } // TODO: add unit test +// checkWhetherAbleToScaleDueToStorage will check whether the storage pressure status have been existed for as least +// LeastStoragePressurePeriodSeconds duration. If not, the operator would wait next round to check again. func checkWhetherAbleToScaleDueToStorage(tac *v1alpha1.TidbClusterAutoScaler, metric v1alpha1.CustomMetrics) (bool, error) { if metric.LeastStoragePressurePeriodSeconds == nil { return false, fmt.Errorf("tac[%s/%s]'s leastStoragePressurePeriodSeconds must be setted before scale out in storage", tac.Namespace, tac.Name) @@ -200,7 +206,7 @@ func checkWhetherAbleToScaleDueToStorage(tac *v1alpha1.TidbClusterAutoScaler, me return false, nil } -// we record the auto-scaling out slot for tikv, in order to add special hot labels when they are created +// updateTacIfTiKVScale update the tac status and syncing annotations if tikv scale-in/out func updateTacIfTiKVScale(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler, recommendedReplicas int32) error { tac.Annotations[label.AnnTiKVLastAutoScalingTimestamp] = fmt.Sprintf("%d", time.Now().Unix()) tc.Spec.TiKV.Replicas = recommendedReplicas diff --git a/pkg/autoscaler/autoscaler/util.go b/pkg/autoscaler/autoscaler/util.go index 1fd223f0257..c9886b029ef 100644 --- a/pkg/autoscaler/autoscaler/util.go +++ b/pkg/autoscaler/autoscaler/util.go @@ -125,14 +125,15 @@ func defaultTAC(tac *v1alpha1.TidbClusterAutoScaler) { tac.Spec.TiKV.MetricsTimeDuration = pointer.StringPtr("3m") } } - for _, m := range tac.Spec.TiKV.Metrics { + for id, m := range tac.Spec.TiKV.Metrics { if m.Resource != nil && m.Resource.Name == corev1.ResourceStorage { if m.LeastStoragePressurePeriodSeconds == nil { m.LeastStoragePressurePeriodSeconds = pointer.Int64Ptr(300) } if m.LeastRemainAvailableStoragePercent == nil { - m.LeastRemainAvailableStoragePercent = pointer.Int64Ptr(90) + m.LeastRemainAvailableStoragePercent = pointer.Int64Ptr(10) } + tac.Spec.TiKV.Metrics[id] = m } } } @@ -170,3 +171,14 @@ func genMetricsEndpoint(tac *v1alpha1.TidbClusterAutoScaler) (string, error) { } return fmt.Sprintf("http://%s-prometheus.%s.svc:9090", tac.Spec.Monitor.Name, tac.Spec.Monitor.Namespace), nil } + +func emptyStoragePressureStatus(tac *v1alpha1.TidbClusterAutoScaler) { + for id, m := range tac.Status.TiKV.MetricsStatusList { + if m.Name == string(corev1.ResourceStorage) { + m.StoragePressure = nil + m.StoragePressureStartTime = nil + tac.Status.TiKV.MetricsStatusList[id] = m + return + } + } +}