From 1f9d2ef52fc0dd551a9c62b2c3c85048d2ba469c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=83=81?= Date: Wed, 21 Apr 2021 12:15:43 +0800 Subject: [PATCH 1/5] add-graphite-scaler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 刘烁 --- pkg/scalers/graphite_api_scaler.go | 166 ++++++++++++++++++++++++++++ pkg/scalers/graphite_scaler_test.go | 65 +++++++++++ pkg/scaling/scale_handler.go | 2 + 3 files changed, 233 insertions(+) create mode 100644 pkg/scalers/graphite_api_scaler.go create mode 100644 pkg/scalers/graphite_scaler_test.go diff --git a/pkg/scalers/graphite_api_scaler.go b/pkg/scalers/graphite_api_scaler.go new file mode 100644 index 00000000000..ec1e9b7b03a --- /dev/null +++ b/pkg/scalers/graphite_api_scaler.go @@ -0,0 +1,166 @@ +package scalers + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + url_pkg "net/url" + "strconv" + + "github.com/tidwall/gjson" + v2beta2 "k8s.io/api/autoscaling/v2beta2" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/metrics/pkg/apis/external_metrics" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + kedautil "github.com/kedacore/keda/v2/pkg/util" +) + +const ( + grapServerAddress = "serverAddress" + grapMetricName = "metricName" + grapQuery = "query" + grapThreshold = "threshold" + grapqueryTime = "queryTime" +) + +type graphiteScaler struct { + metadata *graphiteMetadata +} + +type graphiteMetadata struct { + serverAddress string + metricName string + query string + threshold int + from string +} + +var graphiteLog = logf.Log.WithName("graphite_scaler") + +// NewGraphiteScaler creates a new graphiteScaler +func NewGraphiteScaler(config *ScalerConfig) (Scaler, error) { + meta, err := parseGraphiteMetadata(config) + if err != nil { + return nil, fmt.Errorf("error parsing graphite metadata: %s", err) + } + + return &graphiteScaler{ + metadata: meta, + }, nil +} + +func parseGraphiteMetadata(config *ScalerConfig) (*graphiteMetadata, error) { + meta := graphiteMetadata{} + + if val, ok := config.TriggerMetadata[grapServerAddress]; ok && val != "" { + meta.serverAddress = val + } else { + return nil, fmt.Errorf("no %s given", grapServerAddress) + } + + if val, ok := config.TriggerMetadata[grapQuery]; ok && val != "" { + meta.query = val + } else { + return nil, fmt.Errorf("no %s given", grapQuery) + } + + if val, ok := config.TriggerMetadata[grapMetricName]; ok && val != "" { + meta.metricName = val + } else { + return nil, fmt.Errorf("no %s given", grapMetricName) + } + + if val, ok := config.TriggerMetadata[grapqueryTime]; ok && val != "" { + meta.from = val + } else { + return nil, fmt.Errorf("no %s given", grapqueryTime) + } + + if val, ok := config.TriggerMetadata[grapThreshold]; ok && val != "" { + t, err := strconv.Atoi(val) + if err != nil { + return nil, fmt.Errorf("error parsing %s: %s", grapThreshold, err) + } + + meta.threshold = t + } + + return &meta, nil +} + +func (s *graphiteScaler) IsActive(ctx context.Context) (bool, error) { + val, err := s.ExecuteGrapQuery() + if err != nil { + graphiteLog.Error(err, "error executing graphite query") + return false, err + } + + return val > 0, nil +} + +func (s *graphiteScaler) Close() error { + return nil +} + +func (s *graphiteScaler) GetMetricSpecForScaling() []v2beta2.MetricSpec { + targetMetricValue := resource.NewQuantity(int64(s.metadata.threshold), resource.DecimalSI) + externalMetric := &v2beta2.ExternalMetricSource{ + Metric: v2beta2.MetricIdentifier{ + Name: kedautil.NormalizeString(fmt.Sprintf("%s-%s-%s", "graphite", s.metadata.serverAddress, s.metadata.metricName)), + }, + Target: v2beta2.MetricTarget{ + Type: v2beta2.AverageValueMetricType, + AverageValue: targetMetricValue, + }, + } + metricSpec := v2beta2.MetricSpec{ + External: externalMetric, Type: externalMetricType, + } + return []v2beta2.MetricSpec{metricSpec} +} + +func (s *graphiteScaler) ExecuteGrapQuery() (float64, error) { + queryEscaped := url_pkg.QueryEscape(s.metadata.query) + url := fmt.Sprintf("%s/render?target=%s&format=json", s.metadata.serverAddress, queryEscaped) + r, err := http.Get(url) + if err != nil { + return -1, err + } + + b, err := ioutil.ReadAll(r.Body) + if err != nil { + return -1, err + } + r.Body.Close() + + result := gjson.GetBytes(b, "0.datapoints.#.0") + var v float64 = -1 + for _, valur := range result.Array() { + if valur.String() != "" { + if float64(valur.Int()) > v { + v = float64(valur.Int()) + } + } + } + return v, nil +} + +func (s *graphiteScaler) GetMetrics(ctx context.Context, metricName string, metricSelector labels.Selector) ([]external_metrics.ExternalMetricValue, error) { + val, err := s.ExecuteGrapQuery() + if err != nil { + graphiteLog.Error(err, "error executing graphite query") + return []external_metrics.ExternalMetricValue{}, err + } + + metric := external_metrics.ExternalMetricValue{ + MetricName: metricName, + Value: *resource.NewQuantity(int64(val), resource.DecimalSI), + Timestamp: metav1.Now(), + } + + return append([]external_metrics.ExternalMetricValue{}, metric), nil +} diff --git a/pkg/scalers/graphite_scaler_test.go b/pkg/scalers/graphite_scaler_test.go new file mode 100644 index 00000000000..40bbc5ffd46 --- /dev/null +++ b/pkg/scalers/graphite_scaler_test.go @@ -0,0 +1,65 @@ +package scalers + +import ( + "testing" +) + +type parseGraphiteMetadataTestData struct { + metadata map[string]string + isError bool +} + +type graphiteMetricIdentifier struct { + metadataTestData *parseGraphiteMetadataTestData + name string +} + +var testGrapMetadata = []parseGraphiteMetadataTestData{ + {map[string]string{}, true}, + // all properly formed + {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "up", "disableScaleToZero": "true"}, false}, + // missing serverAddress + {map[string]string{"grapServerAddress": "", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "up", "disableScaleToZero": "true"}, true}, + // missing metricName + {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "", "threshold": "100", "grapQuery": "up", "disableScaleToZero": "true"}, true}, + // malformed threshold + {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "one", "grapQuery": "up", "disableScaleToZero": "true"}, true}, + // missing query + {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "", "disableScaleToZero": "true"}, true}, + // all properly formed, default disableScaleToZero + {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "up"}, false}, +} + +var graphiteMetricIdentifiers = []graphiteMetricIdentifier{ + {&testGrapMetadata[1], "graphite-http---localhost-9090-http_requests_total"}, +} + +func TestGraphiteParseMetadata(t *testing.T) { + for _, testData := range testGrapMetadata { + _, err := parseGraphiteMetadata(&ScalerConfig{TriggerMetadata: testData.metadata}) + if err != nil && !testData.isError { + t.Error("Expected success but got error", err) + } + if testData.isError && err == nil { + t.Error("Expected error but got success") + } + } +} + +func TestGraphiteGetMetricSpecForScaling(t *testing.T) { + for _, testData := range graphiteMetricIdentifiers { + meta, err := parseGraphiteMetadata(&ScalerConfig{TriggerMetadata: testData.metadataTestData.metadata}) + if err != nil { + t.Fatal("Could not parse metadata:", err) + } + mockGraphiteScaler := graphiteScaler{ + metadata: meta, + } + + metricSpec := mockGraphiteScaler.GetMetricSpecForScaling() + metricName := metricSpec[0].External.Metric.Name + if metricName != testData.name { + t.Error("Wrong External metric source name:", metricName) + } + } +} diff --git a/pkg/scaling/scale_handler.go b/pkg/scaling/scale_handler.go index 1adc62e18e6..0ce9c875849 100644 --- a/pkg/scaling/scale_handler.go +++ b/pkg/scaling/scale_handler.go @@ -481,6 +481,8 @@ func buildScaler(triggerType string, config *scalers.ScalerConfig) (scalers.Scal return scalers.NewExternalPushScaler(config) case "gcp-pubsub": return scalers.NewPubSubScaler(config) + case "graphite": + return scalers.NewGraphiteScaler(config) case "huawei-cloudeye": return scalers.NewHuaweiCloudeyeScaler(config) case "ibmmq": From df3695fa902db38996bf7a6bd86c79dd823b58bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=83=81?= Date: Wed, 21 Apr 2021 13:21:21 +0800 Subject: [PATCH 2/5] fix-graphite-test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 刘烁 --- pkg/scalers/graphite_scaler_test.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/scalers/graphite_scaler_test.go b/pkg/scalers/graphite_scaler_test.go index 40bbc5ffd46..b51de1cb05a 100644 --- a/pkg/scalers/graphite_scaler_test.go +++ b/pkg/scalers/graphite_scaler_test.go @@ -17,21 +17,23 @@ type graphiteMetricIdentifier struct { var testGrapMetadata = []parseGraphiteMetadataTestData{ {map[string]string{}, true}, // all properly formed - {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "up", "disableScaleToZero": "true"}, false}, + {map[string]string{"serverAddress": "http://localhost:81", "metricName": "request-count", "threshold": "100", "query": "stats.counters.http.hello-world.request.count.count", "queryTime": "-30Seconds", "disableScaleToZero": "true"}, false}, // missing serverAddress - {map[string]string{"grapServerAddress": "", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "up", "disableScaleToZero": "true"}, true}, + {map[string]string{"serverAddress": "", "grapmetricName": "request-count", "threshold": "100", "query": "stats.counters.http.hello-world.request.count.count", "queryTime": "-30Seconds", "disableScaleToZero": "true"}, true}, // missing metricName - {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "", "threshold": "100", "grapQuery": "up", "disableScaleToZero": "true"}, true}, + {map[string]string{"serverAddress": "http://localhost:81", "metricName": "", "threshold": "100", "query": "stats.counters.http.hello-world.request.count.count", "queryTime": "-30Seconds", "disableScaleToZero": "true"}, true}, // malformed threshold - {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "one", "grapQuery": "up", "disableScaleToZero": "true"}, true}, + {map[string]string{"serverAddress": "http://localhost:81", "metricName": "request-count", "threshold": "one", "query": "stats.counters.http.hello-world.request.count.count", "queryTime": "-30Seconds", "disableScaleToZero": "true"}, true}, // missing query - {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "", "disableScaleToZero": "true"}, true}, + {map[string]string{"serverAddress": "http://localhost:81", "metricName": "request-count", "threshold": "100", "query": "", "queryTime": "-30Seconds", "disableScaleToZero": "true"}, true}, + // missing queryTime + {map[string]string{"serverAddress": "http://localhost:81", "metricName": "request-count", "threshold": "100", "query": "stats.counters.http.hello-world.request.count.count", "queryTime": "", "disableScaleToZero": "true"}, true}, // all properly formed, default disableScaleToZero - {map[string]string{"grapServerAddress": "http://localhost:81", "grapMetricName": "stats.counters.http.hello-world.request.count.count", "threshold": "100", "grapQuery": "up"}, false}, + {map[string]string{"serverAddress": "http://localhost:81", "metricName": "request-count", "threshold": "100", "queryTime": "-30Seconds", "query": "stats.counters.http.hello-world.request.count.count"}, false}, } var graphiteMetricIdentifiers = []graphiteMetricIdentifier{ - {&testGrapMetadata[1], "graphite-http---localhost-9090-http_requests_total"}, + {&testGrapMetadata[1], "graphite-http---localhost-81-request-count"}, } func TestGraphiteParseMetadata(t *testing.T) { From bdce0987018ce69b46d92b6d2c91c7f7ac2cfa14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=83=81?= Date: Fri, 30 Apr 2021 20:40:04 +0800 Subject: [PATCH 3/5] rename to graphite_scaler.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 刘烁 --- pkg/scalers/{graphite_api_scaler.go => graphite_scaler.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/scalers/{graphite_api_scaler.go => graphite_scaler.go} (100%) diff --git a/pkg/scalers/graphite_api_scaler.go b/pkg/scalers/graphite_scaler.go similarity index 100% rename from pkg/scalers/graphite_api_scaler.go rename to pkg/scalers/graphite_scaler.go From 1f9a483eb0aa1e94e18957880ac810ae59a860ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=83=81?= Date: Mon, 10 May 2021 10:17:04 +0800 Subject: [PATCH 4/5] add-basic-auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 刘烁 --- CHANGELOG.md | 1 + pkg/scalers/graphite_scaler.go | 50 +++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c72e2649f1..80c9ca4a380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ## Unreleased +- Add Graphite Scaler ([#1749](https://github.com/kedacore/keda/pull/1749)) - Add Azure Pipelines Scaler ([#1706](https://github.com/kedacore/keda/pull/1706)) - Add OpenStack Metrics Scaler ([#1382](https://github.com/kedacore/keda/issues/1382)) - Fixed goroutine leaks in usage of timers ([#1704](https://github.com/kedacore/keda/pull/1704) | [#1739](https://github.com/kedacore/keda/pull/1739)) diff --git a/pkg/scalers/graphite_scaler.go b/pkg/scalers/graphite_scaler.go index ec1e9b7b03a..d3a04f65631 100644 --- a/pkg/scalers/graphite_scaler.go +++ b/pkg/scalers/graphite_scaler.go @@ -2,11 +2,15 @@ package scalers import ( "context" + "errors" "fmt" + + "io/ioutil" "net/http" url_pkg "net/url" "strconv" + "strings" "github.com/tidwall/gjson" v2beta2 "k8s.io/api/autoscaling/v2beta2" @@ -16,6 +20,7 @@ import ( "k8s.io/metrics/pkg/apis/external_metrics" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" kedautil "github.com/kedacore/keda/v2/pkg/util" ) @@ -37,6 +42,11 @@ type graphiteMetadata struct { query string threshold int from string + + // basic auth + enableBasicAuth bool + username string + password string // +optional } var graphiteLog = logf.Log.WithName("graphite_scaler") @@ -47,7 +57,6 @@ func NewGraphiteScaler(config *ScalerConfig) (Scaler, error) { if err != nil { return nil, fmt.Errorf("error parsing graphite metadata: %s", err) } - return &graphiteScaler{ metadata: meta, }, nil @@ -89,6 +98,31 @@ func parseGraphiteMetadata(config *ScalerConfig) (*graphiteMetadata, error) { meta.threshold = t } + authModes, ok := config.TriggerMetadata["authModes"] + // no authMode specified + if !ok { + return &meta, nil + } + + authTypes := strings.Split(authModes, ",") + for _, t := range authTypes { + authType := authentication.Type(strings.TrimSpace(t)) + switch authType { + case authentication.BasicAuthType: + if len(config.AuthParams["username"]) == 0 { + return nil, errors.New("no username given") + } + + meta.username = config.AuthParams["username"] + // password is optional. For convenience, many application implement basic auth with + // username as apikey and password as empty + meta.password = config.AuthParams["password"] + meta.enableBasicAuth = true + default: + return nil, fmt.Errorf("err incorrect value for authMode is given: %s", t) + } + } + return &meta, nil } @@ -124,9 +158,17 @@ func (s *graphiteScaler) GetMetricSpecForScaling() []v2beta2.MetricSpec { } func (s *graphiteScaler) ExecuteGrapQuery() (float64, error) { + client := &http.Client{} queryEscaped := url_pkg.QueryEscape(s.metadata.query) - url := fmt.Sprintf("%s/render?target=%s&format=json", s.metadata.serverAddress, queryEscaped) - r, err := http.Get(url) + url := fmt.Sprintf("%s/render?from=%s&target=%s&format=json", s.metadata.serverAddress, s.metadata.from, queryEscaped) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return -1, err + } + if s.metadata.enableBasicAuth { + req.SetBasicAuth(s.metadata.username, s.metadata.password) + } + r, err := client.Do(req) if err != nil { return -1, err } @@ -163,4 +205,4 @@ func (s *graphiteScaler) GetMetrics(ctx context.Context, metricName string, metr } return append([]external_metrics.ExternalMetricValue{}, metric), nil -} +} \ No newline at end of file From 6f8641b84be1808d074b48b8a736f074a38ed023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=83=81?= Date: Mon, 10 May 2021 10:55:02 +0800 Subject: [PATCH 5/5] fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 刘烁 --- pkg/scalers/graphite_scaler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/scalers/graphite_scaler.go b/pkg/scalers/graphite_scaler.go index d3a04f65631..ea5471edb3c 100644 --- a/pkg/scalers/graphite_scaler.go +++ b/pkg/scalers/graphite_scaler.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - - "io/ioutil" "net/http" url_pkg "net/url" @@ -205,4 +203,4 @@ func (s *graphiteScaler) GetMetrics(ctx context.Context, metricName string, metr } return append([]external_metrics.ExternalMetricValue{}, metric), nil -} \ No newline at end of file +}