From fc181736c5b9a6cbbb316fa332f07278dc9273c3 Mon Sep 17 00:00:00 2001 From: Darren Gibbard Date: Tue, 26 Jul 2022 12:17:10 +0100 Subject: [PATCH 1/8] Datadog Scaler: Add support for multi-query results Signed-off-by: Darren Gibbard --- go.mod | 2 ++ go.sum | 4 +++ pkg/scalers/datadog_scaler.go | 47 ++++++++++++++++++++++++++--------- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d63e7110492..c2d01797e14 100644 --- a/go.mod +++ b/go.mod @@ -118,6 +118,7 @@ require ( github.com/blang/semver v3.5.1+incompatible // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/chenhg5/collection v0.0.0-20200925143926-f403b87088f9 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -218,6 +219,7 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/cobra v1.2.1 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 3cef9c2d530..743561ee41a 100644 --- a/go.sum +++ b/go.sum @@ -224,6 +224,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenhg5/collection v0.0.0-20200925143926-f403b87088f9 h1:JbMO8sTcYXnBWb7in6XVCiO+idRlLBxa5CoBkz2KULs= +github.com/chenhg5/collection v0.0.0-20200925143926-f403b87088f9/go.mod h1:RE3lB6QNf4YUL8Jl/OONdlltQuN9LfZD8eR3nZZdBLA= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -941,6 +943,8 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= diff --git a/pkg/scalers/datadog_scaler.go b/pkg/scalers/datadog_scaler.go index adfccaf0518..042230bfef8 100644 --- a/pkg/scalers/datadog_scaler.go +++ b/pkg/scalers/datadog_scaler.go @@ -270,10 +270,6 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { series := resp.GetSeries() - if len(series) > 1 { - return 0, fmt.Errorf("query returned more than 1 series; modify the query to return only 1 series") - } - if len(series) == 0 { if !s.metadata.useFiller { return 0, fmt.Errorf("no Datadog metrics returned for the given time window") @@ -281,18 +277,45 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { return s.metadata.fillValue, nil } - points := series[0].GetPointlist() + // Collect all latest point values from any/all series + results := make([]float64, len(series)) + for i := 0; i < len(series); i++ { + points := series[i].GetPointlist() + if len(points) == 0 || len(points[0]) < 2 { + if !s.metadata.useFiller { + return 0, fmt.Errorf("no Datadog metrics returned for the given time window") + } + return s.metadata.fillValue, nil + } + // Return the last point from the series + index := len(points) - 1 + results[i] = float64(*points[index][1]) + } - if len(points) == 0 || len(points[0]) < 2 { - if !s.metadata.useFiller { - return 0, fmt.Errorf("no Datadog metrics returned for the given time window") + // Todo: Allow user-defined aggregator and/or set default to Max + // Todo: Use correct aggregator function based on config + // Todo: See if there's methods already available for min/max/avg/median etc, without writing our own. + + // Aggregate Results - default Max value: + return MaxFromSlices(results) +} + +func MaxFromSlices(results []float64) (float64, error) { + max := results[0] + for _, result := range results { + if result > max { + max = result } - return s.metadata.fillValue, nil } + return max, nil +} - // Return the last point from the series - index := len(points) - 1 - return *points[index][1], nil +func AvgFromSlices(results []float64) (float64, error) { + total := 0.0 + for _, result := range results { + total += result + } + return total / float64(len(results)), nil } // GetMetricSpecForScaling returns the MetricSpec for the Horizontal Pod Autoscaler From c3d9347c6c3daa2d09a9ef0d937f11133ad7dd6e Mon Sep 17 00:00:00 2001 From: Darren Gibbard Date: Tue, 26 Jul 2022 12:17:58 +0100 Subject: [PATCH 2/8] Datadog Scaler: Add support for multi-query results Signed-off-by: Darren Gibbard --- pkg/scalers/datadog_scaler.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/scalers/datadog_scaler.go b/pkg/scalers/datadog_scaler.go index 042230bfef8..22e49cbc8cd 100644 --- a/pkg/scalers/datadog_scaler.go +++ b/pkg/scalers/datadog_scaler.go @@ -292,9 +292,12 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { results[i] = float64(*points[index][1]) } - // Todo: Allow user-defined aggregator and/or set default to Max - // Todo: Use correct aggregator function based on config + // Todo: Allow user-defined aggregator and/or set default to Max (config) + // Todo: Use correct aggregator function here, based on config // Todo: See if there's methods already available for min/max/avg/median etc, without writing our own. + // Todo: Add tests + // Todo: Update docs with example / info on aggregation + // Todo: Changelog // Aggregate Results - default Max value: return MaxFromSlices(results) From 62137a0c9e337be48f7bfae0bb6afc03ea3030a6 Mon Sep 17 00:00:00 2001 From: Darren Gibbard Date: Tue, 26 Jul 2022 12:20:18 +0100 Subject: [PATCH 3/8] Removed unneeded changes Signed-off-by: Darren Gibbard --- go.mod | 2 -- go.sum | 4 ---- 2 files changed, 6 deletions(-) diff --git a/go.mod b/go.mod index c2d01797e14..d63e7110492 100644 --- a/go.mod +++ b/go.mod @@ -118,7 +118,6 @@ require ( github.com/blang/semver v3.5.1+incompatible // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chenhg5/collection v0.0.0-20200925143926-f403b87088f9 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -219,7 +218,6 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect - github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/cobra v1.2.1 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 743561ee41a..3cef9c2d530 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenhg5/collection v0.0.0-20200925143926-f403b87088f9 h1:JbMO8sTcYXnBWb7in6XVCiO+idRlLBxa5CoBkz2KULs= -github.com/chenhg5/collection v0.0.0-20200925143926-f403b87088f9/go.mod h1:RE3lB6QNf4YUL8Jl/OONdlltQuN9LfZD8eR3nZZdBLA= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -943,8 +941,6 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= From cc08baf1396148cf9058e0143555708fca43733c Mon Sep 17 00:00:00 2001 From: Darren Gibbard Date: Wed, 27 Jul 2022 15:25:24 +0100 Subject: [PATCH 4/8] Add queryAggregator meta + utils + tests Signed-off-by: Darren Gibbard --- CHANGELOG.md | 1 + pkg/scalers/datadog_scaler.go | 53 +++++++++++++++--------------- pkg/scalers/datadog_scaler_test.go | 6 ++++ pkg/util/slice_helpers.go | 32 ++++++++++++++++++ pkg/util/slice_helpers_test.go | 40 ++++++++++++++++++++++ 5 files changed, 105 insertions(+), 27 deletions(-) create mode 100644 pkg/util/slice_helpers.go create mode 100644 pkg/util/slice_helpers_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cec6c31d2a3..ff8ceedb336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md - **General:** Use `mili` scale for the returned metrics ([#3135](https://github.com/kedacore/keda/issue/3135)) - **General:** Use more readable timestamps in KEDA Operator logs ([#3066](https://github.com/kedacore/keda/issue/3066)) - **AWS SQS Queue Scaler:** Support for scaling to include in-flight messages. ([#3133](https://github.com/kedacore/keda/issues/3133)) +- **Datadog Scaler:** Support multi-query metrics, and aggregation ([#3423](https://github.com/kedacore/keda/issues/3423)) - **GCP Stackdriver Scaler:** Added aggregation parameters ([#3008](https://github.com/kedacore/keda/issues/3008)) - **Prometheus Scaler:** Add ignoreNullValues to return error when prometheus return null in values ([#3065](https://github.com/kedacore/keda/issues/3065)) - **Selenium Grid Scaler:** Edge active sessions not being properly counted ([#2709](https://github.com/kedacore/keda/issues/2709)) diff --git a/pkg/scalers/datadog_scaler.go b/pkg/scalers/datadog_scaler.go index 22e49cbc8cd..799e5fb30a1 100644 --- a/pkg/scalers/datadog_scaler.go +++ b/pkg/scalers/datadog_scaler.go @@ -28,6 +28,7 @@ type datadogMetadata struct { datadogSite string query string queryValue float64 + queryAggegrator string activationQueryValue float64 vType v2beta2.MetricTargetType metricName string @@ -36,6 +37,9 @@ type datadogMetadata struct { fillValue float64 } +const maxString = "max" +const avgString = "average" + var datadogLog = logf.Log.WithName("datadog_scaler") var filter *regexp.Regexp @@ -109,6 +113,20 @@ func parseDatadogMetadata(config *ScalerConfig) (*datadogMetadata, error) { return nil, fmt.Errorf("no queryValue given") } + allowedQueryAggregators := []string{avgString, maxString} + + if val, ok := config.TriggerMetadata["queryAggregator"]; ok && val != "" { + queryAggregator := strings.ToLower(val) + _, found := kedautil.FindStringInSlice(allowedQueryAggregators, queryAggregator) + if found { + meta.queryAggegrator = queryAggregator + } else { + return nil, fmt.Errorf("queryAggregator has to be one of %+q", queryAggregator) + } + } else { + meta.queryAggegrator = maxString + } + meta.activationQueryValue = 0 if val, ok := config.TriggerMetadata["activationQueryValue"]; ok { activationQueryValue, err := strconv.ParseFloat(val, 64) @@ -134,7 +152,7 @@ func parseDatadogMetadata(config *ScalerConfig) (*datadogMetadata, error) { } val = strings.ToLower(val) switch val { - case "average": + case avgString: meta.vType = v2beta2.AverageValueMetricType case "global": meta.vType = v2beta2.ValueMetricType @@ -289,36 +307,17 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { } // Return the last point from the series index := len(points) - 1 - results[i] = float64(*points[index][1]) + results[i] = *points[index][1] } - // Todo: Allow user-defined aggregator and/or set default to Max (config) - // Todo: Use correct aggregator function here, based on config - // Todo: See if there's methods already available for min/max/avg/median etc, without writing our own. - // Todo: Add tests // Todo: Update docs with example / info on aggregation - // Todo: Changelog - - // Aggregate Results - default Max value: - return MaxFromSlices(results) -} - -func MaxFromSlices(results []float64) (float64, error) { - max := results[0] - for _, result := range results { - if result > max { - max = result - } - } - return max, nil -} - -func AvgFromSlices(results []float64) (float64, error) { - total := 0.0 - for _, result := range results { - total += result + switch s.metadata.queryAggegrator { + case avgString: + return kedautil.AvgFloatFromSlice(results), nil + default: + // Aggregate Results - default Max value: + return kedautil.MaxFloatFromSlice(results), nil } - return total / float64(len(results)), nil } // GetMetricSpecForScaling returns the MetricSpec for the Horizontal Pod Autoscaler diff --git a/pkg/scalers/datadog_scaler_test.go b/pkg/scalers/datadog_scaler_test.go index 8a5f07b1fe6..6df17760db3 100644 --- a/pkg/scalers/datadog_scaler_test.go +++ b/pkg/scalers/datadog_scaler_test.go @@ -35,6 +35,8 @@ var testParseQueries = []datadogQueries{ {"top(per_second(abs(sum:http.requests{service:myapp,dc:us-west-2}.rollup(max, 2))), 5, 'mean', 'desc')", true, false}, {"system.cpu.user{*}.rollup(sum, 30)", true, false}, {"min:system.cpu.user{*}", true, false}, + // Multi-query + {"avg:system.cpu.user{*}.rollup(sum, 30),sum:system.cpu.user{*}.rollup(30)", true, false}, // Missing filter {"min:system.cpu.user", false, true}, @@ -62,6 +64,8 @@ var testDatadogMetadata = []datadogAuthMetadataTestData{ // all properly formed {"", map[string]string{"query": "sum:trace.redis.command.hits{env:none,service:redis}.as_count()", "queryValue": "7", "metricUnavailableValue": "1.5", "type": "average", "age": "60"}, map[string]string{"apiKey": "apiKey", "appKey": "appKey", "datadogSite": "datadogSite"}, false}, + // Multi-query all properly formed + {"", map[string]string{"query": "sum:trace.redis.command.hits{env:none,service:redis}.as_count(),sum:trace.redis.command.hits{env:none,service:redis}.as_count()/2", "queryValue": "7", "queryAggregator": "average", "metricUnavailableValue": "1.5", "type": "average", "age": "60"}, map[string]string{"apiKey": "apiKey", "appKey": "appKey", "datadogSite": "datadogSite"}, false}, // default age {"", map[string]string{"query": "sum:trace.redis.command.hits{env:none,service:redis}.as_count()", "queryValue": "7", "type": "average"}, map[string]string{"apiKey": "apiKey", "appKey": "appKey", "datadogSite": "datadogSite"}, false}, // default type @@ -76,6 +80,8 @@ var testDatadogMetadata = []datadogAuthMetadataTestData{ {"", map[string]string{"query": "sum:trace.redis.command.hits{env:none,service:redis}.as_count()", "type": "average", "age": "60"}, map[string]string{"apiKey": "apiKey", "appKey": "appKey", "datadogSite": "datadogSite"}, true}, // wrong query value type {"", map[string]string{"query": "sum:trace.redis.command.hits{env:none,service:redis}.as_count()", "queryValue": "notanint", "type": "average", "age": "60"}, map[string]string{"apiKey": "apiKey", "appKey": "appKey", "datadogSite": "datadogSite"}, true}, + // wrong queryAggregator value + {"", map[string]string{"query": "sum:trace.redis.command.hits{env:none,service:redis}.as_count()", "queryValue": "notanint", "queryAggegrator": "1.0", "type": "average", "age": "60"}, map[string]string{"apiKey": "apiKey", "appKey": "appKey", "datadogSite": "datadogSite"}, true}, // wrong activation query value type {"", map[string]string{"query": "sum:trace.redis.command.hits{env:none,service:redis}.as_count()", "queryValue": "1", "activationQueryValue": "notanint", "type": "average", "age": "60"}, map[string]string{"apiKey": "apiKey", "appKey": "appKey", "datadogSite": "datadogSite"}, true}, // malformed query diff --git a/pkg/util/slice_helpers.go b/pkg/util/slice_helpers.go new file mode 100644 index 00000000000..7790e5d8c84 --- /dev/null +++ b/pkg/util/slice_helpers.go @@ -0,0 +1,32 @@ +package util + +// Takes a slice of strings, and looks for a string in it. If found it will +// return it's key/index, otherwise it will return -1 and a bool of false. +func FindStringInSlice(slice []string, val string) (int, bool) { + for i, item := range slice { + if item == val { + return i, true + } + } + return -1, false +} + +// Find the largest value in a slice of floats +func MaxFloatFromSlice(results []float64) float64 { + max := results[0] + for _, result := range results { + if result > max { + max = result + } + } + return max +} + +// Find the average value in a slice of floats +func AvgFloatFromSlice(results []float64) float64 { + total := 0.0 + for _, result := range results { + total += result + } + return total / float64(len(results)) +} diff --git a/pkg/util/slice_helpers_test.go b/pkg/util/slice_helpers_test.go new file mode 100644 index 00000000000..90bdae28b12 --- /dev/null +++ b/pkg/util/slice_helpers_test.go @@ -0,0 +1,40 @@ +package util + +import ( + "testing" +) + +func assertEqual(t *testing.T, a interface{}, b interface{}) { + if a == b { + return + } + t.Errorf("%v != %v", a, b) +} + +func TestFindStringInSlice(t *testing.T) { + inputSlice := []string{"this", "looks", "for", "strings"} + inputValue := "looks" + expectedIndex, expectedBool := int(1), bool(true) + + outputIndex, outputBool := FindStringInSlice(inputSlice, inputValue) + assertEqual(t, outputIndex, expectedIndex) + assertEqual(t, outputBool, expectedBool) +} + +func TestMaxFloatFromSlice(t *testing.T) { + input := []float64{1.0, 2.0, 3.0, 4.0} + expectedOutput := float64(4.0) + + output := MaxFloatFromSlice(input) + + assertEqual(t, output, expectedOutput) +} + +func TestAvgFloatFromSlice(t *testing.T) { + input := []float64{1.0, 2.0, 3.0, 4.0} + expectedOutput := float64(2.5) + + output := AvgFloatFromSlice(input) + + assertEqual(t, output, expectedOutput) +} From a121b30fce0c7efcd54ebdc6a33ce8d8e0520753 Mon Sep 17 00:00:00 2001 From: Darren Gibbard Date: Tue, 9 Aug 2022 15:09:18 +0100 Subject: [PATCH 5/8] Move operation methods to scaler code for simplicity --- pkg/scalers/datadog_scaler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/scalers/datadog_scaler.go b/pkg/scalers/datadog_scaler.go index 799e5fb30a1..45c1318ad95 100644 --- a/pkg/scalers/datadog_scaler.go +++ b/pkg/scalers/datadog_scaler.go @@ -310,7 +310,6 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { results[i] = *points[index][1] } - // Todo: Update docs with example / info on aggregation switch s.metadata.queryAggegrator { case avgString: return kedautil.AvgFloatFromSlice(results), nil From 37c4ded67bba49ed65fd0263ba9a72eef891d7ff Mon Sep 17 00:00:00 2001 From: Darren Gibbard Date: Tue, 9 Aug 2022 15:18:04 +0100 Subject: [PATCH 6/8] debug Signed-off-by: Darren Gibbard --- pkg/scalers/datadog_scaler.go | 48 +++++++++++++++++++++++++----- pkg/scalers/datadog_scaler_test.go | 35 ++++++++++++++++++++++ pkg/util/slice_helpers.go | 32 -------------------- pkg/util/slice_helpers_test.go | 40 ------------------------- 4 files changed, 75 insertions(+), 80 deletions(-) delete mode 100644 pkg/util/slice_helpers.go delete mode 100644 pkg/util/slice_helpers_test.go diff --git a/pkg/scalers/datadog_scaler.go b/pkg/scalers/datadog_scaler.go index 45c1318ad95..f8c377a752b 100644 --- a/pkg/scalers/datadog_scaler.go +++ b/pkg/scalers/datadog_scaler.go @@ -117,7 +117,7 @@ func parseDatadogMetadata(config *ScalerConfig) (*datadogMetadata, error) { if val, ok := config.TriggerMetadata["queryAggregator"]; ok && val != "" { queryAggregator := strings.ToLower(val) - _, found := kedautil.FindStringInSlice(allowedQueryAggregators, queryAggregator) + _, found := FindStringInSlice(allowedQueryAggregators, queryAggregator) if found { meta.queryAggegrator = queryAggregator } else { @@ -310,13 +310,14 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { results[i] = *points[index][1] } - switch s.metadata.queryAggegrator { - case avgString: - return kedautil.AvgFloatFromSlice(results), nil - default: - // Aggregate Results - default Max value: - return kedautil.MaxFloatFromSlice(results), nil - } + return -1, fmt.Errorf("DEBUG: %v", s.metadata.queryAggegrator) + // switch s.metadata.queryAggegrator { + // case avgString: + // return AvgFloatFromSlice(results), nil + // default: + // // Aggregate Results - default Max value: + // return MaxFloatFromSlice(results), nil + // } } // GetMetricSpecForScaling returns the MetricSpec for the Horizontal Pod Autoscaler @@ -345,3 +346,34 @@ func (s *datadogScaler) GetMetrics(ctx context.Context, metricName string, metri return append([]external_metrics.ExternalMetricValue{}, metric), nil } + +// Takes a slice of strings, and looks for a string in it. If found it will +// return it's key/index, otherwise it will return -1 and a bool of false. +func FindStringInSlice(slice []string, val string) (int, bool) { + for i, item := range slice { + if item == val { + return i, true + } + } + return -1, false +} + +// Find the largest value in a slice of floats +func MaxFloatFromSlice(results []float64) float64 { + max := results[0] + for _, result := range results { + if result > max { + max = result + } + } + return max +} + +// Find the average value in a slice of floats +func AvgFloatFromSlice(results []float64) float64 { + total := 0.0 + for _, result := range results { + total += result + } + return total / float64(len(results)) +} diff --git a/pkg/scalers/datadog_scaler_test.go b/pkg/scalers/datadog_scaler_test.go index 6df17760db3..182d262f245 100644 --- a/pkg/scalers/datadog_scaler_test.go +++ b/pkg/scalers/datadog_scaler_test.go @@ -26,6 +26,41 @@ type datadogAuthMetadataTestData struct { isError bool } +func assertEqual(t *testing.T, a interface{}, b interface{}) { + if a == b { + return + } + t.Errorf("%v != %v", a, b) +} + +func TestFindStringInSlice(t *testing.T) { + inputSlice := []string{"this", "looks", "for", "strings"} + inputValue := "looks" + expectedIndex, expectedBool := int(1), bool(true) + + outputIndex, outputBool := FindStringInSlice(inputSlice, inputValue) + assertEqual(t, outputIndex, expectedIndex) + assertEqual(t, outputBool, expectedBool) +} + +func TestMaxFloatFromSlice(t *testing.T) { + input := []float64{1.0, 2.0, 3.0, 4.0} + expectedOutput := float64(4.0) + + output := MaxFloatFromSlice(input) + + assertEqual(t, output, expectedOutput) +} + +func TestAvgFloatFromSlice(t *testing.T) { + input := []float64{1.0, 2.0, 3.0, 4.0} + expectedOutput := float64(2.5) + + output := AvgFloatFromSlice(input) + + assertEqual(t, output, expectedOutput) +} + var testParseQueries = []datadogQueries{ {"", false, true}, // All properly formed diff --git a/pkg/util/slice_helpers.go b/pkg/util/slice_helpers.go deleted file mode 100644 index 7790e5d8c84..00000000000 --- a/pkg/util/slice_helpers.go +++ /dev/null @@ -1,32 +0,0 @@ -package util - -// Takes a slice of strings, and looks for a string in it. If found it will -// return it's key/index, otherwise it will return -1 and a bool of false. -func FindStringInSlice(slice []string, val string) (int, bool) { - for i, item := range slice { - if item == val { - return i, true - } - } - return -1, false -} - -// Find the largest value in a slice of floats -func MaxFloatFromSlice(results []float64) float64 { - max := results[0] - for _, result := range results { - if result > max { - max = result - } - } - return max -} - -// Find the average value in a slice of floats -func AvgFloatFromSlice(results []float64) float64 { - total := 0.0 - for _, result := range results { - total += result - } - return total / float64(len(results)) -} diff --git a/pkg/util/slice_helpers_test.go b/pkg/util/slice_helpers_test.go deleted file mode 100644 index 90bdae28b12..00000000000 --- a/pkg/util/slice_helpers_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package util - -import ( - "testing" -) - -func assertEqual(t *testing.T, a interface{}, b interface{}) { - if a == b { - return - } - t.Errorf("%v != %v", a, b) -} - -func TestFindStringInSlice(t *testing.T) { - inputSlice := []string{"this", "looks", "for", "strings"} - inputValue := "looks" - expectedIndex, expectedBool := int(1), bool(true) - - outputIndex, outputBool := FindStringInSlice(inputSlice, inputValue) - assertEqual(t, outputIndex, expectedIndex) - assertEqual(t, outputBool, expectedBool) -} - -func TestMaxFloatFromSlice(t *testing.T) { - input := []float64{1.0, 2.0, 3.0, 4.0} - expectedOutput := float64(4.0) - - output := MaxFloatFromSlice(input) - - assertEqual(t, output, expectedOutput) -} - -func TestAvgFloatFromSlice(t *testing.T) { - input := []float64{1.0, 2.0, 3.0, 4.0} - expectedOutput := float64(2.5) - - output := AvgFloatFromSlice(input) - - assertEqual(t, output, expectedOutput) -} From f08cc53d6a0d85486233775c69ff38e491cfa8a3 Mon Sep 17 00:00:00 2001 From: Darren Gibbard Date: Tue, 9 Aug 2022 16:51:42 +0100 Subject: [PATCH 7/8] Working, but not working --- pkg/scalers/datadog_scaler.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/scalers/datadog_scaler.go b/pkg/scalers/datadog_scaler.go index f8c377a752b..487c9c1574f 100644 --- a/pkg/scalers/datadog_scaler.go +++ b/pkg/scalers/datadog_scaler.go @@ -310,14 +310,13 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { results[i] = *points[index][1] } - return -1, fmt.Errorf("DEBUG: %v", s.metadata.queryAggegrator) - // switch s.metadata.queryAggegrator { - // case avgString: - // return AvgFloatFromSlice(results), nil - // default: - // // Aggregate Results - default Max value: - // return MaxFloatFromSlice(results), nil - // } + switch s.metadata.queryAggegrator { + case avgString: + return AvgFloatFromSlice(results), nil + default: + // Aggregate Results - default Max value: + return MaxFloatFromSlice(results), nil + } } // GetMetricSpecForScaling returns the MetricSpec for the Horizontal Pod Autoscaler From 29d1c517592438d4be6e5ac57b7713dc96431621 Mon Sep 17 00:00:00 2001 From: dgibbard-cisco <57677847+dgibbard-cisco@users.noreply.github.com> Date: Wed, 10 Aug 2022 10:18:57 +0100 Subject: [PATCH 8/8] Update CHANGELOG.md Signed-off-by: dgibbard-cisco <57677847+dgibbard-cisco@users.noreply.github.com> --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ac7848a577..bba5f8cf694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md ### Improvements -- TODO ([#XXX](https://github.com/kedacore/keda/issue/XXX)) +- **Datadog Scaler:** Support multi-query metrics, and aggregation ([#3423](https://github.com/kedacore/keda/issues/3423)) ### Fixes @@ -86,7 +86,6 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md - **General:** Reference ScaledObject's/ScaledJob's name in the scalers log ([3419](https://github.com/kedacore/keda/issues/3419)) - **General:** Use `mili` scale for the returned metrics ([#3135](https://github.com/kedacore/keda/issue/3135)) - **General:** Use more readable timestamps in KEDA Operator logs ([#3066](https://github.com/kedacore/keda/issue/3066)) -- **Datadog Scaler:** Support multi-query metrics, and aggregation ([#3423](https://github.com/kedacore/keda/issues/3423)) - **Kafka Scaler:** Handle Sarama errors properly ([#3056](https://github.com/kedacore/keda/issues/3056)) ### Fixes