diff --git a/expr/functions/holtWintersAberration/function.go b/expr/functions/holtWintersAberration/function.go index 035077062..c0082a188 100644 --- a/expr/functions/holtWintersAberration/function.go +++ b/expr/functions/holtWintersAberration/function.go @@ -31,44 +31,60 @@ func New(configFile string) []interfaces.FunctionMetadata { } func (f *holtWintersAberration) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { - bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, 7*86400) + bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, holtwinters.DefaultBootstrapInterval) if err != nil { return nil, err } - args, err := helper.GetSeriesArg(ctx, e.Arg(0), from-bootstrapInterval, until, values) + args, err := helper.GetSeriesArg(ctx, e.Arg(0), from, until, values) if err != nil { return nil, err } + // Note: additional fetch requests are added with an adjusted start time in expr.Metrics() (in + // pkg/parser/parser.go) so that the appropriate data corresponding to the adjusted start time + // can be pre-fetched. + adjustedStartArgs, err := helper.GetSeriesArg(ctx, e.Arg(0), from-bootstrapInterval, until, values) + if err != nil { + return nil, err + } + + adjustedStartSeries := make(map[string]*types.MetricData) + for _, serie := range adjustedStartArgs { + adjustedStartSeries[serie.Name] = serie + } + delta, err := e.GetFloatNamedOrPosArgDefault("delta", 1, 3) if err != nil { return nil, err } + seasonality, err := e.GetIntervalNamedOrPosArgDefault("seasonality", 3, 1, holtwinters.DefaultSeasonality) + if err != nil { + return nil, err + } + results := make([]*types.MetricData, 0, len(args)) for _, arg := range args { var ( aberration []float64 - series []float64 ) stepTime := arg.StepTime - lowerBand, upperBand := holtwinters.HoltWintersConfidenceBands(arg.Values, stepTime, delta, bootstrapInterval/86400) - - windowPoints := int(bootstrapInterval / stepTime) - if len(arg.Values) > windowPoints { - series = arg.Values[windowPoints:] + if v, ok := adjustedStartSeries[arg.Name]; !ok || v == nil { + continue } - for i := range series { - if math.IsNaN(series[i]) { + lowerBand, upperBand := holtwinters.HoltWintersConfidenceBands(adjustedStartSeries[arg.Name].Values, stepTime, delta, bootstrapInterval, seasonality) + + for i, v := range arg.Values { + if math.IsNaN(v) { aberration = append(aberration, 0) - } else if !math.IsNaN(upperBand[i]) && series[i] > upperBand[i] { - aberration = append(aberration, series[i]-upperBand[i]) - } else if !math.IsNaN(lowerBand[i]) && series[i] < lowerBand[i] { - aberration = append(aberration, series[i]-lowerBand[i]) + } else if !math.IsNaN(upperBand[i]) && v > upperBand[i] { + aberration = append(aberration, v-upperBand[i]) + } else if !math.IsNaN(lowerBand[i]) && v < lowerBand[i] { + aberration = append(aberration, v-lowerBand[i]) } else { aberration = append(aberration, 0) } @@ -80,7 +96,7 @@ func (f *holtWintersAberration) Do(ctx context.Context, e parser.Expr, from, unt Name: name, Values: aberration, StepTime: arg.StepTime, - StartTime: arg.StartTime + bootstrapInterval, + StartTime: arg.StartTime, StopTime: arg.StopTime, PathExpression: name, ConsolidationFunc: arg.ConsolidationFunc, diff --git a/expr/functions/holtWintersAberration/function_test.go b/expr/functions/holtWintersAberration/function_test.go new file mode 100644 index 000000000..ed4849866 --- /dev/null +++ b/expr/functions/holtWintersAberration/function_test.go @@ -0,0 +1,126 @@ +package holtWintersAberration + +import ( + "github.com/go-graphite/carbonapi/expr/holtwinters" + "testing" + + "github.com/go-graphite/carbonapi/expr/helper" + "github.com/go-graphite/carbonapi/expr/metadata" + "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/parser" + th "github.com/go-graphite/carbonapi/tests" +) + +func init() { + md := New("") + evaluator := th.EvaluatorFromFunc(md[0].F) + metadata.SetEvaluator(evaluator) + helper.SetEvaluator(evaluator) + for _, m := range md { + metadata.RegisterFunction(m.Name, m.F) + } +} + +func TestHoltWintersAberration(t *testing.T) { + var startTime int64 = 2678400 + var step int64 = 600 + var points int64 = 10 + var seconds int64 = 86400 + + tests := []th.EvalTestItemWithRange{ + { + Target: "holtWintersAberration(metric*)", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric*", startTime, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, points*step, step, 0), step, startTime), + types.MakeMetricData("metric2", generateHwRange(0, points*step, step, 10), step, startTime), + }, + {"metric*", startTime - holtwinters.DefaultBootstrapInterval, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, ((holtwinters.DefaultBootstrapInterval/step)+points)*step, step, 0), step, startTime-holtwinters.DefaultBootstrapInterval), + types.MakeMetricData("metric2", generateHwRange(0, ((holtwinters.DefaultBootstrapInterval/step)+points)*step, step, 10), step, startTime-holtwinters.DefaultBootstrapInterval), + }, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersAberration(metric1)", []float64{-0.2841206166091448, -0.05810270987744115, 0, 0, 0, 0, 0, 0, 0, 0}, step, startTime).SetTag("holtWintersAberration", "1"), + types.MakeMetricData("holtWintersAberration(metric2)", []float64{-0.284120616609151, -0.05810270987744737, 0, 0, 0, 0, 0, 0, 0, 0}, step, startTime).SetTag("holtWintersAberration", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersAberration(metric*,4,'4d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric*", startTime, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, points*step, step, 0), step, startTime), + types.MakeMetricData("metric2", generateHwRange(0, points*step, step, 10), step, startTime), + }, + {"metric*", startTime - 4*seconds, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, ((4*seconds/step)+points)*step, step, 0), step, startTime-4*seconds), + types.MakeMetricData("metric2", generateHwRange(0, ((4*seconds/step)+points)*step, step, 10), step, startTime-4*seconds), + }, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersAberration(metric1)", []float64{-1.4410544085511923, -0.5199507849641569, 0, 0, 0, 0, 0, 0, 0, 0.09386319244056907}, step, startTime).SetTag("holtWintersAberration", "1"), + types.MakeMetricData("holtWintersAberration(metric2)", []float64{-1.4410544085511923, -0.5199507849641609, 0, 0, 0, 0, 0, 0, 0, 0.09386319244056551}, step, startTime).SetTag("holtWintersAberration", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersAberration(metric*,4,'1d','2d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric*", startTime, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, points*step, step, 0), step, startTime), + types.MakeMetricData("metric2", generateHwRange(0, points*step, step, 10), step, startTime), + }, + {"metric*", startTime - seconds, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, ((seconds/step)+points)*step, step, 0), step, startTime-seconds), + types.MakeMetricData("metric2", generateHwRange(0, ((seconds/step)+points)*step, step, 10), step, startTime-seconds), + }, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersAberration(metric1)", []float64{-4.106587168490873, -2.8357974803355406, -1.5645896296885762, -0.4213549577359168, 0, 0, 0, 0.5073914761326588, 2.4432248533746543, 4.186719764193769}, step, startTime).SetTag("holtWintersAberration", "1"), + types.MakeMetricData("holtWintersAberration(metric2)", []float64{-4.1065871684908775, -2.8357974803355486, -1.5645896296885837, -0.42135495773592346, 0, 0, 0, 0.5073914761326499, 2.4432248533746446, 4.186719764193759}, step, startTime).SetTag("holtWintersAberration", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersAberration(metric*)", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric*", startTime, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, points*step, step, 0), step, startTime), + types.MakeMetricData("metric2", generateHwRange(0, points*step, step, 10), step, startTime), + }, + {"metric*", startTime - holtwinters.DefaultBootstrapInterval, startTime + step*points}: { + types.MakeMetricData("metric1", generateHwRange(0, ((holtwinters.DefaultBootstrapInterval/step)+points)*step, step, 0), step, startTime-holtwinters.DefaultBootstrapInterval), + types.MakeMetricData("metric2", generateHwRange(0, ((holtwinters.DefaultBootstrapInterval/step)+points)*step, step, 10), step, startTime-holtwinters.DefaultBootstrapInterval), + types.MakeMetricData("metric3", generateHwRange(0, ((holtwinters.DefaultBootstrapInterval/step)+points)*step, step, 20), step, startTime-holtwinters.DefaultBootstrapInterval), // Verify that metrics that don't match those fetched with the unadjusted start time are not included in the results + }, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersAberration(metric1)", []float64{-0.2841206166091448, -0.05810270987744115, 0, 0, 0, 0, 0, 0, 0, 0}, step, startTime).SetTag("holtWintersAberration", "1"), + types.MakeMetricData("holtWintersAberration(metric2)", []float64{-0.284120616609151, -0.05810270987744737, 0, 0, 0, 0, 0, 0, 0, 0}, step, startTime).SetTag("holtWintersAberration", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + } + + for _, tt := range tests { + testName := tt.Target + t.Run(testName, func(t *testing.T) { + th.TestEvalExprWithRange(t, &tt) + }) + } +} + +func generateHwRange(x, y, jump, t int64) []float64 { + var valuesList []float64 + for x < y { + val := float64(t + (x/jump)%10) + valuesList = append(valuesList, val) + x += jump + } + return valuesList +} diff --git a/expr/functions/holtWintersConfidenceArea/function_cairo.go b/expr/functions/holtWintersConfidenceArea/function_cairo.go index 2d39ff515..95b0fa5f9 100644 --- a/expr/functions/holtWintersConfidenceArea/function_cairo.go +++ b/expr/functions/holtWintersConfidenceArea/function_cairo.go @@ -34,7 +34,7 @@ func New(configFile string) []interfaces.FunctionMetadata { } func (f *holtWintersConfidenceArea) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { - bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, 7*86400) + bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, holtwinters.DefaultBootstrapInterval) if err != nil { return nil, err } @@ -49,22 +49,27 @@ func (f *holtWintersConfidenceArea) Do(ctx context.Context, e parser.Expr, from, return nil, err } + seasonality, err := e.GetIntervalNamedOrPosArgDefault("seasonality", 3, 1, holtwinters.DefaultSeasonality) + if err != nil { + return nil, err + } + results := make([]*types.MetricData, 0, len(args)*2) for _, arg := range args { stepTime := arg.StepTime - lowerBand, upperBand := holtwinters.HoltWintersConfidenceBands(arg.Values, stepTime, delta, bootstrapInterval/86400) + lowerBand, upperBand := holtwinters.HoltWintersConfidenceBands(arg.Values, stepTime, delta, bootstrapInterval, seasonality) lowerSeries := types.MetricData{ FetchResponse: pb.FetchResponse{ - Name: fmt.Sprintf("holtWintersConfidenceLower(%s)", arg.Name), + Name: fmt.Sprintf("holtWintersConfidenceArea(%s)", arg.Name), Values: lowerBand, StepTime: arg.StepTime, StartTime: arg.StartTime + bootstrapInterval, StopTime: arg.StopTime, ConsolidationFunc: arg.ConsolidationFunc, XFilesFactor: arg.XFilesFactor, - PathExpression: fmt.Sprintf("holtWintersConfidenceLower(%s)", arg.Name), + PathExpression: fmt.Sprintf("holtWintersConfidenceArea(%s)", arg.Name), }, Tags: helper.CopyTags(arg), GraphOptions: types.GraphOptions{ @@ -77,14 +82,14 @@ func (f *holtWintersConfidenceArea) Do(ctx context.Context, e parser.Expr, from, upperSeries := types.MetricData{ FetchResponse: pb.FetchResponse{ - Name: fmt.Sprintf("holtWintersConfidenceUpper(%s)", arg.Name), + Name: fmt.Sprintf("holtWintersConfidenceArea(%s)", arg.Name), Values: upperBand, StepTime: arg.StepTime, StartTime: arg.StartTime + bootstrapInterval, StopTime: arg.StopTime, ConsolidationFunc: arg.ConsolidationFunc, XFilesFactor: arg.XFilesFactor, - PathExpression: fmt.Sprintf("holtWintersConfidenceLower(%s)", arg.Name), + PathExpression: fmt.Sprintf("holtWintersConfidenceArea(%s)", arg.Name), }, Tags: helper.CopyTags(arg), GraphOptions: types.GraphOptions{ diff --git a/expr/functions/holtWintersConfidenceArea/function_test.go b/expr/functions/holtWintersConfidenceArea/function_test.go index f00ae9244..90a4c896b 100644 --- a/expr/functions/holtWintersConfidenceArea/function_test.go +++ b/expr/functions/holtWintersConfidenceArea/function_test.go @@ -4,6 +4,7 @@ package holtWintersConfidenceArea import ( + "github.com/go-graphite/carbonapi/expr/holtwinters" "testing" "github.com/go-graphite/carbonapi/expr/helper" @@ -27,19 +28,40 @@ func TestHoltWintersConfidenceArea(t *testing.T) { var startTime int64 = 2678400 var step int64 = 600 var points int64 = 10 - var weekSeconds int64 = 7 * 86400 - - seriesValues := generateHwRange(0, ((weekSeconds/step)+points)*step, step) tests := []th.EvalTestItemWithRange{ { Target: "holtWintersConfidenceArea(metric1)", M: map[parser.MetricRequest][]*types.MetricData{ - {"metric1", startTime - weekSeconds, startTime + step*points}: {types.MakeMetricData("metric1", seriesValues, step, startTime-weekSeconds)}, + {"metric1", startTime - holtwinters.DefaultBootstrapInterval, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((holtwinters.DefaultBootstrapInterval/step)+points)*step, step), step, startTime-holtwinters.DefaultBootstrapInterval)}, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersConfidenceArea(metric1)", []float64{0.2841206166091448, 1.0581027098774411, 0.3338172102994683, 0.5116859493263242, -0.18199175514936972, 0.2366173792019426, -1.2941554508809152, -0.513426806531049, -0.7970905542723132, 0.09868900726536012}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), + types.MakeMetricData("holtWintersConfidenceArea(metric1)", []float64{8.424944558327624, 9.409422251880809, 10.607070189221787, 10.288439865038768, 9.491556863132963, 9.474595784593738, 8.572310478053845, 8.897670449095346, 8.941566968508148, 9.409728797779282}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersConfidenceArea(metric1,4,'6d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric1", startTime - 6*holtwinters.SecondsPerDay, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((6*holtwinters.SecondsPerDay/step)+points)*step, step), step, startTime-6*holtwinters.SecondsPerDay)}, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersConfidenceArea(metric1)", []float64{-0.6535362793382391, -0.26554972418633316, -1.1060549683277792, -0.5788026852576289, -1.4594446935142829, -0.6933311085203409, -1.6566119269969288, -1.251651025511391, -1.7938581852717226, -1.1791817117029604}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), + types.MakeMetricData("holtWintersConfidenceArea(metric1)", []float64{8.166528156512886, 8.759008839563066, 9.250962452510654, 9.994110161265208, 10.511931730022393, 11.34313475259535, 12.639554646464758, 11.972601342482212, 10.920216551100442, 10.618692557967133}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersConfidenceArea(metric1,4,'1d','2d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric1", startTime - holtwinters.SecondsPerDay, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((holtwinters.SecondsPerDay/step)+points)*step, step), step, startTime-holtwinters.SecondsPerDay)}, }, Want: []*types.MetricData{ - types.MakeMetricData("holtWintersConfidenceLower(metric1)", []float64{0.2841206166091448, 1.0581027098774411, 0.3338172102994683, 0.5116859493263242, -0.18199175514936972, 0.2366173792019426, -1.2941554508809152, -0.513426806531049, -0.7970905542723132, 0.09868900726536012}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), - types.MakeMetricData("holtWintersConfidenceUpper(metric1)", []float64{8.424944558327624, 9.409422251880809, 10.607070189221787, 10.288439865038768, 9.491556863132963, 9.474595784593738, 8.572310478053845, 8.897670449095346, 8.941566968508148, 9.409728797779282}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), + types.MakeMetricData("holtWintersConfidenceArea(metric1)", []float64{4.106587168490873, 3.8357974803355406, 3.564589629688576, 3.421354957735917, 3.393696278743315, 3.470415673952413, 3.2748850646377368, 3.3539750816574316, 3.5243322056965765, 3.7771201010598134}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), + types.MakeMetricData("holtWintersConfidenceArea(metric1)", []float64{4.24870339314537, 4.501056063000946, 4.956252698437961, 5.466294981886822, 6.0258698337471355, 6.630178145979606, 7.6413984841547204, 6.492608523867341, 5.556775146625346, 4.813280235806231}, step, startTime).SetTag("holtWintersConfidenceArea", "1"), }, From: startTime, Until: startTime + step*points, diff --git a/expr/functions/holtWintersConfidenceBands/function.go b/expr/functions/holtWintersConfidenceBands/function.go index ee2f43a78..efc7fc22d 100644 --- a/expr/functions/holtWintersConfidenceBands/function.go +++ b/expr/functions/holtWintersConfidenceBands/function.go @@ -2,7 +2,6 @@ package holtWintersConfidenceBands import ( "context" - "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/holtwinters" "github.com/go-graphite/carbonapi/expr/interfaces" @@ -30,7 +29,7 @@ func New(configFile string) []interfaces.FunctionMetadata { } func (f *holtWintersConfidenceBands) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { - bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, 7*86400) + bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, holtwinters.DefaultBootstrapInterval) if err != nil { return nil, err } @@ -45,11 +44,16 @@ func (f *holtWintersConfidenceBands) Do(ctx context.Context, e parser.Expr, from return nil, err } + seasonality, err := e.GetIntervalNamedOrPosArgDefault("seasonality", 3, 1, holtwinters.DefaultSeasonality) + if err != nil { + return nil, err + } + results := make([]*types.MetricData, 0, len(args)*2) for _, arg := range args { stepTime := arg.StepTime - lowerBand, upperBand := holtwinters.HoltWintersConfidenceBands(arg.Values, stepTime, delta, bootstrapInterval/86400) + lowerBand, upperBand := holtwinters.HoltWintersConfidenceBands(arg.Values, stepTime, delta, bootstrapInterval, seasonality) name := "holtWintersConfidenceLower(" + arg.Name + ")" lowerSeries := &types.MetricData{ diff --git a/expr/functions/holtWintersConfidenceBands/function_test.go b/expr/functions/holtWintersConfidenceBands/function_test.go index e07547be5..c48d7c60c 100644 --- a/expr/functions/holtWintersConfidenceBands/function_test.go +++ b/expr/functions/holtWintersConfidenceBands/function_test.go @@ -1,6 +1,7 @@ package holtWintersConfidenceBands import ( + "github.com/go-graphite/carbonapi/expr/holtwinters" "testing" "github.com/go-graphite/carbonapi/expr/helper" @@ -24,15 +25,12 @@ func TestHoltWintersConfidenceBands(t *testing.T) { var startTime int64 = 2678400 var step int64 = 600 var points int64 = 10 - var weekSeconds int64 = 7 * 86400 - - seriesValues := generateHwRange(0, ((weekSeconds/step)+points)*step, step) tests := []th.EvalTestItemWithRange{ { Target: "holtWintersConfidenceBands(metric1)", M: map[parser.MetricRequest][]*types.MetricData{ - {"metric1", startTime - weekSeconds, startTime + step*points}: {types.MakeMetricData("metric1", seriesValues, step, startTime-weekSeconds)}, + {"metric1", startTime - holtwinters.DefaultBootstrapInterval, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, (((holtwinters.DefaultBootstrapInterval)/step)+points)*step, step), step, startTime-(holtwinters.DefaultBootstrapInterval))}, }, Want: []*types.MetricData{ types.MakeMetricData("holtWintersConfidenceLower(metric1)", []float64{0.2841206166091448, 1.0581027098774411, 0.3338172102994683, 0.5116859493263242, -0.18199175514936972, 0.2366173792019426, -1.2941554508809152, -0.513426806531049, -0.7970905542723132, 0.09868900726536012}, step, startTime).SetTag("holtWintersConfidenceLower", "1"), @@ -41,6 +39,30 @@ func TestHoltWintersConfidenceBands(t *testing.T) { From: startTime, Until: startTime + step*points, }, + { + Target: "holtWintersConfidenceBands(metric1,4,'6d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric1", startTime - 6*holtwinters.SecondsPerDay, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((6*holtwinters.SecondsPerDay/step)+points)*step, step), step, startTime-6*holtwinters.SecondsPerDay)}, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersConfidenceLower(metric1)", []float64{-0.6535362793382391, -0.26554972418633316, -1.1060549683277792, -0.5788026852576289, -1.4594446935142829, -0.6933311085203409, -1.6566119269969288, -1.251651025511391, -1.7938581852717226, -1.1791817117029604}, step, startTime).SetTag("holtWintersConfidenceLower", "1"), + types.MakeMetricData("holtWintersConfidenceUpper(metric1)", []float64{8.166528156512886, 8.759008839563066, 9.250962452510654, 9.994110161265208, 10.511931730022393, 11.34313475259535, 12.639554646464758, 11.972601342482212, 10.920216551100442, 10.618692557967133}, step, startTime).SetTag("holtWintersConfidenceUpper", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersConfidenceBands(metric1,4,'1d','2d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric1", startTime - holtwinters.SecondsPerDay, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((holtwinters.SecondsPerDay/step)+points)*step, step), step, startTime-holtwinters.SecondsPerDay)}, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersConfidenceLower(metric1)", []float64{4.106587168490873, 3.8357974803355406, 3.564589629688576, 3.421354957735917, 3.393696278743315, 3.470415673952413, 3.2748850646377368, 3.3539750816574316, 3.5243322056965765, 3.7771201010598134}, step, startTime).SetTag("holtWintersConfidenceLower", "1"), + types.MakeMetricData("holtWintersConfidenceUpper(metric1)", []float64{4.24870339314537, 4.501056063000946, 4.956252698437961, 5.466294981886822, 6.0258698337471355, 6.630178145979606, 7.6413984841547204, 6.492608523867341, 5.556775146625346, 4.813280235806231}, step, startTime).SetTag("holtWintersConfidenceUpper", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, } for _, tt := range tests { diff --git a/expr/functions/holtWintersForecast/function.go b/expr/functions/holtWintersForecast/function.go index 0049817e6..2195fc953 100644 --- a/expr/functions/holtWintersForecast/function.go +++ b/expr/functions/holtWintersForecast/function.go @@ -30,7 +30,7 @@ func New(configFile string) []interfaces.FunctionMetadata { } func (f *holtWintersForecast) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { - bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, 7*86400) + bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 1, 1, holtwinters.DefaultBootstrapInterval) if err != nil { return nil, err } @@ -40,12 +40,17 @@ func (f *holtWintersForecast) Do(ctx context.Context, e parser.Expr, from, until return nil, err } + seasonality, err := e.GetIntervalNamedOrPosArgDefault("seasonality", 2, 1, holtwinters.DefaultSeasonality) + if err != nil { + return nil, err + } + var predictionsOfInterest []float64 results := make([]*types.MetricData, len(args)) for i, arg := range args { stepTime := arg.StepTime - predictions, _ := holtwinters.HoltWintersAnalysis(arg.Values, stepTime) + predictions, _ := holtwinters.HoltWintersAnalysis(arg.Values, stepTime, seasonality) windowPoints := int(bootstrapInterval / stepTime) if len(predictions) < windowPoints { @@ -68,7 +73,7 @@ func (f *holtWintersForecast) Do(ctx context.Context, e parser.Expr, from, until }, Tags: helper.CopyTags(arg), } - r.Tags["holtWintersConfidenceBands"] = "1" + r.Tags["holtWintersForecast"] = "1" results[i] = r } return results, nil diff --git a/expr/functions/holtWintersForecast/function_test.go b/expr/functions/holtWintersForecast/function_test.go new file mode 100644 index 000000000..6283df4c3 --- /dev/null +++ b/expr/functions/holtWintersForecast/function_test.go @@ -0,0 +1,81 @@ +package holtWintersForecast + +import ( + "testing" + + "github.com/go-graphite/carbonapi/expr/helper" + "github.com/go-graphite/carbonapi/expr/metadata" + "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/parser" + th "github.com/go-graphite/carbonapi/tests" +) + +func init() { + md := New("") + evaluator := th.EvaluatorFromFunc(md[0].F) + metadata.SetEvaluator(evaluator) + helper.SetEvaluator(evaluator) + for _, m := range md { + metadata.RegisterFunction(m.Name, m.F) + } +} + +func TestHoltWintersForecast(t *testing.T) { + var startTime int64 = 2678400 + var step int64 = 600 + var points int64 = 10 + var seconds int64 = 86400 + + tests := []th.EvalTestItemWithRange{ + { + Target: "holtWintersForecast(metric1)", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric1", startTime - 7*seconds, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((7*seconds/step)+points)*step, step), step, startTime-7*seconds)}, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersForecast(metric1)", []float64{4.354532587468384, 5.233762480879125, 5.470443699760628, 5.400062907182546, 4.654782553991797, 4.85560658189784, 3.639077513586465, 4.192121821282148, 4.072238207117917, 4.754208902522321}, step, startTime).SetTag("holtWintersForecast", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersForecast(metric1,'6d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric1", startTime - 6*seconds, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((6*seconds/step)+points)*step, step), step, startTime-6*seconds)}, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersForecast(metric1)", []float64{3.756495938587323, 4.246729557688366, 4.0724537420914375, 4.707653738003789, 4.526243518254055, 5.324901822037504, 5.491471359733914, 5.360475158485411, 4.56317918291436, 4.719755423132087}, step, startTime).SetTag("holtWintersForecast", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + { + Target: "holtWintersForecast(metric1,'1d','2d')", + M: map[parser.MetricRequest][]*types.MetricData{ + {"metric1", startTime - seconds, startTime + step*points}: {types.MakeMetricData("metric1", generateHwRange(0, ((seconds/step)+points)*step, step), step, startTime-seconds)}, + }, + Want: []*types.MetricData{ + types.MakeMetricData("holtWintersForecast(metric1)", []float64{4.177645280818122, 4.168426771668243, 4.260421164063269, 4.443824969811369, 4.709783056245225, 5.0502969099660096, 5.458141774396228, 4.923291802762386, 4.540553676160961, 4.2952001684330225}, step, startTime).SetTag("holtWintersForecast", "1"), + }, + From: startTime, + Until: startTime + step*points, + }, + } + + for _, tt := range tests { + testName := tt.Target + t.Run(testName, func(t *testing.T) { + th.TestEvalExprWithRange(t, &tt) + }) + } +} + +func generateHwRange(x, y, jump int64) []float64 { + var valuesList []float64 + for x < y { + val := float64((x / jump) % 10) + valuesList = append(valuesList, val) + x += jump + } + return valuesList +} diff --git a/expr/holtwinters/hw.go b/expr/holtwinters/hw.go index 73a6b82c3..b169a72bc 100644 --- a/expr/holtwinters/hw.go +++ b/expr/holtwinters/hw.go @@ -7,6 +7,12 @@ import ( "math" ) +const ( + DefaultSeasonality = 86400 // Seconds in 1 day + DefaultBootstrapInterval = 604800 // Seconds in 7 days + SecondsPerDay = 86400 +) + func holtWintersIntercept(alpha, actual, lastSeason, lastIntercept, lastSlope float64) float64 { return alpha*(actual-lastSeason) + (1-alpha)*(lastIntercept+lastSlope) } @@ -27,15 +33,14 @@ func holtWintersDeviation(gamma, actual, prediction, lastSeasonalDev float64) fl } // HoltWintersAnalysis do Holt-Winters Analysis -func HoltWintersAnalysis(series []float64, step int64) ([]float64, []float64) { +func HoltWintersAnalysis(series []float64, step int64, seasonality int64) ([]float64, []float64) { const ( alpha = 0.1 beta = 0.0035 gamma = 0.1 ) - // season is currently one day - seasonLength := 24 * 60 * 60 / int(step) + seasonLength := seasonality / step // seasonLength needs to be at least 2, so we force it // GraphiteWeb has the same problem, still needs fixing https://github.com/graphite-project/graphite-web/issues/2780 @@ -52,7 +57,7 @@ func HoltWintersAnalysis(series []float64, step int64) ([]float64, []float64) { ) getLastSeasonal := func(i int) float64 { - j := i - seasonLength + j := i - int(seasonLength) if j >= 0 { return seasonals[j] } @@ -60,7 +65,7 @@ func HoltWintersAnalysis(series []float64, step int64) ([]float64, []float64) { } getLastDeviation := func(i int) float64 { - j := i - seasonLength + j := i - int(seasonLength) if j >= 0 { return deviations[j] } @@ -122,12 +127,12 @@ func HoltWintersAnalysis(series []float64, step int64) ([]float64, []float64) { } // HoltWintersConfidenceBands do Holt-Winters Confidence Bands -func HoltWintersConfidenceBands(series []float64, step int64, delta float64, days int64) ([]float64, []float64) { +func HoltWintersConfidenceBands(series []float64, step int64, delta float64, bootstrapInterval int64, seasonality int64) ([]float64, []float64) { var lowerBand, upperBand []float64 - predictions, deviations := HoltWintersAnalysis(series, step) + predictions, deviations := HoltWintersAnalysis(series, step, seasonality) - windowPoints := int(days * 86400 / step) + windowPoints := int(bootstrapInterval / step) var ( predictionsOfInterest []float64 diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index eeb8c140f..5d816b01a 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -3,6 +3,7 @@ package parser import ( "bytes" "fmt" + "github.com/go-graphite/carbonapi/expr/holtwinters" "regexp" "strconv" "strings" @@ -196,8 +197,8 @@ func (e *expr) Metrics(from, until int64) []MetricRequest { } return r2 - case "holtWintersForecast", "holtWintersConfidenceBands", "holtWintersConfidenceArea", "holtWintersAberration": - bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, 7*86400) + case "holtWintersForecast": + bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 1, 1, holtwinters.DefaultBootstrapInterval) if err != nil { return nil } @@ -205,6 +206,32 @@ func (e *expr) Metrics(from, until int64) []MetricRequest { for i := range r { r[i].From -= bootstrapInterval } + case "holtWintersConfidenceBands", "holtWintersConfidenceArea": + bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, holtwinters.DefaultBootstrapInterval) + if err != nil { + return nil + } + + for i := range r { + r[i].From -= bootstrapInterval + } + case "holtWintersAberration": + bootstrapInterval, err := e.GetIntervalNamedOrPosArgDefault("bootstrapInterval", 2, 1, holtwinters.DefaultBootstrapInterval) + if err != nil { + return nil + } + + // For this function, we also need to pull data with an adjusted From time, + // so additional requests are added with the adjusted start time based on the + // bootstrapInterval + for i := range r { + adjustedReq := MetricRequest{ + Metric: r[i].Metric, + From: r[i].From - bootstrapInterval, + Until: r[i].Until, + } + r = append(r, adjustedReq) + } case "movingAverage", "movingMedian", "movingMin", "movingMax", "movingSum", "exponentialMovingAverage": if len(e.args) < 2 { return nil diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index 2a31b9a25..27b152efa 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -718,6 +718,58 @@ func TestMetrics(t *testing.T) { }, }, }, + { + "holtWintersAberration(metric1)", + &expr{ + target: "holtWintersAberration", + etype: EtFunc, + args: []*expr{ + {target: "metric1"}, + }, + argString: "metric1", + }, + 1410346740, + 1410346865, + []MetricRequest{ + { + Metric: "metric1", + From: 1410346740, + Until: 1410346865, + }, + { + Metric: "metric1", + From: 1409741940, + Until: 1410346865, + }, + }, + }, + { + "holtWintersAberration(metric1,3,'6d')", + &expr{ + target: "holtWintersAberration", + etype: EtFunc, + args: []*expr{ + {target: "metric1"}, + {valStr: "3", etype: EtConst}, + {valStr: "6d", etype: EtString}, + }, + argString: "metric1, 3, '6d'", + }, + 1410346740, + 1410346865, + []MetricRequest{ + { + Metric: "metric1", + From: 1410346740, + Until: 1410346865, + }, + { + Metric: "metric1", + From: 1409828340, + Until: 1410346865, + }, + }, + }, { "holtWintersConfidenceBands(metric1)", &expr{ @@ -726,7 +778,7 @@ func TestMetrics(t *testing.T) { args: []*expr{ {target: "metric1"}, }, - argString: "metric1, 4, '1d'", + argString: "metric1", }, 1410346740, 1410346865, @@ -784,6 +836,26 @@ func TestMetrics(t *testing.T) { }, }, }, + { + "holtWintersForecast(metric1,'1d')", + &expr{ + target: "holtWintersConfidenceBands", + etype: EtFunc, + args: []*expr{ + {target: "metric1"}, + }, + argString: "metric1", + }, + 1410346740, + 1410346865, + []MetricRequest{ + { + Metric: "metric1", + From: 1409741940, + Until: 1410346865, + }, + }, + }, { "smartSummarize(metric1, '1h', 'sum', 'seconds')", &expr{