Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[pkg/ottl] Add support for scaling values #33246

Merged
Merged
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
b782416
[processor/transform] Add support for scaling values
bacherfl May 27, 2024
adcda48
revert change to e2e test
bacherfl May 28, 2024
4ebb38a
rename helper function
bacherfl May 28, 2024
3ba07ac
Merge branch 'main' into feat/16214/scale-metric-values
bacherfl May 28, 2024
d610bf0
add documentation for Scale function
bacherfl May 28, 2024
9d741ac
Merge remote-tracking branch 'bacherfl/feat/16214/scale-metric-values…
bacherfl May 28, 2024
d098a77
adapt to possible types returned by metric context path
bacherfl May 28, 2024
f019ec0
Merge branch 'main' into feat/16214/scale-metric-values
bacherfl May 28, 2024
15a2006
additional test for exponential histogram
bacherfl May 28, 2024
213fcce
Merge remote-tracking branch 'bacherfl/feat/16214/scale-metric-values…
bacherfl May 28, 2024
285a37d
use type switch for handling different types as Scale argument
bacherfl May 29, 2024
fc1dc7f
Merge branch 'main' into feat/16214/scale-metric-values
bacherfl May 29, 2024
4084b53
incorporate feedback from PR review
bacherfl Jun 3, 2024
35064e1
Merge remote-tracking branch 'bacherfl/feat/16214/scale-metric-values…
bacherfl Jun 3, 2024
343ce4e
add license comment
bacherfl Jun 3, 2024
549bdc3
address PR review comments, add changelog entry
bacherfl Jun 4, 2024
fb04e21
fix linting
bacherfl Jun 5, 2024
75e24b1
Merge branch 'main' into feat/16214/scale-metric-values
evan-bradley Jun 5, 2024
49321a2
Update pkg/ottl/ottlfuncs/func_scale.go
bacherfl Jun 6, 2024
47d6834
fix e2e tests
bacherfl Jun 6, 2024
313cd7e
Merge branch 'main' into feat/16214/scale-metric-values
bacherfl Jun 6, 2024
d76e6cc
Merge branch 'main' into feat/16214/scale-metric-values
bacherfl Jun 13, 2024
44997a1
Merge branch 'main' into feat/16214/scale-metric-values
evan-bradley Jun 14, 2024
0f07a49
Merge branch 'main' into feat/16214/scale-metric-values
bacherfl Jun 17, 2024
92d33ed
adapt scale function to be an editor function
bacherfl Jun 17, 2024
f660ca6
remove unused testing struct
bacherfl Jun 17, 2024
37f16bb
remove unused testing struct; adapt test
bacherfl Jun 17, 2024
62a539b
adapt test
bacherfl Jun 17, 2024
61e2a09
fix linting
bacherfl Jun 17, 2024
75a6d67
adapt the function to only work with metrics
bacherfl Jun 18, 2024
14bbdd8
remove obsolete test
bacherfl Jun 18, 2024
0090fcc
apply suggestions from PR review
bacherfl Jun 19, 2024
d398d64
move scale function to transformprocessor
bacherfl Jun 19, 2024
e495e03
fix failing checks
bacherfl Jun 20, 2024
c83e5ba
fix linting
bacherfl Jun 20, 2024
66c5bd5
appease linter
bacherfl Jun 20, 2024
bf746e7
make unit an optional value
bacherfl Jun 24, 2024
ffed4f1
Merge branch 'main' into feat/16214/scale-metric-values
bacherfl Jun 24, 2024
719daca
adapt unit tests to recent changes
bacherfl Jun 24, 2024
721c537
apply suggestion from code review
bacherfl Jul 2, 2024
a6c39a1
adapt to pr review
bacherfl Jul 2, 2024
876afc1
Apply suggestions from code review
bacherfl Jul 23, 2024
42d3140
incorporate review comments
bacherfl Jul 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .chloggen/add-scale-function.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: pkg/ottl

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Added support for scaling values
bacherfl marked this conversation as resolved.
Show resolved Hide resolved

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [16214]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: []
64 changes: 64 additions & 0 deletions pkg/ottl/e2e/e2e_test.go
evan-bradley marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ import (
"go.opentelemetry.io/collector/component/componenttest"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/plog"
"go.opentelemetry.io/collector/pdata/pmetric"

"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/ottllog"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/ottlmetric"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/plogtest"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest"
)

var (
Expand Down Expand Up @@ -54,6 +57,7 @@ func Test_e2e_editors(t *testing.T) {
tCtx.GetLogRecord().Attributes().Remove("flags")
tCtx.GetLogRecord().Attributes().Remove("total.string")
tCtx.GetLogRecord().Attributes().Remove("foo")
tCtx.GetLogRecord().Attributes().Remove("double_value")
},
},
{
Expand Down Expand Up @@ -81,6 +85,7 @@ func Test_e2e_editors(t *testing.T) {
m.PutStr("test.foo.flags", "pass")
m.PutStr("test.foo.slice.0", "val")
m.PutStr("test.foo.nested.test", "pass")
m.PutDouble("test.double_value", 10.5)
m.CopyTo(tCtx.GetLogRecord().Attributes())
},
},
Expand All @@ -102,6 +107,7 @@ func Test_e2e_editors(t *testing.T) {
m.PutStr("foo.bar", "pass")
m.PutStr("foo.flags", "pass")
m.PutStr("foo.slice.0", "val")
m.PutDouble("double_value", 10.5)
m2 := m.PutEmptyMap("foo.nested")
m2.PutStr("test", "pass")
m.CopyTo(tCtx.GetLogRecord().Attributes())
Expand All @@ -114,6 +120,7 @@ func Test_e2e_editors(t *testing.T) {
tCtx.GetLogRecord().Attributes().Remove("http.path")
tCtx.GetLogRecord().Attributes().Remove("http.url")
tCtx.GetLogRecord().Attributes().Remove("foo")
tCtx.GetLogRecord().Attributes().Remove("double_value")
},
},
{
Expand All @@ -128,6 +135,7 @@ func Test_e2e_editors(t *testing.T) {
tCtx.GetLogRecord().Attributes().Remove("http.url")
tCtx.GetLogRecord().Attributes().Remove("flags")
tCtx.GetLogRecord().Attributes().Remove("foo")
tCtx.GetLogRecord().Attributes().Remove("double_value")
},
},
{
Expand Down Expand Up @@ -297,6 +305,38 @@ func Test_e2e_editors(t *testing.T) {
}
}

func Test_e2e_metricEditors(t *testing.T) {
tests := []struct {
statement string
want func(tCtx ottlmetric.TransformContext)
}{
{
statement: `scale_metric(data_points,0.1)`,
want: func(tCtx ottlmetric.TransformContext) {
tCtx.GetMetric().Gauge().DataPoints().At(0).SetDoubleValue(1.0)
},
},
}

for _, tt := range tests {
t.Run(tt.statement, func(t *testing.T) {
settings := componenttest.NewNopTelemetrySettings()
metricParser, err := ottlmetric.NewParser(ottlfuncs.StandardFuncs[ottlmetric.TransformContext](), settings)
assert.NoError(t, err)
logStatements, err := metricParser.ParseStatement(tt.statement)
assert.NoError(t, err)

tCtx := constructMetricTransformContext()
_, _, _ = logStatements.Execute(context.Background(), tCtx)

exTCtx := constructMetricTransformContext()
tt.want(exTCtx)

assert.NoError(t, pmetrictest.CompareResourceMetrics(newResourceMetrics(exTCtx), newResourceMetrics(tCtx)))
})
}
}

func Test_e2e_converters(t *testing.T) {
tests := []struct {
statement string
Expand Down Expand Up @@ -816,6 +856,19 @@ func Test_e2e_ottl_features(t *testing.T) {
}
}

func constructMetricTransformContext() ottlmetric.TransformContext {
resource := pcommon.NewResource()
resource.Attributes().PutStr("host.name", "localhost")

scope := pcommon.NewInstrumentationScope()
scope.SetName("scope")

metric := pmetric.NewMetric()
metric.SetEmptyGauge().DataPoints().AppendEmpty().SetDoubleValue(10.0)

return ottlmetric.NewTransformContext(metric, pmetric.NewMetricSlice(), scope, resource)
}

func constructLogTransformContext() ottllog.TransformContext {
resource := pcommon.NewResource()
resource.Attributes().PutStr("host.name", "localhost")
Expand All @@ -837,6 +890,7 @@ func constructLogTransformContext() ottllog.TransformContext {
logRecord.Attributes().PutStr("http.url", "http://localhost/health")
logRecord.Attributes().PutStr("flags", "A|B|C")
logRecord.Attributes().PutStr("total.string", "123456789")
logRecord.Attributes().PutDouble("double_value", 10.5)
m := logRecord.Attributes().PutEmptyMap("foo")
m.PutStr("bar", "pass")
m.PutStr("flags", "pass")
Expand All @@ -858,3 +912,13 @@ func newResourceLogs(tCtx ottllog.TransformContext) plog.ResourceLogs {
tCtx.GetLogRecord().CopyTo(l)
return rl
}

func newResourceMetrics(tCtx ottlmetric.TransformContext) pmetric.ResourceMetrics {
rl := pmetric.NewResourceMetrics()
tCtx.GetResource().CopyTo(rl.Resource())
sl := rl.ScopeMetrics().AppendEmpty()
tCtx.GetInstrumentationScope().CopyTo(sl.Scope())
l := sl.Metrics().AppendEmpty()
tCtx.GetMetric().CopyTo(l)
return rl
}
17 changes: 17 additions & 0 deletions pkg/ottl/ottlfuncs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ Available Editors:
- [replace_pattern](#replace_pattern)
- [set](#set)
- [truncate_all](#truncate_all)
- [scale_metric](#scale_metric)

### append

Expand Down Expand Up @@ -402,6 +403,22 @@ Examples:

- `truncate_all(resource.attributes, 50)`

### scale_metric

`scale_metric(value, factor)`

The `Scale` function multiplies the original `value` by the `factor`.
bacherfl marked this conversation as resolved.
Show resolved Hide resolved
The supported data types are:

- `data_points` - Supported metric types are `Gauge`, `Sum` and `Histogram`.
To scale a metric of these types, the `data_points` property of the respective metric needs to be passed to the function,
as indicated in the examples below.

Examples:

- `scale_metric(10.0, 0.1)`: Trivial example
- `scale_metric(data_points, 10.0)`: Modifies the metric's `data_points` by multiplying them with the factor `10.0`.

## Converters

Converters are pure functions that take OTTL values as input and output a single value for use within a statement.
Expand Down
124 changes: 124 additions & 0 deletions pkg/ottl/ottlfuncs/func_scale.go
TylerHelmuth marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs"

import (
"context"
"errors"
"fmt"

"go.opentelemetry.io/collector/pdata/pmetric"

"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl"
)

type ScaleArguments[K any] struct {
Value ottl.GetSetter[K]
TylerHelmuth marked this conversation as resolved.
Show resolved Hide resolved
Multiplier float64
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Like @TylerHelmuth mentioned, could we add an optional parameter to adjust the unit? I think it would be nice for usability to do all scaling in one statement.

}

func NewScaleFactory[K any]() ottl.Factory[K] {
return ottl.NewFactory("scale_metric", &ScaleArguments[K]{}, createScaleFunction[K])
}

func createScaleFunction[K any](_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) {
args, ok := oArgs.(*ScaleArguments[K])

if !ok {
return nil, fmt.Errorf("ScaleFactory args must be of type *ScaleArguments[K]")
}

return Scale(args.Value, args.Multiplier)
}

func Scale[K any](getSetter ottl.GetSetter[K], multiplier float64) (ottl.ExprFunc[K], error) {
return func(ctx context.Context, tCtx K) (any, error) {
got, err := getSetter.Get(ctx, tCtx)
if err != nil {
return nil, err
}

switch value := got.(type) {
case pmetric.NumberDataPointSlice:
scaleMetric(value, multiplier)
return nil, nil
case pmetric.HistogramDataPointSlice:
scaleHistogram(value, multiplier)
return nil, nil
case pmetric.SummaryDataPointValueAtQuantileSlice:
scaleSummaryDataPointValueAtQuantileSlice(value, multiplier)
return nil, nil
case pmetric.ExemplarSlice:
scaleExemplarSlice(value, multiplier)
return nil, nil
case pmetric.ExponentialHistogramDataPointSlice:
return nil, errors.New("exponential histograms are not supported by the 'scale_metric' function")
default:
bacherfl marked this conversation as resolved.
Show resolved Hide resolved
return nil, fmt.Errorf("unsupported data type: '%T'", value)
}
}, nil
}

func scaleExemplarSlice(values pmetric.ExemplarSlice, multiplier float64) {
for i := 0; i < values.Len(); i++ {
ex := values.At(i)
scaleExemplar(&ex, multiplier)
}
}

func scaleExemplar(ex *pmetric.Exemplar, multiplier float64) {
switch ex.ValueType() {
case pmetric.ExemplarValueTypeInt:
ex.SetIntValue(int64(float64(ex.IntValue()) * multiplier))
case pmetric.ExemplarValueTypeDouble:
ex.SetDoubleValue(ex.DoubleValue() * multiplier)
}
}

func scaleSummaryDataPointValueAtQuantileSlice(values pmetric.SummaryDataPointValueAtQuantileSlice, multiplier float64) {
for i := 0; i < values.Len(); i++ {
dp := values.At(i)

dp.SetValue(dp.Value() * multiplier)
}
}

func scaleHistogram(datapoints pmetric.HistogramDataPointSlice, multiplier float64) {
for i := 0; i < datapoints.Len(); i++ {
dp := datapoints.At(i)

if dp.HasSum() {
dp.SetSum(dp.Sum() * multiplier)
}
if dp.HasMin() {
dp.SetMin(dp.Min() * multiplier)
}
if dp.HasMax() {
dp.SetMax(dp.Max() * multiplier)
}

for bounds, bi := dp.ExplicitBounds(), 0; bi < bounds.Len(); bi++ {
bounds.SetAt(bi, bounds.At(bi)*multiplier)
}

for exemplars, ei := dp.Exemplars(), 0; ei < exemplars.Len(); ei++ {
exemplar := exemplars.At(ei)
scaleExemplar(&exemplar, multiplier)
}
}
}

func scaleMetric(points pmetric.NumberDataPointSlice, multiplier float64) {
for i := 0; i < points.Len(); i++ {
dp := points.At(i)
switch dp.ValueType() {
case pmetric.NumberDataPointValueTypeInt:
dp.SetIntValue(int64(float64(dp.IntValue()) * multiplier))

case pmetric.NumberDataPointValueTypeDouble:
dp.SetDoubleValue(dp.DoubleValue() * multiplier)
default:
}
}
}
Loading