diff --git a/src/dbnode/encoding/m3tsz/m3tsz.go b/src/dbnode/encoding/m3tsz/m3tsz.go index 1b92a523cc..26a138b41a 100644 --- a/src/dbnode/encoding/m3tsz/m3tsz.go +++ b/src/dbnode/encoding/m3tsz/m3tsz.go @@ -88,14 +88,16 @@ func convertToIntFloat(v float64, curMaxMult uint8) (float64, uint8, bool, error return 0.0, 0, false, errInvalidMultiplier } - val := v * multipliers[int(curMaxMult)] sign := 1.0 if v < 0 { sign = -1.0 - val = val * -1.0 } - for mult := curMaxMult; mult <= maxMult && val < maxOptInt; mult++ { + for mult := curMaxMult; mult <= maxMult; mult++ { + val := v * multipliers[int(mult)] * sign + if val >= maxOptInt { + break + } i, r := math.Modf(val) if r == 0 { return sign * i, mult, false, nil @@ -111,7 +113,6 @@ func convertToIntFloat(v float64, curMaxMult uint8) (float64, uint8, bool, error return sign * next, mult, false, nil } } - val = val * 10.0 } return v, 0, true, nil diff --git a/src/dbnode/encoding/m3tsz/roundtrip_test.go b/src/dbnode/encoding/m3tsz/roundtrip_test.go index 893848b452..867b8bb5d9 100644 --- a/src/dbnode/encoding/m3tsz/roundtrip_test.go +++ b/src/dbnode/encoding/m3tsz/roundtrip_test.go @@ -92,6 +92,21 @@ func TestMixedRoundTrip(t *testing.T) { } } +func TestPrecision(t *testing.T) { + var ( + num = 100 + input = make([]ts.Datapoint, 0, num) + timestamp = xtime.Now() + ) + + for i := 0; i < num; i++ { + input = append(input, ts.Datapoint{TimestampNanos: timestamp, Value: 187.80131100000006}) + timestamp = timestamp.Add(time.Minute) + } + + testRoundTrip(t, input) +} + func TestIntOverflow(t *testing.T) { testRoundTrip(t, generateOverflowDatapoints()) } @@ -159,10 +174,10 @@ func validateRoundTrip(t *testing.T, input []ts.Datapoint, intOpt bool) { expectedAnnotation = nil } - require.Equal(t, input[i].TimestampNanos, v.TimestampNanos) - require.Equal(t, input[i].Value, v.Value) - require.Equal(t, timeUnits[i], u) - require.Equal(t, expectedAnnotation, a) + require.Equal(t, input[i].TimestampNanos, v.TimestampNanos, "datapoint #%d", i) + require.Equal(t, input[i].Value, v.Value, "datapoint #%d", i) + require.Equal(t, timeUnits[i], u, "datapoint #%d", i) + require.Equal(t, expectedAnnotation, a, "datapoint #%d", i) i++ }