[dbnode] Fix m3tsz encoding precision issue (#3872)

src/dbnode/encoding/m3tsz/m3tsz.go

@@ -88,14 +88,16 @@ func convertToIntFloat(v float64, curMaxMult uint8) (float64, uint8, bool, error
 		return 0.0, 0, false, errInvalidMultiplier
 	}
 
-	val := v * multipliers[int(curMaxMult)]
 	sign := 1.0
 	if v < 0 {
 		sign = -1.0
-		val = val * -1.0
 	}
 
-	for mult := curMaxMult; mult <= maxMult && val < maxOptInt; mult++ {
+	for mult := curMaxMult; mult <= maxMult; mult++ {
+		val := v * multipliers[int(mult)] * sign
+		if val >= maxOptInt {
+			break
+		}
 		i, r := math.Modf(val)
 		if r == 0 {
 			return sign * i, mult, false, nil
@@ -111,7 +113,6 @@ func convertToIntFloat(v float64, curMaxMult uint8) (float64, uint8, bool, error
 				return sign * next, mult, false, nil
 			}
 		}
-		val = val * 10.0
 	}
 
 	return v, 0, true, nil

src/dbnode/encoding/m3tsz/roundtrip_test.go

@@ -92,6 +92,21 @@ func TestMixedRoundTrip(t *testing.T) {
 	}
 }
 
+func TestPrecision(t *testing.T) {
+	var (
+		num       = 100
+		input     = make([]ts.Datapoint, 0, num)
+		timestamp = xtime.Now()
+	)
+
+	for i := 0; i < num; i++ {
+		input = append(input, ts.Datapoint{TimestampNanos: timestamp, Value: 187.80131100000006})
+		timestamp = timestamp.Add(time.Minute)
+	}
+
+	testRoundTrip(t, input)
+}
+
 func TestIntOverflow(t *testing.T) {
 	testRoundTrip(t, generateOverflowDatapoints())
 }
@@ -159,10 +174,10 @@ func validateRoundTrip(t *testing.T, input []ts.Datapoint, intOpt bool) {
 			expectedAnnotation = nil
 		}
 
-		require.Equal(t, input[i].TimestampNanos, v.TimestampNanos)
-		require.Equal(t, input[i].Value, v.Value)
-		require.Equal(t, timeUnits[i], u)
-		require.Equal(t, expectedAnnotation, a)
+		require.Equal(t, input[i].TimestampNanos, v.TimestampNanos, "datapoint #%d", i)
+		require.Equal(t, input[i].Value, v.Value, "datapoint #%d", i)
+		require.Equal(t, timeUnits[i], u, "datapoint #%d", i)
+		require.Equal(t, expectedAnnotation, a, "datapoint #%d", i)
 
 		i++
 	}