From 13cc3a1904f7997346d292ef3de608f1d4f75c6c Mon Sep 17 00:00:00 2001 From: wjHuang Date: Wed, 14 Apr 2021 16:03:52 +0800 Subject: [PATCH] cherry pick #23559 to release-5.0 Signed-off-by: ti-srebot --- types/datum.go | 53 ++--------------------------- types/time.go | 12 ------- util/ranger/points.go | 70 +++++++++++++++++++++++++++++--------- util/ranger/ranger_test.go | 6 ++-- 4 files changed, 60 insertions(+), 81 deletions(-) diff --git a/types/datum.go b/types/datum.go index d8965b1b4bf44..f0d4a35d3ddcd 100644 --- a/types/datum.go +++ b/types/datum.go @@ -832,7 +832,7 @@ func (d *Datum) ConvertTo(sc *stmtctx.StatementContext, target *FieldType) (Datu case mysql.TypeNewDecimal: return d.convertToMysqlDecimal(sc, target) case mysql.TypeYear: - return d.convertToMysqlYear(sc, target) + return d.ConvertToMysqlYear(sc, target) case mysql.TypeEnum: return d.convertToMysqlEnum(sc, target) case mysql.TypeBit: @@ -1350,7 +1350,8 @@ func ProduceDecWithSpecifiedTp(dec *MyDecimal, tp *FieldType, sc *stmtctx.Statem return dec, err } -func (d *Datum) convertToMysqlYear(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) { +// ConvertToMysqlYear converts a datum to MySQLYear. +func (d *Datum) ConvertToMysqlYear(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) { var ( ret Datum y int64 @@ -1395,54 +1396,6 @@ func (d *Datum) convertToMysqlYear(sc *stmtctx.StatementContext, target *FieldTy return ret, errors.Trace(err) } -// ConvertDatumToFloatYear converts datum into MySQL year with float type -func ConvertDatumToFloatYear(sc *stmtctx.StatementContext, d Datum) (Datum, error) { - return d.convertToMysqlFloatYear(sc, types.NewFieldType(mysql.TypeYear)) -} - -func (d *Datum) convertToMysqlFloatYear(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) { - var ( - ret Datum - y float64 - err error - adjust bool - ) - switch d.k { - case KindString, KindBytes: - s := d.GetString() - trimS := strings.TrimSpace(s) - y, err = StrToFloat(sc, trimS, false) - if err != nil { - ret.SetFloat64(0) - return ret, errors.Trace(err) - } - // condition: - // parsed to 0, not a string of length 4, the first valid char is a 0 digit - if len(s) != 4 && y == 0 && strings.HasPrefix(trimS, "0") { - adjust = true - } - case KindMysqlTime: - y = float64(d.GetMysqlTime().Year()) - case KindMysqlDuration: - y = float64(time.Now().Year()) - case KindNull: - // if datum is NULL, we should keep it as it is, instead of setting it to zero or any other value. - ret = *d - return ret, nil - default: - ret, err = d.convertToFloat(sc, NewFieldType(mysql.TypeDouble)) - if err != nil { - _, err = invalidConv(d, target.Tp) - ret.SetFloat64(0) - return ret, err - } - y = ret.GetFloat64() - } - y = adjustYearForFloat(y, adjust) - ret.SetFloat64(y) - return ret, err -} - func (d *Datum) convertStringToMysqlBit(sc *stmtctx.StatementContext) (uint64, error) { bitStr, err := ParseBitStr(BinaryLiteral(d.b).ToString()) if err != nil { diff --git a/types/time.go b/types/time.go index 615bb86ed6437..cd9c3252c3ff7 100644 --- a/types/time.go +++ b/types/time.go @@ -1260,18 +1260,6 @@ func AdjustYear(y int64, adjustZero bool) (int64, error) { return y, nil } -func adjustYearForFloat(y float64, shouldAdjust bool) float64 { - if y == 0 && !shouldAdjust { - return y - } - if y >= 0 && y <= 69 { - y = 2000 + y - } else if y >= 70 && y <= 99 { - y = 1900 + y - } - return y -} - // NewDuration construct duration with time. func NewDuration(hour, minute, second, microsecond int, fsp int8) Duration { return Duration{ diff --git a/util/ranger/points.go b/util/ranger/points.go index 094bba0e6b489..d9a97b1506b9b 100644 --- a/util/ranger/points.go +++ b/util/ranger/points.go @@ -135,6 +135,13 @@ func getFullRange() []*point { } } +func getNotNullFullRange() []*point { + return []*point{ + {value: types.MinNotNullDatum(), start: true}, + {value: types.MaxValueDatum()}, + } +} + // FullIntRange is used for table range. Since table range cannot accept MaxValueDatum as the max value. // So we need to set it to MaxInt64. func FullIntRange(isUnsigned bool) []*Range { @@ -221,31 +228,58 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []*point { ft *types.FieldType ) - // refineValue refines the constant datum: + // refineValueAndOp refines the constant datum and operator: // 1. for string type since we may eval the constant to another collation instead of its own collation. // 2. for year type since 2-digit year value need adjustment, see https://dev.mysql.com/doc/refman/5.6/en/year.html +<<<<<<< HEAD refineValue := func(col *expression.Column, value *types.Datum) (err error) { if col.RetType.EvalType() == types.ETString && value.Kind() == types.KindString { +======= + refineValueAndOp := func(col *expression.Column, value *types.Datum, op *string) (err error) { + if col.RetType.EvalType() == types.ETString && (value.Kind() == types.KindString || value.Kind() == types.KindBinaryLiteral) { +>>>>>>> 670b5fbcf... ranger: fix the range construction behavior when the column's type is `YEAR` (#23559) value.SetString(value.GetString(), col.RetType.Collate) } if col.GetType().Tp == mysql.TypeYear { - *value, err = types.ConvertDatumToFloatYear(r.sc, *value) + // If the original value is adjusted, we need to change the condition. + // For example, col < 2156. Since the max year is 2155, 2156 is changed to 2155. + // col < 2155 is wrong. It should be col <= 2155. + preValue, err1 := value.ToInt64(r.sc) + if err1 != nil { + return err1 + } + *value, err = value.ConvertToMysqlYear(r.sc, col.RetType) + if errors.ErrorEqual(err, types.ErrInvalidYear) { + // Keep err for EQ and NE. + switch *op { + case ast.GT: + if value.GetInt64() > preValue { + *op = ast.GE + } + err = nil + case ast.LT: + if value.GetInt64() < preValue { + *op = ast.LE + } + err = nil + case ast.GE, ast.LE: + err = nil + } + } } return } - if col, ok := expr.GetArgs()[0].(*expression.Column); ok { + var col *expression.Column + var ok bool + if col, ok = expr.GetArgs()[0].(*expression.Column); ok { ft = col.RetType value, err = expr.GetArgs()[1].Eval(chunk.Row{}) if err != nil { return nil } - err = refineValue(col, &value) - if err != nil { - return nil - } op = expr.FuncName.L } else { - col, ok := expr.GetArgs()[1].(*expression.Column) + col, ok = expr.GetArgs()[1].(*expression.Column) if !ok { return nil } @@ -254,11 +288,6 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []*point { if err != nil { return nil } - err = refineValue(col, &value) - if err != nil { - return nil - } - switch expr.FuncName.L { case ast.GE: op = ast.LE @@ -275,6 +304,15 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []*point { if op != ast.NullEQ && value.IsNull() { return nil } + err = refineValueAndOp(col, &value, &op) + if err != nil { + if op == ast.NE { + // col != an impossible value (not valid year) + return getNotNullFullRange() + } + // col = an impossible value (not valid year) + return nil + } value, op, isValidRange := handleUnsignedCol(ft, value, op) if !isValidRange { @@ -472,10 +510,10 @@ func (r *builder) buildFromIn(expr *expression.ScalarFunction) ([]*point, bool) dt.SetString(dt.GetString(), colCollate) } if expr.GetArgs()[0].GetType().Tp == mysql.TypeYear { - dt, err = types.ConvertDatumToFloatYear(r.sc, dt) + dt, err = dt.ConvertToMysqlYear(r.sc, expr.GetArgs()[0].GetType()) if err != nil { - r.err = ErrUnsupportedType.GenWithStack("expr:%v is not converted to year", e) - return getFullRange(), hasNull + // in (..., an impossible value (not valid year), ...), the range is empty, so skip it. + continue } } var startValue, endValue types.Datum diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 4038e197f8721..194ba1e779dc3 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1530,7 +1530,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { exprStr: `a not in (-1, 1, 2)`, accessConds: "[not(in(test.t.a, -1, 1, 2))]", filterConds: "[]", - resultStr: `[(NULL,0) [0,2001) (2002,+inf]]`, + resultStr: `[(NULL,2001) (2002,+inf]]`, }, { indexPos: 0, @@ -1558,7 +1558,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { exprStr: `a not in (1, 2, 15698)`, accessConds: "[not(in(test.t.a, 1, 2, 15698))]", filterConds: "[]", - resultStr: `[(NULL,2001) (2002,2155] (2155,+inf]]`, + resultStr: `[(NULL,2001) (2002,+inf]]`, }, { indexPos: 0, @@ -1586,7 +1586,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { exprStr: `a != 2156`, accessConds: "[ne(test.t.a, 2156)]", filterConds: "[]", - resultStr: `[[-inf,2155] (2155,+inf]]`, + resultStr: `[[-inf,+inf]]`, }, { exprStr: "a < 99 or a > 01",