diff --git a/cmd/explaintest/r/explain_generate_column_substitute.result b/cmd/explaintest/r/explain_generate_column_substitute.result index 78d631987f1d0..0cb7623929c11 100644 --- a/cmd/explaintest/r/explain_generate_column_substitute.result +++ b/cmd/explaintest/r/explain_generate_column_substitute.result @@ -496,6 +496,19 @@ desc format = 'brief' select b from t; id estRows task access object operator info IndexReader 10000.00 root index:IndexFullScan └─IndexFullScan 10000.00 cop[tikv] table:t, index:b(b) keep order:false, stats:pseudo +create table t01(a varchar(20)); +insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙"); +alter table t01 add index eidx ((concat_ws('expression_index', a, 'test'))); +select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) not like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test')); +a +insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙"); +select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test')); +a +齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙 +齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙 +drop table if exists t1; +create table t1(a char, b varchar(20), c char, d varchar(20)); +alter table t1 add index eidx ((export_set(3, a, c, ',', 5))); create table t02 (a varchar(20)); insert into t02 values ('a'), ('b'), ('c'); select * from t02 where lower(a) < 'c'; diff --git a/cmd/explaintest/t/explain_generate_column_substitute.test b/cmd/explaintest/t/explain_generate_column_substitute.test index be461f9a37184..c6060780d62f5 100644 --- a/cmd/explaintest/t/explain_generate_column_substitute.test +++ b/cmd/explaintest/t/explain_generate_column_substitute.test @@ -217,6 +217,16 @@ create table t(a int, b int as (a+1), key((a+1)), key(b)); desc format = 'brief' select a+1 from t; desc format = 'brief' select b from t; +create table t01(a varchar(20)); +insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙"); +alter table t01 add index eidx ((concat_ws('expression_index', a, 'test'))); +select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) not like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test')); +insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙"); +select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test')); + +drop table if exists t1; +create table t1(a char, b varchar(20), c char, d varchar(20)); +alter table t1 add index eidx ((export_set(3, a, c, ',', 5))); create table t02 (a varchar(20)); insert into t02 values ('a'), ('b'), ('c'); select * from t02 where lower(a) < 'c'; diff --git a/ddl/error.go b/ddl/error.go index 1c2f37ce5751f..42ba073829e72 100644 --- a/ddl/error.go +++ b/ddl/error.go @@ -295,4 +295,6 @@ var ( errFunctionalIndexOnJSONOrGeometryFunction = dbterror.ClassDDL.NewStd(mysql.ErrFunctionalIndexOnJSONOrGeometryFunction) // errDependentByFunctionalIndex returns when the dropped column depends by expression index. errDependentByFunctionalIndex = dbterror.ClassDDL.NewStd(mysql.ErrDependentByFunctionalIndex) + // errFunctionalIndexOnBlob when the expression of expression index returns blob or text. + errFunctionalIndexOnBlob = dbterror.ClassDDL.NewStd(mysql.ErrFunctionalIndexOnBlob) ) diff --git a/ddl/index.go b/ddl/index.go index 244e177f2fb31..27cfb7ba1e8e7 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -140,6 +140,9 @@ func checkIndexColumn(col *model.ColumnInfo, indexColumnLen int) error { // Length must be specified and non-zero for BLOB and TEXT column indexes. if types.IsTypeBlob(col.FieldType.Tp) { if indexColumnLen == types.UnspecifiedLength { + if col.Hidden { + return errFunctionalIndexOnBlob + } return errors.Trace(errBlobKeyWithoutLength.GenWithStackByArgs(col.Name.O)) } if indexColumnLen == types.ErrorLength { diff --git a/errno/errcode.go b/errno/errcode.go index 1b8f1f0bc4d55..6e13223d129aa 100644 --- a/errno/errcode.go +++ b/errno/errcode.go @@ -891,7 +891,7 @@ const ( ErrFunctionalIndexRefAutoIncrement = 3754 ErrCannotDropColumnFunctionalIndex = 3755 ErrFunctionalIndexPrimaryKey = 3756 - ErrFunctionalIndexOnLob = 3757 + ErrFunctionalIndexOnBlob = 3757 ErrFunctionalIndexFunctionIsNotAllowed = 3758 ErrFulltextFunctionalIndex = 3759 ErrSpatialFunctionalIndex = 3760 diff --git a/errno/errname.go b/errno/errname.go index df323a213220e..b52e528eb146e 100644 --- a/errno/errname.go +++ b/errno/errname.go @@ -887,7 +887,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrFunctionalIndexRefAutoIncrement: mysql.Message("Expression index '%s' cannot refer to an auto-increment column", nil), ErrCannotDropColumnFunctionalIndex: mysql.Message("Cannot drop column '%s' because it is used by an expression index. In order to drop the column, you must remove the expression index", nil), ErrFunctionalIndexPrimaryKey: mysql.Message("The primary key cannot be an expression index", nil), - ErrFunctionalIndexOnLob: mysql.Message("Cannot create an expression index on an expression that returns a BLOB or TEXT. Please consider using CAST", nil), + ErrFunctionalIndexOnBlob: mysql.Message("Cannot create an expression index on an expression that returns a BLOB or TEXT. Please consider using CAST", nil), ErrFunctionalIndexFunctionIsNotAllowed: mysql.Message("Expression of expression index '%s' contains a disallowed function", nil), ErrFulltextFunctionalIndex: mysql.Message("Fulltext expression index is not supported", nil), ErrSpatialFunctionalIndex: mysql.Message("Spatial expression index is not supported", nil), diff --git a/expression/builtin_string.go b/expression/builtin_string.go index f81de802c49ef..94210daf924ba 100644 --- a/expression/builtin_string.go +++ b/expression/builtin_string.go @@ -366,8 +366,8 @@ func (c *concatWSFunctionClass) getFunction(ctx sessionctx.Context, args []Expre bf.tp.Flen = mysql.MaxBlobWidth logutil.BgLogger().Warn("unexpected `Flen` value(-1) in CONCAT_WS's args", zap.Int("arg's index", i)) } - bf.tp.Flen += argType.Flen } + bf.tp.Flen += argType.Flen } // add separator @@ -1588,8 +1588,8 @@ func (c *hexFunctionClass) getFunction(ctx sessionctx.Context, args []Expression return nil, err } bf.tp.Charset, bf.tp.Collate = ctx.GetSessionVars().GetCharsetInfo() - // Use UTF-8 as default - bf.tp.Flen = args[0].GetType().Flen * 3 * 2 + // Use UTF8MB4 as default. + bf.tp.Flen = args[0].GetType().Flen * 4 * 2 sig := &builtinHexStrArgSig{bf} sig.setPbCode(tipb.ScalarFuncSig_HexStrArg) return sig, nil @@ -1665,10 +1665,10 @@ func (c *unhexFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi argEvalTp := argType.EvalType() switch argEvalTp { case types.ETString, types.ETDatetime, types.ETTimestamp, types.ETDuration, types.ETJson: - // Use UTF-8 as default charset, so there're (Flen * 3 + 1) / 2 byte-pairs - retFlen = (argType.Flen*3 + 1) / 2 + // Use UTF8MB4 as default charset, so there're (Flen * 4 + 1) / 2 byte-pairs. + retFlen = (argType.Flen*4 + 1) / 2 case types.ETInt, types.ETReal, types.ETDecimal: - // For number value, there're (Flen + 1) / 2 byte-pairs + // For number value, there're (Flen + 1) / 2 byte-pairs. retFlen = (argType.Flen + 1) / 2 default: return nil, errors.Errorf("Unhex invalid args, need int or string but get %s", argType) @@ -3073,7 +3073,16 @@ func (c *exportSetFunctionClass) getFunction(ctx sessionctx.Context, args []Expr if err != nil { return nil, err } - bf.tp.Flen = mysql.MaxBlobWidth + // Calculate the flen as MySQL does. + l := args[1].GetType().Flen + if args[2].GetType().Flen > l { + l = args[2].GetType().Flen + } + sepL := 1 + if len(args) > 3 { + sepL = args[3].GetType().Flen + } + bf.tp.Flen = (l*64 + sepL*63) * 4 switch len(args) { case 3: sig = &builtinExportSet3ArgSig{bf} @@ -3407,7 +3416,15 @@ func (c *fromBase64FunctionClass) getFunction(ctx sessionctx.Context, args []Exp if err != nil { return nil, err } - bf.tp.Flen = mysql.MaxBlobWidth + // The calculation of Flen is the same as MySQL. + if args[0].GetType().Flen == types.UnspecifiedLength { + bf.tp.Flen = types.UnspecifiedLength + } else { + bf.tp.Flen = args[0].GetType().Flen * 3 + if bf.tp.Flen > mysql.MaxBlobWidth { + bf.tp.Flen = mysql.MaxBlobWidth + } + } valStr, _ := ctx.GetSessionVars().GetSystemVar(variable.MaxAllowedPacket) maxAllowedPacket, err := strconv.ParseUint(valStr, 10, 64) diff --git a/expression/typeinfer_test.go b/expression/typeinfer_test.go index 335fa4b1aaa25..293ea722a2824 100644 --- a/expression/typeinfer_test.go +++ b/expression/typeinfer_test.go @@ -244,8 +244,8 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase { {"CONCAT(c_bchar, 0x80)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 23, types.UnspecifiedLength}, {"CONCAT('T', 'i', 'DB')", mysql.TypeVarString, charset.CharsetUTF8MB4, 0 | mysql.NotNullFlag, 4, types.UnspecifiedLength}, {"CONCAT('T', 'i', 'DB', c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 24, types.UnspecifiedLength}, - {"CONCAT_WS('-', 'T', 'i', 'DB')", mysql.TypeVarString, charset.CharsetUTF8MB4, 0 | mysql.NotNullFlag, 6, types.UnspecifiedLength}, - {"CONCAT_WS(',', 'TiDB', c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 25, types.UnspecifiedLength}, + {"CONCAT_WS('-', 'T', 'i', 'DB')", mysql.TypeVarString, charset.CharsetUTF8MB4, 0 | mysql.NotNullFlag, 7, types.UnspecifiedLength}, + {"CONCAT_WS(',', 'TiDB', c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 26, types.UnspecifiedLength}, {"left(c_int_d, c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, {"right(c_int_d, c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, {"lower(c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, @@ -261,10 +261,10 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase { {"bit_length(c_char)", mysql.TypeLonglong, charset.CharsetBin, mysql.BinaryFlag, 10, 0}, {"substring_index(c_int_d, '.', 1)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, {"substring_index(c_binary, '.', 1)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 20, types.UnspecifiedLength}, - {"hex(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 120, types.UnspecifiedLength}, + {"hex(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 160, types.UnspecifiedLength}, {"hex(c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 22, types.UnspecifiedLength}, {"unhex(c_int_d)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 6, types.UnspecifiedLength}, - {"unhex(c_char)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 30, types.UnspecifiedLength}, + {"unhex(c_char)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 40, types.UnspecifiedLength}, {"ltrim(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, {"ltrim(c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 20, types.UnspecifiedLength}, {"rtrim(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, @@ -301,22 +301,22 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase { {"rpad(c_char, c_int_d, c_binary)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, {"rpad(c_char, c_int_d, c_char )", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_int_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_bigint_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_float_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_double_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_decimal )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_datetime )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_time_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_timestamp_d)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_char )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_varchar )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_text_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_binary )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_varbinary )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_blob_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_set )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"from_base64(c_enum )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, + {"from_base64(c_int_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength}, + {"from_base64(c_bigint_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength}, + {"from_base64(c_float_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, types.UnspecifiedLength, types.UnspecifiedLength}, + {"from_base64(c_double_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, types.UnspecifiedLength, types.UnspecifiedLength}, + {"from_base64(c_decimal )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 24, types.UnspecifiedLength}, + {"from_base64(c_datetime )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 66, types.UnspecifiedLength}, + {"from_base64(c_time_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 30, types.UnspecifiedLength}, + {"from_base64(c_timestamp_d)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 57, types.UnspecifiedLength}, + {"from_base64(c_char )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength}, + {"from_base64(c_varchar )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength}, + {"from_base64(c_text_d )", mysql.TypeMediumBlob, charset.CharsetBin, mysql.BinaryFlag, 196605, types.UnspecifiedLength}, + {"from_base64(c_binary )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength}, + {"from_base64(c_varbinary )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength}, + {"from_base64(c_blob_d )", mysql.TypeMediumBlob, charset.CharsetBin, mysql.BinaryFlag, 196605, types.UnspecifiedLength}, + {"from_base64(c_set )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 15, types.UnspecifiedLength}, + {"from_base64(c_enum )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 3, types.UnspecifiedLength}, {"bin(c_int_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 64, types.UnspecifiedLength}, {"bin(c_bigint_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 64, types.UnspecifiedLength}, @@ -472,9 +472,9 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase { {"insert(c_binary, c_int_d, c_int_d, c_varchar)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, {"insert(c_binary, c_int_d, c_int_d, c_binary)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"export_set(c_double_d, c_text_d, c_text_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"export_set(c_double_d, c_text_d, c_text_d, c_text_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength}, - {"export_set(c_double_d, c_text_d, c_text_d, c_text_d, c_int_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength}, + {"export_set(c_double_d, c_text_d, c_text_d)", mysql.TypeMediumBlob, charset.CharsetUTF8MB4, 0, 16777212, types.UnspecifiedLength}, + {"export_set(c_double_d, c_text_d, c_text_d, c_text_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, 33291780, types.UnspecifiedLength}, + {"export_set(c_double_d, c_text_d, c_text_d, c_text_d, c_int_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, 33291780, types.UnspecifiedLength}, {"format(c_double_d, c_double_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength}, {"format(c_double_d, c_double_d, c_binary)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength},