Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl, expression: fix some string functions' flen and refine error message for expression index #27376

Merged
merged 14 commits into from
Aug 20, 2021
Merged
13 changes: 13 additions & 0 deletions cmd/explaintest/r/explain_generate_column_substitute.result
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,19 @@ desc format = 'brief' select b from t;
id estRows task access object operator info
IndexReader 10000.00 root index:IndexFullScan
└─IndexFullScan 10000.00 cop[tikv] table:t, index:b(b) keep order:false, stats:pseudo
create table t01(a varchar(20));
insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙");
alter table t01 add index eidx ((concat_ws('expression_index', a, 'test')));
select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) not like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test'));
a
insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙");
select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test'));
a
齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙
齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙
drop table if exists t1;
create table t1(a char, b varchar(20), c char, d varchar(20));
alter table t1 add index eidx ((export_set(3, a, c, ',', 5)));
create table t02 (a varchar(20));
insert into t02 values ('a'), ('b'), ('c');
select * from t02 where lower(a) < 'c';
Expand Down
10 changes: 10 additions & 0 deletions cmd/explaintest/t/explain_generate_column_substitute.test
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,16 @@ create table t(a int, b int as (a+1), key((a+1)), key(b));
desc format = 'brief' select a+1 from t;
desc format = 'brief' select b from t;

create table t01(a varchar(20));
insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙");
alter table t01 add index eidx ((concat_ws('expression_index', a, 'test')));
select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) not like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test'));
insert into t01 values ("齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙");
select * from t01 use index (eidx) where (concat_ws('expression_index', a, 'test')) like (concat_ws('expression_index', "齆斮聒蚆髙锐潊贩哨啅捸爖斥圱犳飁綴纜牖蚙", 'test'));

drop table if exists t1;
create table t1(a char, b varchar(20), c char, d varchar(20));
alter table t1 add index eidx ((export_set(3, a, c, ',', 5)));
create table t02 (a varchar(20));
insert into t02 values ('a'), ('b'), ('c');
select * from t02 where lower(a) < 'c';
Expand Down
2 changes: 2 additions & 0 deletions ddl/error.go
Original file line number Diff line number Diff line change
Expand Up @@ -295,4 +295,6 @@ var (
errFunctionalIndexOnJSONOrGeometryFunction = dbterror.ClassDDL.NewStd(mysql.ErrFunctionalIndexOnJSONOrGeometryFunction)
// errDependentByFunctionalIndex returns when the dropped column depends by expression index.
errDependentByFunctionalIndex = dbterror.ClassDDL.NewStd(mysql.ErrDependentByFunctionalIndex)
// errFunctionalIndexOnBlob when the expression of expression index returns blob or text.
errFunctionalIndexOnBlob = dbterror.ClassDDL.NewStd(mysql.ErrFunctionalIndexOnBlob)
)
3 changes: 3 additions & 0 deletions ddl/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,9 @@ func checkIndexColumn(col *model.ColumnInfo, indexColumnLen int) error {
// Length must be specified and non-zero for BLOB and TEXT column indexes.
if types.IsTypeBlob(col.FieldType.Tp) {
if indexColumnLen == types.UnspecifiedLength {
if col.Hidden {
return errFunctionalIndexOnBlob
}
return errors.Trace(errBlobKeyWithoutLength.GenWithStackByArgs(col.Name.O))
}
if indexColumnLen == types.ErrorLength {
Expand Down
2 changes: 1 addition & 1 deletion errno/errcode.go
Original file line number Diff line number Diff line change
Expand Up @@ -891,7 +891,7 @@ const (
ErrFunctionalIndexRefAutoIncrement = 3754
ErrCannotDropColumnFunctionalIndex = 3755
ErrFunctionalIndexPrimaryKey = 3756
ErrFunctionalIndexOnLob = 3757
ErrFunctionalIndexOnBlob = 3757
AilinKid marked this conversation as resolved.
Show resolved Hide resolved
ErrFunctionalIndexFunctionIsNotAllowed = 3758
ErrFulltextFunctionalIndex = 3759
ErrSpatialFunctionalIndex = 3760
Expand Down
2 changes: 1 addition & 1 deletion errno/errname.go
Original file line number Diff line number Diff line change
Expand Up @@ -887,7 +887,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrFunctionalIndexRefAutoIncrement: mysql.Message("Expression index '%s' cannot refer to an auto-increment column", nil),
ErrCannotDropColumnFunctionalIndex: mysql.Message("Cannot drop column '%s' because it is used by an expression index. In order to drop the column, you must remove the expression index", nil),
ErrFunctionalIndexPrimaryKey: mysql.Message("The primary key cannot be an expression index", nil),
ErrFunctionalIndexOnLob: mysql.Message("Cannot create an expression index on an expression that returns a BLOB or TEXT. Please consider using CAST", nil),
ErrFunctionalIndexOnBlob: mysql.Message("Cannot create an expression index on an expression that returns a BLOB or TEXT. Please consider using CAST", nil),
ErrFunctionalIndexFunctionIsNotAllowed: mysql.Message("Expression of expression index '%s' contains a disallowed function", nil),
ErrFulltextFunctionalIndex: mysql.Message("Fulltext expression index is not supported", nil),
ErrSpatialFunctionalIndex: mysql.Message("Spatial expression index is not supported", nil),
Expand Down
33 changes: 25 additions & 8 deletions expression/builtin_string.go
Original file line number Diff line number Diff line change
Expand Up @@ -366,8 +366,8 @@ func (c *concatWSFunctionClass) getFunction(ctx sessionctx.Context, args []Expre
bf.tp.Flen = mysql.MaxBlobWidth
logutil.BgLogger().Warn("unexpected `Flen` value(-1) in CONCAT_WS's args", zap.Int("arg's index", i))
}
bf.tp.Flen += argType.Flen
}
bf.tp.Flen += argType.Flen
}

// add separator
Expand Down Expand Up @@ -1588,8 +1588,8 @@ func (c *hexFunctionClass) getFunction(ctx sessionctx.Context, args []Expression
return nil, err
}
bf.tp.Charset, bf.tp.Collate = ctx.GetSessionVars().GetCharsetInfo()
// Use UTF-8 as default
bf.tp.Flen = args[0].GetType().Flen * 3 * 2
// Use UTF8MB4 as default.
bf.tp.Flen = args[0].GetType().Flen * 4 * 2
sig := &builtinHexStrArgSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_HexStrArg)
return sig, nil
Expand Down Expand Up @@ -1665,10 +1665,10 @@ func (c *unhexFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi
argEvalTp := argType.EvalType()
switch argEvalTp {
case types.ETString, types.ETDatetime, types.ETTimestamp, types.ETDuration, types.ETJson:
// Use UTF-8 as default charset, so there're (Flen * 3 + 1) / 2 byte-pairs
retFlen = (argType.Flen*3 + 1) / 2
// Use UTF8MB4 as default charset, so there're (Flen * 4 + 1) / 2 byte-pairs.
retFlen = (argType.Flen*4 + 1) / 2
case types.ETInt, types.ETReal, types.ETDecimal:
// For number value, there're (Flen + 1) / 2 byte-pairs
// For number value, there're (Flen + 1) / 2 byte-pairs.
retFlen = (argType.Flen + 1) / 2
default:
return nil, errors.Errorf("Unhex invalid args, need int or string but get %s", argType)
Expand Down Expand Up @@ -3073,7 +3073,16 @@ func (c *exportSetFunctionClass) getFunction(ctx sessionctx.Context, args []Expr
if err != nil {
return nil, err
}
bf.tp.Flen = mysql.MaxBlobWidth
// Calculate the flen as MySQL does.
l := args[1].GetType().Flen
if args[2].GetType().Flen > l {
l = args[2].GetType().Flen
}
sepL := 1
if len(args) > 3 {
sepL = args[3].GetType().Flen
}
bf.tp.Flen = (l*64 + sepL*63) * 4
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any calculation reference link for this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refer to @wshwsh12 's comment.

switch len(args) {
case 3:
sig = &builtinExportSet3ArgSig{bf}
Expand Down Expand Up @@ -3407,7 +3416,15 @@ func (c *fromBase64FunctionClass) getFunction(ctx sessionctx.Context, args []Exp
if err != nil {
return nil, err
}
bf.tp.Flen = mysql.MaxBlobWidth
// The calculation of Flen is the same as MySQL.
if args[0].GetType().Flen == types.UnspecifiedLength {
bf.tp.Flen = types.UnspecifiedLength
} else {
bf.tp.Flen = args[0].GetType().Flen * 3
if bf.tp.Flen > mysql.MaxBlobWidth {
bf.tp.Flen = mysql.MaxBlobWidth
}
}

valStr, _ := ctx.GetSessionVars().GetSystemVar(variable.MaxAllowedPacket)
maxAllowedPacket, err := strconv.ParseUint(valStr, 10, 64)
Expand Down
46 changes: 23 additions & 23 deletions expression/typeinfer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase {
{"CONCAT(c_bchar, 0x80)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 23, types.UnspecifiedLength},
{"CONCAT('T', 'i', 'DB')", mysql.TypeVarString, charset.CharsetUTF8MB4, 0 | mysql.NotNullFlag, 4, types.UnspecifiedLength},
{"CONCAT('T', 'i', 'DB', c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 24, types.UnspecifiedLength},
{"CONCAT_WS('-', 'T', 'i', 'DB')", mysql.TypeVarString, charset.CharsetUTF8MB4, 0 | mysql.NotNullFlag, 6, types.UnspecifiedLength},
{"CONCAT_WS(',', 'TiDB', c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 25, types.UnspecifiedLength},
{"CONCAT_WS('-', 'T', 'i', 'DB')", mysql.TypeVarString, charset.CharsetUTF8MB4, 0 | mysql.NotNullFlag, 7, types.UnspecifiedLength},
{"CONCAT_WS(',', 'TiDB', c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 26, types.UnspecifiedLength},
{"left(c_int_d, c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength},
{"right(c_int_d, c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength},
{"lower(c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength},
Expand All @@ -261,10 +261,10 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase {
{"bit_length(c_char)", mysql.TypeLonglong, charset.CharsetBin, mysql.BinaryFlag, 10, 0},
{"substring_index(c_int_d, '.', 1)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength},
{"substring_index(c_binary, '.', 1)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 20, types.UnspecifiedLength},
{"hex(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 120, types.UnspecifiedLength},
{"hex(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 160, types.UnspecifiedLength},
{"hex(c_int_d)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 22, types.UnspecifiedLength},
{"unhex(c_int_d)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 6, types.UnspecifiedLength},
{"unhex(c_char)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 30, types.UnspecifiedLength},
{"unhex(c_char)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 40, types.UnspecifiedLength},
{"ltrim(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength},
{"ltrim(c_binary)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 20, types.UnspecifiedLength},
{"rtrim(c_char)", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength},
Expand Down Expand Up @@ -301,22 +301,22 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase {
{"rpad(c_char, c_int_d, c_binary)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"rpad(c_char, c_int_d, c_char )", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength},

{"from_base64(c_int_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_bigint_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_float_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_double_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_decimal )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_datetime )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_time_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_timestamp_d)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_char )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_varchar )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_text_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_binary )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_varbinary )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_blob_d )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_set )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_enum )", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"from_base64(c_int_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength},
{"from_base64(c_bigint_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength},
{"from_base64(c_float_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, types.UnspecifiedLength, types.UnspecifiedLength},
{"from_base64(c_double_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, types.UnspecifiedLength, types.UnspecifiedLength},
{"from_base64(c_decimal )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 24, types.UnspecifiedLength},
{"from_base64(c_datetime )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 66, types.UnspecifiedLength},
{"from_base64(c_time_d )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 30, types.UnspecifiedLength},
{"from_base64(c_timestamp_d)", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 57, types.UnspecifiedLength},
{"from_base64(c_char )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength},
{"from_base64(c_varchar )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength},
{"from_base64(c_text_d )", mysql.TypeMediumBlob, charset.CharsetBin, mysql.BinaryFlag, 196605, types.UnspecifiedLength},
{"from_base64(c_binary )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength},
{"from_base64(c_varbinary )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 60, types.UnspecifiedLength},
{"from_base64(c_blob_d )", mysql.TypeMediumBlob, charset.CharsetBin, mysql.BinaryFlag, 196605, types.UnspecifiedLength},
{"from_base64(c_set )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 15, types.UnspecifiedLength},
{"from_base64(c_enum )", mysql.TypeVarString, charset.CharsetBin, mysql.BinaryFlag, 3, types.UnspecifiedLength},

{"bin(c_int_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 64, types.UnspecifiedLength},
{"bin(c_bigint_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 64, types.UnspecifiedLength},
Expand Down Expand Up @@ -472,9 +472,9 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase {
{"insert(c_binary, c_int_d, c_int_d, c_varchar)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"insert(c_binary, c_int_d, c_int_d, c_binary)", mysql.TypeLongBlob, charset.CharsetBin, mysql.BinaryFlag, mysql.MaxBlobWidth, types.UnspecifiedLength},

{"export_set(c_double_d, c_text_d, c_text_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"export_set(c_double_d, c_text_d, c_text_d, c_text_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"export_set(c_double_d, c_text_d, c_text_d, c_text_d, c_int_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"export_set(c_double_d, c_text_d, c_text_d)", mysql.TypeMediumBlob, charset.CharsetUTF8MB4, 0, 16777212, types.UnspecifiedLength},
{"export_set(c_double_d, c_text_d, c_text_d, c_text_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, 33291780, types.UnspecifiedLength},
{"export_set(c_double_d, c_text_d, c_text_d, c_text_d, c_int_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, 33291780, types.UnspecifiedLength},

{"format(c_double_d, c_double_d)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength},
{"format(c_double_d, c_double_d, c_binary)", mysql.TypeLongBlob, charset.CharsetUTF8MB4, 0, mysql.MaxBlobWidth, types.UnspecifiedLength},
Expand Down