From d312f8aa2c4326ad7251c54c8aa2532de0fcb2f8 Mon Sep 17 00:00:00 2001 From: pingcap-github-bot Date: Mon, 16 Dec 2019 12:27:52 +0800 Subject: [PATCH] ddl: fix index length calculation (#13779) --- ddl/db_integration_test.go | 16 ++--- ddl/index.go | 59 +++++++++------- session/session_test.go | 136 ++++++++++++++++++++----------------- 3 files changed, 112 insertions(+), 99 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 4a021883d33d2..48090df027a37 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -1478,19 +1478,11 @@ func (s *testIntegrationSuite4) TestAlterColumn(c *C) { c.Assert(err, NotNil) s.tk.MustExec("drop table if exists t") - // TODO: fix me, below sql should execute successfully. Currently, the result of calculate key length is wrong. - //s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3071),index(a),index(a,b),index (c,d));") - s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3068),index(a),index(a,b),index (c,d));") - _, err = s.tk.Exec("alter table t1 modify column a varchar(3000);") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes") + s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3071),index(a),index(a,b),index (c,d)) charset = ascii;") + s.tk.MustGetErrCode("alter table t1 modify column a varchar(3000);", mysql.ErrTooLongKey) // check modify column with rename column. - _, err = s.tk.Exec("alter table t1 change column a x varchar(3000);") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes") - _, err = s.tk.Exec("alter table t1 modify column c bigint;") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes") + s.tk.MustGetErrCode("alter table t1 change column a x varchar(3000);", mysql.ErrTooLongKey) + s.tk.MustGetErrCode("alter table t1 modify column c bigint;", mysql.ErrTooLongKey) s.tk.MustExec("drop table if exists multi_unique") s.tk.MustExec("create table multi_unique (a int unique unique)") diff --git a/ddl/index.go b/ddl/index.go index b9b5f28b8f574..71912361e8d30 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" ddlutil "github.com/pingcap/tidb/ddl/util" @@ -160,36 +161,46 @@ func checkIndexColumn(col *model.ColumnInfo, ic *ast.IndexColName) error { return nil } +// getIndexColumnLength calculate the bytes number required in an index column. func getIndexColumnLength(col *model.ColumnInfo, colLen int) (int, error) { - // Take care of the sum of length of all index columns. + length := types.UnspecifiedLength if colLen != types.UnspecifiedLength { - return colLen, nil - } - // Specified data types. - if col.Flen != types.UnspecifiedLength { - // Special case for the bit type. - if col.FieldType.Tp == mysql.TypeBit { - return (col.Flen + 7) >> 3, nil - } - return col.Flen, nil - + length = colLen + } else if col.Flen != types.UnspecifiedLength { + length = col.Flen } - length, ok := mysql.DefaultLengthOfMysqlTypes[col.FieldType.Tp] - if !ok { - return length, errUnknownTypeLength.GenWithStackByArgs(col.FieldType.Tp) + switch col.Tp { + case mysql.TypeBit: + return (length + 7) >> 3, nil + case mysql.TypeVarchar, mysql.TypeString: + // Different charsets occupy different numbers of bytes on each character. + desc, err := charset.GetCharsetDesc(col.Charset) + if err != nil { + return 0, errUnsupportedCharset.GenWithStackByArgs(col.Charset, col.Collate) + } + return desc.Maxlen * length, nil + case mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: + return length, nil + case mysql.TypeTiny, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeShort: + return mysql.DefaultLengthOfMysqlTypes[col.Tp], nil + case mysql.TypeFloat: + if length <= mysql.MaxFloatPrecisionLength { + return mysql.DefaultLengthOfMysqlTypes[mysql.TypeFloat], nil + } + return mysql.DefaultLengthOfMysqlTypes[mysql.TypeDouble], nil + case mysql.TypeDecimal, mysql.TypeNewDecimal: + return calcBytesLengthForDecimal(length), nil + case mysql.TypeYear, mysql.TypeDate, mysql.TypeDuration, mysql.TypeDatetime, mysql.TypeTimestamp: + return mysql.DefaultLengthOfMysqlTypes[col.Tp], nil + default: + return length, nil } +} - // Special case for time fraction. - if types.IsTypeFractionable(col.FieldType.Tp) && - col.FieldType.Decimal != types.UnspecifiedLength { - decimalLength, ok := mysql.DefaultLengthOfTimeFraction[col.FieldType.Decimal] - if !ok { - return length, errUnknownFractionLength.GenWithStackByArgs(col.FieldType.Tp, col.FieldType.Decimal) - } - length += decimalLength - } - return length, nil +// Decimal using a binary format that packs nine decimal (base 10) digits into four bytes. +func calcBytesLengthForDecimal(m int) int { + return (m / 9 * 4) + ((m%9)+1)/2 } func buildIndexInfo(tblInfo *model.TableInfo, indexName model.CIStr, idxColNames []*ast.IndexColName, state model.SchemaState) (*model.IndexInfo, error) { diff --git a/session/session_test.go b/session/session_test.go index 87624a73cbcff..05f549cf1fa09 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -1206,78 +1206,88 @@ func (s *testSessionSuite) TestFieldText(c *C) { func (s *testSessionSuite) TestIndexMaxLength(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists t;") + tk.MustExec("create database test_index_max_length") + tk.MustExec("use test_index_max_length") // create simple index at table creation - _, err := tk.Exec("create table t (c1 varchar(3073), index(c1));") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) + tk.MustGetErrCode("create table t (c1 varchar(3073), index(c1)) charset = ascii;", mysql.ErrTooLongKey) // create simple index after table creation - tk.MustExec("create table t (c1 varchar(3073));") - _, err = tk.Exec("create index idx_c1 on t(c1) ") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) + tk.MustExec("create table t (c1 varchar(3073)) charset = ascii;") + tk.MustGetErrCode("create index idx_c1 on t(c1) ", mysql.ErrTooLongKey) + tk.MustExec("drop table t;") // create compound index at table creation - tk.MustExec("drop table if exists t;") - _, err = tk.Exec("create table t (c1 varchar(3072), c2 varchar(1), index(c1, c2));") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - _, err = tk.Exec("create table t (c1 varchar(3072), c2 char(1), index(c1, c2));") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - _, err = tk.Exec("create table t (c1 varchar(3072), c2 char, index(c1, c2));") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - _, err = tk.Exec("create table t (c1 varchar(3072), c2 date, index(c1, c2));") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - _, err = tk.Exec("create table t (c1 varchar(3068), c2 timestamp(1), index(c1, c2));") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - tk.MustExec("create table t (c1 varchar(3068), c2 bit(26), index(c1, c2));") // 26 bit = 4 bytes - tk.MustExec("drop table if exists t;") - tk.MustExec("create table t (c1 varchar(3068), c2 bit(32), index(c1, c2));") // 32 bit = 4 bytes - tk.MustExec("drop table if exists t;") - _, err = tk.Exec("create table t (c1 varchar(3068), c2 bit(33), index(c1, c2));") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) + tk.MustGetErrCode("create table t (c1 varchar(3072), c2 varchar(1), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey) + tk.MustGetErrCode("create table t (c1 varchar(3072), c2 char(1), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey) + tk.MustGetErrCode("create table t (c1 varchar(3072), c2 char, index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey) + tk.MustGetErrCode("create table t (c1 varchar(3072), c2 date, index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey) + tk.MustGetErrCode("create table t (c1 varchar(3069), c2 timestamp(1), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey) + + tk.MustExec("create table t (c1 varchar(3068), c2 bit(26), index(c1, c2)) charset = ascii;") // 26 bit = 4 bytes + tk.MustExec("drop table t;") + tk.MustExec("create table t (c1 varchar(3068), c2 bit(32), index(c1, c2)) charset = ascii;") // 32 bit = 4 bytes + tk.MustExec("drop table t;") + tk.MustGetErrCode("create table t (c1 varchar(3068), c2 bit(33), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey) // create compound index after table creation - tk.MustExec("create table t (c1 varchar(3072), c2 varchar(1));") - _, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - tk.MustExec("drop table if exists t;") - tk.MustExec("create table t (c1 varchar(3072), c2 char(1));") - _, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - tk.MustExec("drop table if exists t;") - tk.MustExec("create table t (c1 varchar(3072), c2 char);") - _, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) - - tk.MustExec("drop table if exists t;") - tk.MustExec("create table t (c1 varchar(3072), c2 date);") - _, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) + tk.MustExec("create table t (c1 varchar(3072), c2 varchar(1)) charset = ascii;") + tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey) + tk.MustExec("drop table t;") + + tk.MustExec("create table t (c1 varchar(3072), c2 char(1)) charset = ascii;") + tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey) + tk.MustExec("drop table t;") + + tk.MustExec("create table t (c1 varchar(3072), c2 char) charset = ascii;") + tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey) + tk.MustExec("drop table t;") + + tk.MustExec("create table t (c1 varchar(3072), c2 date) charset = ascii;") + tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey) + tk.MustExec("drop table t;") + + tk.MustExec("create table t (c1 varchar(3069), c2 timestamp(1)) charset = ascii;") + tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey) + tk.MustExec("drop table t;") + + // Test charsets other than `ascii`. + assertCharsetLimit := func(charset string, bytesPerChar int) { + base := 3072 / bytesPerChar + tk.MustGetErrCode(fmt.Sprintf("create table t (a varchar(%d) primary key) charset=%s", base+1, charset), mysql.ErrTooLongKey) + tk.MustExec(fmt.Sprintf("create table t (a varchar(%d) primary key) charset=%s", base, charset)) + tk.MustExec("drop table if exists t") + } + assertCharsetLimit("binary", 1) + assertCharsetLimit("latin1", 1) + assertCharsetLimit("utf8", 3) + assertCharsetLimit("utf8mb4", 4) + + // Test types bit length limit. + assertTypeLimit := func(tp string, limitBitLength int) { + base := 3072 - limitBitLength + tk.MustGetErrCode(fmt.Sprintf("create table t (a blob(10000), b %s, index idx(a(%d), b))", tp, base+1), mysql.ErrTooLongKey) + tk.MustExec(fmt.Sprintf("create table t (a blob(10000), b %s, index idx(a(%d), b))", tp, base)) + tk.MustExec("drop table if exists t") + } - tk.MustExec("drop table if exists t;") - tk.MustExec("create table t (c1 varchar(3068), c2 timestamp(1));") - _, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);") - // ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes - c.Assert(err, NotNil) + assertTypeLimit("tinyint", 1) + assertTypeLimit("smallint", 2) + assertTypeLimit("mediumint", 3) + assertTypeLimit("int", 4) + assertTypeLimit("integer", 4) + assertTypeLimit("bigint", 8) + assertTypeLimit("float", 4) + assertTypeLimit("float(24)", 4) + assertTypeLimit("float(25)", 8) + assertTypeLimit("decimal(9)", 4) + assertTypeLimit("decimal(10)", 5) + assertTypeLimit("decimal(17)", 8) + assertTypeLimit("year", 1) + assertTypeLimit("date", 3) + assertTypeLimit("time", 3) + assertTypeLimit("datetime", 8) + assertTypeLimit("timestamp", 4) } func (s *testSessionSuite) TestIndexColumnLength(c *C) {