Skip to content

Commit

Permalink
ddl: fix index length calculation (#13779)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sre-bot @bb7133
sre-bot authored and bb7133 committed Dec 16, 2019
1 parent be7cb4d commit d312f8a
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 99 deletions.
16 changes: 4 additions & 12 deletions ddl/db_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1478,19 +1478,11 @@ func (s *testIntegrationSuite4) TestAlterColumn(c *C) {
c.Assert(err, NotNil)

s.tk.MustExec("drop table if exists t")
// TODO: fix me, below sql should execute successfully. Currently, the result of calculate key length is wrong.
//s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3071),index(a),index(a,b),index (c,d));")
s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3068),index(a),index(a,b),index (c,d));")
_, err = s.tk.Exec("alter table t1 modify column a varchar(3000);")
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes")
s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3071),index(a),index(a,b),index (c,d)) charset = ascii;")
s.tk.MustGetErrCode("alter table t1 modify column a varchar(3000);", mysql.ErrTooLongKey)
// check modify column with rename column.
_, err = s.tk.Exec("alter table t1 change column a x varchar(3000);")
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes")
_, err = s.tk.Exec("alter table t1 modify column c bigint;")
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes")
s.tk.MustGetErrCode("alter table t1 change column a x varchar(3000);", mysql.ErrTooLongKey)
s.tk.MustGetErrCode("alter table t1 modify column c bigint;", mysql.ErrTooLongKey)

s.tk.MustExec("drop table if exists multi_unique")
s.tk.MustExec("create table multi_unique (a int unique unique)")
Expand Down
59 changes: 35 additions & 24 deletions ddl/index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/ast"
"github.com/pingcap/parser/charset"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
ddlutil "github.com/pingcap/tidb/ddl/util"
Expand Down Expand Up @@ -160,36 +161,46 @@ func checkIndexColumn(col *model.ColumnInfo, ic *ast.IndexColName) error {
return nil
}

// getIndexColumnLength calculate the bytes number required in an index column.
func getIndexColumnLength(col *model.ColumnInfo, colLen int) (int, error) {
// Take care of the sum of length of all index columns.
length := types.UnspecifiedLength
if colLen != types.UnspecifiedLength {
return colLen, nil
}
// Specified data types.
if col.Flen != types.UnspecifiedLength {
// Special case for the bit type.
if col.FieldType.Tp == mysql.TypeBit {
return (col.Flen + 7) >> 3, nil
}
return col.Flen, nil

length = colLen
} else if col.Flen != types.UnspecifiedLength {
length = col.Flen
}

length, ok := mysql.DefaultLengthOfMysqlTypes[col.FieldType.Tp]
if !ok {
return length, errUnknownTypeLength.GenWithStackByArgs(col.FieldType.Tp)
switch col.Tp {
case mysql.TypeBit:
return (length + 7) >> 3, nil
case mysql.TypeVarchar, mysql.TypeString:
// Different charsets occupy different numbers of bytes on each character.
desc, err := charset.GetCharsetDesc(col.Charset)
if err != nil {
return 0, errUnsupportedCharset.GenWithStackByArgs(col.Charset, col.Collate)
}
return desc.Maxlen * length, nil
case mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob:
return length, nil
case mysql.TypeTiny, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeShort:
return mysql.DefaultLengthOfMysqlTypes[col.Tp], nil
case mysql.TypeFloat:
if length <= mysql.MaxFloatPrecisionLength {
return mysql.DefaultLengthOfMysqlTypes[mysql.TypeFloat], nil
}
return mysql.DefaultLengthOfMysqlTypes[mysql.TypeDouble], nil
case mysql.TypeDecimal, mysql.TypeNewDecimal:
return calcBytesLengthForDecimal(length), nil
case mysql.TypeYear, mysql.TypeDate, mysql.TypeDuration, mysql.TypeDatetime, mysql.TypeTimestamp:
return mysql.DefaultLengthOfMysqlTypes[col.Tp], nil
default:
return length, nil
}
}

// Special case for time fraction.
if types.IsTypeFractionable(col.FieldType.Tp) &&
col.FieldType.Decimal != types.UnspecifiedLength {
decimalLength, ok := mysql.DefaultLengthOfTimeFraction[col.FieldType.Decimal]
if !ok {
return length, errUnknownFractionLength.GenWithStackByArgs(col.FieldType.Tp, col.FieldType.Decimal)
}
length += decimalLength
}
return length, nil
// Decimal using a binary format that packs nine decimal (base 10) digits into four bytes.
func calcBytesLengthForDecimal(m int) int {
return (m / 9 * 4) + ((m%9)+1)/2
}

func buildIndexInfo(tblInfo *model.TableInfo, indexName model.CIStr, idxColNames []*ast.IndexColName, state model.SchemaState) (*model.IndexInfo, error) {
Expand Down
136 changes: 73 additions & 63 deletions session/session_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1206,78 +1206,88 @@ func (s *testSessionSuite) TestFieldText(c *C) {

func (s *testSessionSuite) TestIndexMaxLength(c *C) {
tk := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("drop table if exists t;")
tk.MustExec("create database test_index_max_length")
tk.MustExec("use test_index_max_length")

// create simple index at table creation
_, err := tk.Exec("create table t (c1 varchar(3073), index(c1));")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)
tk.MustGetErrCode("create table t (c1 varchar(3073), index(c1)) charset = ascii;", mysql.ErrTooLongKey)

// create simple index after table creation
tk.MustExec("create table t (c1 varchar(3073));")
_, err = tk.Exec("create index idx_c1 on t(c1) ")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)
tk.MustExec("create table t (c1 varchar(3073)) charset = ascii;")
tk.MustGetErrCode("create index idx_c1 on t(c1) ", mysql.ErrTooLongKey)
tk.MustExec("drop table t;")

// create compound index at table creation
tk.MustExec("drop table if exists t;")
_, err = tk.Exec("create table t (c1 varchar(3072), c2 varchar(1), index(c1, c2));")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

_, err = tk.Exec("create table t (c1 varchar(3072), c2 char(1), index(c1, c2));")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

_, err = tk.Exec("create table t (c1 varchar(3072), c2 char, index(c1, c2));")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

_, err = tk.Exec("create table t (c1 varchar(3072), c2 date, index(c1, c2));")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

_, err = tk.Exec("create table t (c1 varchar(3068), c2 timestamp(1), index(c1, c2));")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

tk.MustExec("create table t (c1 varchar(3068), c2 bit(26), index(c1, c2));") // 26 bit = 4 bytes
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (c1 varchar(3068), c2 bit(32), index(c1, c2));") // 32 bit = 4 bytes
tk.MustExec("drop table if exists t;")
_, err = tk.Exec("create table t (c1 varchar(3068), c2 bit(33), index(c1, c2));")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)
tk.MustGetErrCode("create table t (c1 varchar(3072), c2 varchar(1), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey)
tk.MustGetErrCode("create table t (c1 varchar(3072), c2 char(1), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey)
tk.MustGetErrCode("create table t (c1 varchar(3072), c2 char, index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey)
tk.MustGetErrCode("create table t (c1 varchar(3072), c2 date, index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey)
tk.MustGetErrCode("create table t (c1 varchar(3069), c2 timestamp(1), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey)

tk.MustExec("create table t (c1 varchar(3068), c2 bit(26), index(c1, c2)) charset = ascii;") // 26 bit = 4 bytes
tk.MustExec("drop table t;")
tk.MustExec("create table t (c1 varchar(3068), c2 bit(32), index(c1, c2)) charset = ascii;") // 32 bit = 4 bytes
tk.MustExec("drop table t;")
tk.MustGetErrCode("create table t (c1 varchar(3068), c2 bit(33), index(c1, c2)) charset = ascii;", mysql.ErrTooLongKey)

// create compound index after table creation
tk.MustExec("create table t (c1 varchar(3072), c2 varchar(1));")
_, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (c1 varchar(3072), c2 char(1));")
_, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (c1 varchar(3072), c2 char);")
_, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)

tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (c1 varchar(3072), c2 date);")
_, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)
tk.MustExec("create table t (c1 varchar(3072), c2 varchar(1)) charset = ascii;")
tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey)
tk.MustExec("drop table t;")

tk.MustExec("create table t (c1 varchar(3072), c2 char(1)) charset = ascii;")
tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey)
tk.MustExec("drop table t;")

tk.MustExec("create table t (c1 varchar(3072), c2 char) charset = ascii;")
tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey)
tk.MustExec("drop table t;")

tk.MustExec("create table t (c1 varchar(3072), c2 date) charset = ascii;")
tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey)
tk.MustExec("drop table t;")

tk.MustExec("create table t (c1 varchar(3069), c2 timestamp(1)) charset = ascii;")
tk.MustGetErrCode("create index idx_c1_c2 on t(c1, c2);", mysql.ErrTooLongKey)
tk.MustExec("drop table t;")

// Test charsets other than `ascii`.
assertCharsetLimit := func(charset string, bytesPerChar int) {
base := 3072 / bytesPerChar
tk.MustGetErrCode(fmt.Sprintf("create table t (a varchar(%d) primary key) charset=%s", base+1, charset), mysql.ErrTooLongKey)
tk.MustExec(fmt.Sprintf("create table t (a varchar(%d) primary key) charset=%s", base, charset))
tk.MustExec("drop table if exists t")
}
assertCharsetLimit("binary", 1)
assertCharsetLimit("latin1", 1)
assertCharsetLimit("utf8", 3)
assertCharsetLimit("utf8mb4", 4)

// Test types bit length limit.
assertTypeLimit := func(tp string, limitBitLength int) {
base := 3072 - limitBitLength
tk.MustGetErrCode(fmt.Sprintf("create table t (a blob(10000), b %s, index idx(a(%d), b))", tp, base+1), mysql.ErrTooLongKey)
tk.MustExec(fmt.Sprintf("create table t (a blob(10000), b %s, index idx(a(%d), b))", tp, base))
tk.MustExec("drop table if exists t")
}

tk.MustExec("drop table if exists t;")
tk.MustExec("create table t (c1 varchar(3068), c2 timestamp(1));")
_, err = tk.Exec("create index idx_c1_c2 on t(c1, c2);")
// ERROR 1071 (42000): Specified key was too long; max key length is 3072 bytes
c.Assert(err, NotNil)
assertTypeLimit("tinyint", 1)
assertTypeLimit("smallint", 2)
assertTypeLimit("mediumint", 3)
assertTypeLimit("int", 4)
assertTypeLimit("integer", 4)
assertTypeLimit("bigint", 8)
assertTypeLimit("float", 4)
assertTypeLimit("float(24)", 4)
assertTypeLimit("float(25)", 8)
assertTypeLimit("decimal(9)", 4)
assertTypeLimit("decimal(10)", 5)
assertTypeLimit("decimal(17)", 8)
assertTypeLimit("year", 1)
assertTypeLimit("date", 3)
assertTypeLimit("time", 3)
assertTypeLimit("datetime", 8)
assertTypeLimit("timestamp", 4)
}

func (s *testSessionSuite) TestIndexColumnLength(c *C) {
Expand Down

0 comments on commit d312f8a

Please sign in to comment.