From 0a0d676bb2799f9d6f617008c6c29f9213875806 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 30 Sep 2018 11:42:06 +0800 Subject: [PATCH 1/3] admin: fix admin check table compare bug (#7818) --- executor/admin_test.go | 8 ++++++++ util/admin/admin.go | 22 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/executor/admin_test.go b/executor/admin_test.go index c3f078257285f..0a1282c582388 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -498,6 +498,14 @@ func (s *testSuite) TestAdminCheckTable(c *C) { tk.MustExec(`ALTER TABLE t1 ADD INDEX idx5 (c5)`) tk.MustExec(`ALTER TABLE t1 ADD INDEX idx6 (c6)`) tk.MustExec(`admin check table t1`) + + // Test add index on decimal column. + tk.MustExec(`drop table if exists td1;`) + tk.MustExec(`CREATE TABLE td1 (c2 INT NULL DEFAULT '70');`) + tk.MustExec(`INSERT INTO td1 SET c2 = '5';`) + tk.MustExec(`ALTER TABLE td1 ADD COLUMN c4 DECIMAL(12,8) NULL DEFAULT '213.41598062';`) + tk.MustExec(`ALTER TABLE td1 ADD INDEX id2 (c4) ;`) + tk.MustExec(`ADMIN CHECK TABLE td1;`) } func (s *testSuite) TestAdminCheckPrimaryIndex(c *C) { diff --git a/util/admin/admin.go b/util/admin/admin.go index d71434e7bcf3b..3e767f73021d3 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -353,6 +353,7 @@ func checkIndexAndRecord(sessCtx sessionctx.Context, txn kv.Transaction, t table if err != nil { return errors.Trace(err) } + sc := new(stmtctx.StatementContext) for { vals1, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { @@ -375,7 +376,7 @@ func checkIndexAndRecord(sessCtx sessionctx.Context, txn kv.Transaction, t table return errors.Trace(err) } adjustDatumKind(vals1, vals2) - if !reflect.DeepEqual(vals1, vals2) { + if !compareDatumSlice(sc, vals1, vals2) { record1 := &RecordData{Handle: h, Values: vals1} record2 := &RecordData{Handle: h, Values: vals2} return ErrDataInConsistent.GenWithStack("index:%#v != record:%#v", record1, record2) @@ -385,6 +386,25 @@ func checkIndexAndRecord(sessCtx sessionctx.Context, txn kv.Transaction, t table return nil } +func compareDatumSlice(sc *stmtctx.StatementContext, val1s, val2s []types.Datum) bool { + if len(val1s) != len(val2s) { + return false + } + for i, v := range val1s { + if v.Kind() == types.KindMysqlDecimal { + res, err := v.CompareDatum(sc, &val2s[i]) + if err != nil || res != 0 { + return false + } + } else { + if !reflect.DeepEqual(v, val2s[i]) { + return false + } + } + } + return true +} + // CheckRecordAndIndex is exported for testing. func CheckRecordAndIndex(sessCtx sessionctx.Context, txn kv.Transaction, t table.Table, idx table.Index, genExprs map[string]expression.Expression) error { sc := sessCtx.GetSessionVars().StmtCtx From 670d0b3c4c5ef936ac5ca8577922338d33bb7d44 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 16 Oct 2018 18:24:58 +0800 Subject: [PATCH 2/3] admin: fix admin check table bug of byte compare (#7887) * admin: remove reflect deepEqual --- executor/executor_test.go | 35 +++++++++++++++++++++++++++++++++++ types/datum_test.go | 38 ++++++++++++++++++++++++++++++++++++++ util/admin/admin.go | 18 ++++++------------ 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 1ae0306ad0911..988aa248a89de 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -274,6 +274,41 @@ func (s *testSuite) TestAdmin(c *C) { tk.MustExec("ALTER TABLE t1 ADD INDEX idx3 (c4);") tk.MustExec("admin check table t1;") + // For add index on virtual column + tk.MustExec("drop table if exists t1;") + tk.MustExec(`create table t1 ( + a int as (JSON_EXTRACT(k,'$.a')), + c double as (JSON_EXTRACT(k,'$.c')), + d decimal(20,10) as (JSON_EXTRACT(k,'$.d')), + e char(10) as (JSON_EXTRACT(k,'$.e')), + f date as (JSON_EXTRACT(k,'$.f')), + g time as (JSON_EXTRACT(k,'$.g')), + h datetime as (JSON_EXTRACT(k,'$.h')), + i timestamp as (JSON_EXTRACT(k,'$.i')), + j year as (JSON_EXTRACT(k,'$.j')), + k json);`) + + tk.MustExec("insert into t1 set k='{\"a\": 100,\"c\":1.234,\"d\":1.2340000000,\"e\":\"abcdefg\",\"f\":\"2018-09-28\",\"g\":\"12:59:59\",\"h\":\"2018-09-28 12:59:59\",\"i\":\"2018-09-28 16:40:33\",\"j\":\"2018\"}';") + tk.MustExec("alter table t1 add index idx_a(a);") + tk.MustExec("alter table t1 add index idx_c(c);") + tk.MustExec("alter table t1 add index idx_d(d);") + tk.MustExec("alter table t1 add index idx_e(e);") + tk.MustExec("alter table t1 add index idx_f(f);") + tk.MustExec("alter table t1 add index idx_g(g);") + tk.MustExec("alter table t1 add index idx_h(h);") + tk.MustExec("alter table t1 add index idx_j(j);") + tk.MustExec("alter table t1 add index idx_i(i);") + tk.MustExec("alter table t1 add index idx_m(a,c,d,e,f,g,h,i,j);") + tk.MustExec("admin check table t1;") + + tk.MustExec("drop table if exists t1;") + tk.MustExec("CREATE TABLE t1 (c1 int);") + tk.MustExec("INSERT INTO t1 SET c1 = 1;") + tk.MustExec("ALTER TABLE t1 ADD COLUMN cc1 CHAR(36) NULL DEFAULT '';") + tk.MustExec("ALTER TABLE t1 ADD COLUMN cc2 VARCHAR(36) NULL DEFAULT ''") + tk.MustExec("ALTER TABLE t1 ADD INDEX idx1 (cc1);") + tk.MustExec("ALTER TABLE t1 ADD INDEX idx2 (cc2);") + tk.MustExec("admin check table t1;") } func (s *testSuite) fillData(tk *testkit.TestKit, table string) { diff --git a/types/datum_test.go b/types/datum_test.go index b6a3d0bfb49fb..fc925ee7ecc5b 100644 --- a/types/datum_test.go +++ b/types/datum_test.go @@ -14,6 +14,8 @@ package types import ( + "reflect" + "testing" "time" . "github.com/pingcap/check" @@ -362,3 +364,39 @@ func (ts *testDatumSuite) TestCopyDatum(c *C) { } } } + +func prepareCompareDatums() ([]Datum, []Datum) { + vals := make([]Datum, 0, 5) + vals = append(vals, NewIntDatum(1)) + vals = append(vals, NewFloat64Datum(1.23)) + vals = append(vals, NewStringDatum("abcde")) + vals = append(vals, NewDecimalDatum(NewDecFromStringForTest("1.2345"))) + vals = append(vals, NewTimeDatum(Time{Time: FromGoTime(time.Date(2018, 3, 8, 16, 1, 0, 315313000, time.UTC)), Fsp: 6, Type: mysql.TypeTimestamp})) + + vals1 := make([]Datum, 0, 5) + vals1 = append(vals1, NewIntDatum(1)) + vals1 = append(vals1, NewFloat64Datum(1.23)) + vals1 = append(vals1, NewStringDatum("abcde")) + vals1 = append(vals1, NewDecimalDatum(NewDecFromStringForTest("1.2345"))) + vals1 = append(vals1, NewTimeDatum(Time{Time: FromGoTime(time.Date(2018, 3, 8, 16, 1, 0, 315313000, time.UTC)), Fsp: 6, Type: mysql.TypeTimestamp})) + return vals, vals1 +} + +func BenchmarkCompareDatum(b *testing.B) { + vals, vals1 := prepareCompareDatums() + sc := new(stmtctx.StatementContext) + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j, v := range vals { + v.CompareDatum(sc, &vals1[j]) + } + } +} + +func BenchmarkCompareDatumByReflect(b *testing.B) { + vals, vals1 := prepareCompareDatums() + b.ResetTimer() + for i := 0; i < b.N; i++ { + reflect.DeepEqual(vals, vals1) + } +} diff --git a/util/admin/admin.go b/util/admin/admin.go index 3e767f73021d3..325d09df98fc4 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -16,7 +16,6 @@ package admin import ( "fmt" "io" - "reflect" "sort" "github.com/pingcap/tidb/expression" @@ -353,7 +352,7 @@ func checkIndexAndRecord(sessCtx sessionctx.Context, txn kv.Transaction, t table if err != nil { return errors.Trace(err) } - sc := new(stmtctx.StatementContext) + sc := sessCtx.GetSessionVars().StmtCtx for { vals1, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { @@ -391,15 +390,9 @@ func compareDatumSlice(sc *stmtctx.StatementContext, val1s, val2s []types.Datum) return false } for i, v := range val1s { - if v.Kind() == types.KindMysqlDecimal { - res, err := v.CompareDatum(sc, &val2s[i]) - if err != nil || res != 0 { - return false - } - } else { - if !reflect.DeepEqual(v, val2s[i]) { - return false - } + res, err := v.CompareDatum(sc, &val2s[i]) + if err != nil || res != 0 { + return false } } return true @@ -523,6 +516,7 @@ func CompareTableRecord(sessCtx sessionctx.Context, txn kv.Transaction, t table. } startKey := t.RecordKey(0) + sc := sessCtx.GetSessionVars().StmtCtx filterFunc := func(h int64, vals []types.Datum, cols []*table.Column) (bool, error) { vals2, ok := m[h] if !ok { @@ -534,7 +528,7 @@ func CompareTableRecord(sessCtx sessionctx.Context, txn kv.Transaction, t table. return true, nil } - if !reflect.DeepEqual(vals, vals2) { + if !compareDatumSlice(sc, vals, vals2) { record1 := &RecordData{Handle: h, Values: vals2} record2 := &RecordData{Handle: h, Values: vals} return false, ErrDataInConsistent.GenWithStack("data:%#v != record:%#v", record1, record2) From c7c23c5b497a072bf2dba0a48203f12a7868d65a Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 21 Oct 2018 23:01:20 +0800 Subject: [PATCH 3/3] remove test add index on virtual column --- executor/executor_test.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 988aa248a89de..ccc48b19cbebd 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -273,42 +273,6 @@ func (s *testSuite) TestAdmin(c *C) { tk.MustExec("ALTER TABLE t1 ADD COLUMN c4 bit(10) default 127;") tk.MustExec("ALTER TABLE t1 ADD INDEX idx3 (c4);") tk.MustExec("admin check table t1;") - - // For add index on virtual column - tk.MustExec("drop table if exists t1;") - tk.MustExec(`create table t1 ( - a int as (JSON_EXTRACT(k,'$.a')), - c double as (JSON_EXTRACT(k,'$.c')), - d decimal(20,10) as (JSON_EXTRACT(k,'$.d')), - e char(10) as (JSON_EXTRACT(k,'$.e')), - f date as (JSON_EXTRACT(k,'$.f')), - g time as (JSON_EXTRACT(k,'$.g')), - h datetime as (JSON_EXTRACT(k,'$.h')), - i timestamp as (JSON_EXTRACT(k,'$.i')), - j year as (JSON_EXTRACT(k,'$.j')), - k json);`) - - tk.MustExec("insert into t1 set k='{\"a\": 100,\"c\":1.234,\"d\":1.2340000000,\"e\":\"abcdefg\",\"f\":\"2018-09-28\",\"g\":\"12:59:59\",\"h\":\"2018-09-28 12:59:59\",\"i\":\"2018-09-28 16:40:33\",\"j\":\"2018\"}';") - tk.MustExec("alter table t1 add index idx_a(a);") - tk.MustExec("alter table t1 add index idx_c(c);") - tk.MustExec("alter table t1 add index idx_d(d);") - tk.MustExec("alter table t1 add index idx_e(e);") - tk.MustExec("alter table t1 add index idx_f(f);") - tk.MustExec("alter table t1 add index idx_g(g);") - tk.MustExec("alter table t1 add index idx_h(h);") - tk.MustExec("alter table t1 add index idx_j(j);") - tk.MustExec("alter table t1 add index idx_i(i);") - tk.MustExec("alter table t1 add index idx_m(a,c,d,e,f,g,h,i,j);") - tk.MustExec("admin check table t1;") - - tk.MustExec("drop table if exists t1;") - tk.MustExec("CREATE TABLE t1 (c1 int);") - tk.MustExec("INSERT INTO t1 SET c1 = 1;") - tk.MustExec("ALTER TABLE t1 ADD COLUMN cc1 CHAR(36) NULL DEFAULT '';") - tk.MustExec("ALTER TABLE t1 ADD COLUMN cc2 VARCHAR(36) NULL DEFAULT ''") - tk.MustExec("ALTER TABLE t1 ADD INDEX idx1 (cc1);") - tk.MustExec("ALTER TABLE t1 ADD INDEX idx2 (cc2);") - tk.MustExec("admin check table t1;") } func (s *testSuite) fillData(tk *testkit.TestKit, table string) {