diff --git a/drainer/translator/kafka.go b/drainer/translator/kafka.go index b14e429ff..3127b43bb 100644 --- a/drainer/translator/kafka.go +++ b/drainer/translator/kafka.go @@ -100,6 +100,42 @@ func genTable(schema string, tableInfo *model.TableInfo) (table *obinlog.Table) } table.ColumnInfo = columnInfos + // If PKIsHandle, tableInfo.Indices *will not* contains the primary key + // so we add it here. + // If !PKIsHandle tableInfo.Indices *will* contains the primary key + if tableInfo.PKIsHandle { + pkName := tableInfo.GetPkName() + key := &obinlog.Key{ + Name: proto.String("PRIMARY"), + ColumnNames: []string{pkName.O}, + } + table.UniqueKeys = append(table.UniqueKeys, key) + } + + for _, index := range tableInfo.Indices { + if !index.Unique && !index.Primary { + continue + } + + // just a protective check + if tableInfo.PKIsHandle && index.Name.O == "PRIMARY" { + log.Warn("PKIsHandle and also contains PRIMARY index TableInfo.Indices") + continue + } + + key := new(obinlog.Key) + table.UniqueKeys = append(table.UniqueKeys, key) + + key.Name = proto.String(index.Name.O) + + names := make([]string, len(index.Columns)) + for i, col := range index.Columns { + names[i] = col.Name.O + } + + key.ColumnNames = names + } + return } diff --git a/drainer/translator/kafka_test.go b/drainer/translator/kafka_test.go index 7c635c244..d7663e2bc 100644 --- a/drainer/translator/kafka_test.go +++ b/drainer/translator/kafka_test.go @@ -18,6 +18,8 @@ import ( "github.com/golang/protobuf/proto" "github.com/pingcap/check" + "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" obinlog "github.com/pingcap/tidb-tools/tidb-binlog/slave_binlog_proto/go-binlog" "github.com/pingcap/tidb/types" ) @@ -117,3 +119,107 @@ func checkColumn(c *check.C, info *obinlog.ColumnInfo, col *obinlog.Column, datu c.Assert(colV, check.Equals, datumV) } + +func (t *testKafkaSuite) TestGenTable(c *check.C) { + schema := "test" + table := "test" + + // a table test.test(c1, c2, c3) with: + // primary key: (c1) + // unique key: (c2, c3) + // non-unique key: (c3) + info := &model.TableInfo{ + Name: model.NewCIStr(table), + Columns: []*model.ColumnInfo{ + { + Name: model.NewCIStr("c1"), + FieldType: types.FieldType{ + Flag: mysql.PriKeyFlag, + Tp: mysql.TypeLong, + }, + }, + { + Name: model.NewCIStr("c2"), + FieldType: types.FieldType{ + Tp: mysql.TypeLong, + }, + }, + { + Name: model.NewCIStr("c3"), + FieldType: types.FieldType{ + Tp: mysql.TypeLong, + }, + }, + }, + Indices: []*model.IndexInfo{ + { + Name: model.NewCIStr("PRIMARY"), + Primary: true, + Unique: true, + Columns: []*model.IndexColumn{ + { + Offset: 0, + Name: model.NewCIStr("c1"), + }, + }, + }, + { + Name: model.NewCIStr("idx1"), + Unique: true, + Columns: []*model.IndexColumn{ + { + Offset: 1, + Name: model.NewCIStr("c2"), + }, + { + Offset: 2, + Name: model.NewCIStr("c3"), + }, + }, + }, + { + Name: model.NewCIStr("idx2"), + Unique: false, + Columns: []*model.IndexColumn{ + { + Offset: 1, + Name: model.NewCIStr("c3"), + }, + }, + }, + }, + } + + expectTable := &obinlog.Table{ + SchemaName: proto.String(schema), + TableName: proto.String(table), + ColumnInfo: []*obinlog.ColumnInfo{ + { + Name: "c1", + IsPrimaryKey: true, + MysqlType: "int", + }, + { + Name: "c2", + MysqlType: "int", + }, + { + Name: "c3", + MysqlType: "int", + }, + }, + UniqueKeys: []*obinlog.Key{ + { + Name: proto.String("PRIMARY"), + ColumnNames: []string{"c1"}, + }, + { + Name: proto.String("idx1"), + ColumnNames: []string{"c2", "c3"}, + }, + }, + } + + getTable := genTable(schema, info) + c.Assert(expectTable, check.DeepEquals, getTable) +} diff --git a/go.mod b/go.mod index 4eff1ca4b..039ae72a3 100644 --- a/go.mod +++ b/go.mod @@ -19,17 +19,17 @@ require ( github.com/pingcap/errors v0.11.4 github.com/pingcap/kvproto v0.0.0-20191118050206-47672e7eabc0 github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd - github.com/pingcap/parser v0.0.0-20191224043251-93f4d5ec2623 + github.com/pingcap/parser v0.0.0-20200103153514-95649ba8a872 github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 github.com/pingcap/tidb v1.1.0-beta.0.20191120070053-5a7ecfeb94fd - github.com/pingcap/tidb-tools v3.0.6-0.20191125061035-b087739b71f1+incompatible + github.com/pingcap/tidb-tools v3.1.0-beta.0.20191227034743-57985f125c52+incompatible github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a github.com/samuel/go-zookeeper v0.0.0-20170815201139-e6b59f6144be github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 - github.com/siddontang/go-mysql v0.0.0-20190618002340-dbe0224ac097 // indirect + github.com/sirupsen/logrus v1.4.1 // indirect github.com/soheilhy/cmux v0.1.4 github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect diff --git a/go.sum b/go.sum index 14efb01d8..41ffc5bee 100644 --- a/go.sum +++ b/go.sum @@ -208,20 +208,17 @@ github.com/pingcap/kvproto v0.0.0-20191118050206-47672e7eabc0 h1:CHOC95Ct4abJ9Ed github.com/pingcap/kvproto v0.0.0-20191118050206-47672e7eabc0/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/parser v0.0.0-20191118062434-7c5018645942 h1:JAPbnAxPryeAO50UO89/9MDYJK38Ts7mykTDqgUS14k= github.com/pingcap/parser v0.0.0-20191118062434-7c5018645942/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/parser v0.0.0-20191224043251-93f4d5ec2623 h1:/BJjVyJlNKWMMrgPsbzk5Y9VPJWwHKYttj3oWxnFQ9U= -github.com/pingcap/parser v0.0.0-20191224043251-93f4d5ec2623/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20200103153514-95649ba8a872 h1:PCY0XeNh/+6bGEHI6yWqwRXVDxvhpw2T6eE/umI5iXw= +github.com/pingcap/parser v0.0.0-20200103153514-95649ba8a872/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v1.1.0-beta.0.20190912093418-dc03c839debd/go.mod h1:I7TEby5BHTYIxgHszfsOJSBsk8b2Qt8QrSIgdv5n5QQ= github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bcJxpuSrEH4H7/nlf5YdmpS+dU9lNIt8= github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI= github.com/pingcap/tidb v1.1.0-beta.0.20191120070053-5a7ecfeb94fd h1:A7ZhUliNg9NA2+S9pMu7vC1R2ZmRZkpZocX50Mk/YjQ= github.com/pingcap/tidb v1.1.0-beta.0.20191120070053-5a7ecfeb94fd/go.mod h1:+SjmTI6LG3FvK4lIVDsOSyW2lWA4qtCSLKgwnN6FKxI= github.com/pingcap/tidb-tools v3.0.6-0.20191119150227-ff0a3c6e5763+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tidb-tools v3.0.6-0.20191125061035-b087739b71f1+incompatible h1:nndliEgwxjmupxW5ZOXHe4b7xQPXCaCm61Bi0dlQPpM= -github.com/pingcap/tidb-tools v3.0.6-0.20191125061035-b087739b71f1+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tidb-tools v3.1.0-beta.0.20191227034743-57985f125c52+incompatible h1:MfCru+JGzAww5YbKCCeobjCaCj5/jT01RTxiMEMqILw= +github.com/pingcap/tidb-tools v3.1.0-beta.0.20191227034743-57985f125c52+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6 h1:HPgqtaqIFIZXTvQNiZoJ9Y79AXz3pmDpYFL28KraTKE= github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -254,20 +251,13 @@ github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qq github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/samuel/go-zookeeper v0.0.0-20170815201139-e6b59f6144be h1:9prAjluC9cN0nht+csYXr3isly4OvAEfch9+VVFnn4U= github.com/samuel/go-zookeeper v0.0.0-20170815201139-e6b59f6144be/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs= github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= -github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= -github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= -github.com/siddontang/go-mysql v0.0.0-20190618002340-dbe0224ac097 h1:7krIUFu42TgBwT1wLQwpgR0Jcd/AkjgCGKSjhVWedaI= -github.com/siddontang/go-mysql v0.0.0-20190618002340-dbe0224ac097/go.mod h1:Bl4lryU44qtIXEXNbP0k0pD646Nkw/qHn21wfZVGJx4= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= @@ -375,15 +365,13 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c h1:S/FtSvpNLtFBgjTqcKsRpsa6aVsI6iztaz1bQd9BJwE= golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=