From 9d3d45f8e1175b99cc880f761c96bf67d30c1a97 Mon Sep 17 00:00:00 2001 From: luancheng Date: Tue, 22 Dec 2020 19:24:26 +0800 Subject: [PATCH 1/6] restore: fix error lost in create schema --- lightning/restore/tidb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index d2c0fbb4c..27d3259ba 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -154,6 +154,7 @@ func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema task := logger.Begin(zap.InfoLevel, "create tables") var sqlCreateStmts []string +loopCreate: for tbl, sqlCreateTable := range tablesSchema { task.Debug("create table", zap.String("schema", sqlCreateTable)) @@ -171,7 +172,7 @@ func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema logger.With(zap.String("table", common.UniqueTable(database, tbl))), ) if err != nil { - break + break loopCreate } } } From 4cf8d4f47d45d0e5d491fbce46efabd28b124704 Mon Sep 17 00:00:00 2001 From: luancheng Date: Tue, 22 Dec 2020 19:48:07 +0800 Subject: [PATCH 2/6] add test --- lightning/restore/tidb.go | 4 ++++ lightning/restore/tidb_test.go | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index 27d3259ba..a301fba56 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -22,6 +22,7 @@ import ( tmysql "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/format" @@ -171,6 +172,9 @@ loopCreate: "create table", logger.With(zap.String("table", common.UniqueTable(database, tbl))), ) + failpoint.Inject("sqlCreateStmts", func() { + err = errors.Errorf("create %s failed", tbl) + }) if err != nil { break loopCreate } diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index 4b32834bc..e977a4d72 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -22,6 +22,7 @@ import ( "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" tmysql "github.com/pingcap/parser/mysql" @@ -224,6 +225,24 @@ func (s *tidbSuite) TestInitSchemaSyntaxError(c *C) { c.Assert(err, NotNil) } +func (s *tidbSuite) TestInitSchemaErrorLost(c *C) { + ctx := context.Background() + + s.mockDB. + ExpectExec("CREATE DATABASE IF NOT EXISTS `db`"). + WillReturnResult(sqlmock.NewResult(1, 1)) + s.mockDB. + ExpectClose() + + failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts", "return") + err := InitSchema(ctx, s.tiGlue, "db", map[string]string{ + "t1": "create table `t1` (a int);", + "t2": "create table t2 (a int primary key, b varchar(200));", + }) + failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts") + c.Assert(err, ErrorMatches, "create t1 failed") +} + func (s *tidbSuite) TestInitSchemaUnsupportedSchemaError(c *C) { ctx := context.Background() From aa8de148c0d73c7e91cfaec09926c30f80a5c2e1 Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 23 Dec 2020 12:02:51 +0800 Subject: [PATCH 3/6] fix test --- lightning/restore/tidb.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index a301fba56..cc83fbbdd 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -17,6 +17,7 @@ import ( "context" "database/sql" "fmt" + "sort" "strconv" "strings" @@ -155,8 +156,15 @@ func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema task := logger.Begin(zap.InfoLevel, "create tables") var sqlCreateStmts []string + tables := make([]string, 0, len(tablesSchema)) + for tbl := range tablesSchema { + tables = append(tables, tbl) + } + // sort tables for failpoint test + sort.Strings(tables) loopCreate: - for tbl, sqlCreateTable := range tablesSchema { + for i, tbl := range tables { + sqlCreateTable := tablesSchema[tbl] task.Debug("create table", zap.String("schema", sqlCreateTable)) sqlCreateStmts, err = createTableIfNotExistsStmt(g.GetParser(), sqlCreateTable, database, tbl) @@ -173,7 +181,11 @@ loopCreate: logger.With(zap.String("table", common.UniqueTable(database, tbl))), ) failpoint.Inject("sqlCreateStmts", func() { - err = errors.Errorf("create %s failed", tbl) + if i == 0 { + err = errors.Errorf("create %s failed", tbl) + } else { + err = nil + } }) if err != nil { break loopCreate From e75ceed0cc21c41766decbb01959ad3481001a85 Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 23 Dec 2020 15:28:57 +0800 Subject: [PATCH 4/6] address comment --- lightning/restore/tidb.go | 18 ++++-------------- lightning/restore/tidb_test.go | 4 ++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index cc83fbbdd..2d8b92468 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -17,7 +17,6 @@ import ( "context" "database/sql" "fmt" - "sort" "strconv" "strings" @@ -30,6 +29,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb-lightning/lightning/glue" . "github.com/pingcap/tidb-lightning/lightning/checkpoints" @@ -156,15 +156,8 @@ func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema task := logger.Begin(zap.InfoLevel, "create tables") var sqlCreateStmts []string - tables := make([]string, 0, len(tablesSchema)) - for tbl := range tablesSchema { - tables = append(tables, tbl) - } - // sort tables for failpoint test - sort.Strings(tables) loopCreate: - for i, tbl := range tables { - sqlCreateTable := tablesSchema[tbl] + for tbl, sqlCreateTable := range tablesSchema { task.Debug("create table", zap.String("schema", sqlCreateTable)) sqlCreateStmts, err = createTableIfNotExistsStmt(g.GetParser(), sqlCreateTable, database, tbl) @@ -181,11 +174,8 @@ loopCreate: logger.With(zap.String("table", common.UniqueTable(database, tbl))), ) failpoint.Inject("sqlCreateStmts", func() { - if i == 0 { - err = errors.Errorf("create %s failed", tbl) - } else { - err = nil - } + err = errors.Errorf("create %s failed", tbl) + failpoint.Return() }) if err != nil { break loopCreate diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index e977a4d72..d22a28419 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -234,13 +234,13 @@ func (s *tidbSuite) TestInitSchemaErrorLost(c *C) { s.mockDB. ExpectClose() - failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts", "return") + failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts", "1*return") err := InitSchema(ctx, s.tiGlue, "db", map[string]string{ "t1": "create table `t1` (a int);", "t2": "create table t2 (a int primary key, b varchar(200));", }) failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts") - c.Assert(err, ErrorMatches, "create t1 failed") + c.Assert(err, ErrorMatches, "create failed") } func (s *tidbSuite) TestInitSchemaUnsupportedSchemaError(c *C) { From 5bea64255fcd9dc7fce550999aec629f1373961d Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 23 Dec 2020 15:37:57 +0800 Subject: [PATCH 5/6] address comment --- lightning/restore/tidb.go | 1 - lightning/restore/tidb_test.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index 2d8b92468..e67e1ba86 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -175,7 +175,6 @@ loopCreate: ) failpoint.Inject("sqlCreateStmts", func() { err = errors.Errorf("create %s failed", tbl) - failpoint.Return() }) if err != nil { break loopCreate diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index d22a28419..efd4cc0af 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -240,7 +240,8 @@ func (s *tidbSuite) TestInitSchemaErrorLost(c *C) { "t2": "create table t2 (a int primary key, b varchar(200));", }) failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts") - c.Assert(err, ErrorMatches, "create failed") + // cannot predict which table will failed since map is unorder. + c.Assert(err, ErrorMatches, "create (.*) failed") } func (s *tidbSuite) TestInitSchemaUnsupportedSchemaError(c *C) { From 9a62c408b501f834454ec10898cf71739cf5b49b Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 23 Dec 2020 17:03:29 +0800 Subject: [PATCH 6/6] address comment --- lightning/restore/tidb.go | 4 ---- lightning/restore/tidb_test.go | 17 +++++++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index e67e1ba86..d60b88c8f 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -22,7 +22,6 @@ import ( tmysql "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/format" @@ -173,9 +172,6 @@ loopCreate: "create table", logger.With(zap.String("table", common.UniqueTable(database, tbl))), ) - failpoint.Inject("sqlCreateStmts", func() { - err = errors.Errorf("create %s failed", tbl) - }) if err != nil { break loopCreate } diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index efd4cc0af..664e47067 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -22,14 +22,14 @@ import ( "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" tmysql "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb-lightning/lightning/glue" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/util/mock" + "github.com/pingcap/tidb-lightning/lightning/glue" + "github.com/pingcap/tidb-lightning/lightning/checkpoints" "github.com/pingcap/tidb-lightning/lightning/mydump" ) @@ -231,17 +231,22 @@ func (s *tidbSuite) TestInitSchemaErrorLost(c *C) { s.mockDB. ExpectExec("CREATE DATABASE IF NOT EXISTS `db`"). WillReturnResult(sqlmock.NewResult(1, 1)) + + s.mockDB. + ExpectExec("CREATE TABLE IF NOT EXISTS.*"). + WillReturnError(&mysql.MySQLError{ + Number: tmysql.ErrTooBigFieldlength, + Message: "Column length too big", + }) + s.mockDB. ExpectClose() - failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts", "1*return") err := InitSchema(ctx, s.tiGlue, "db", map[string]string{ "t1": "create table `t1` (a int);", "t2": "create table t2 (a int primary key, b varchar(200));", }) - failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/restore/sqlCreateStmts") - // cannot predict which table will failed since map is unorder. - c.Assert(err, ErrorMatches, "create (.*) failed") + c.Assert(err, ErrorMatches, ".*Column length too big.*") } func (s *tidbSuite) TestInitSchemaUnsupportedSchemaError(c *C) {