From 7f1b54eec89f0af7279bbe61a0b06346421deefb Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Mon, 23 Aug 2021 22:34:04 -0500 Subject: [PATCH] prepare: filter databases before show table status (#324) --- v4/export/block_allow_list.go | 17 +++++++++++++++++ v4/export/block_allow_list_test.go | 5 +++++ v4/export/dump.go | 2 +- v4/export/prepare.go | 10 ++++++++-- v4/export/prepare_test.go | 11 ++++++----- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/v4/export/block_allow_list.go b/v4/export/block_allow_list.go index 381bea64..194be0e2 100644 --- a/v4/export/block_allow_list.go +++ b/v4/export/block_allow_list.go @@ -8,6 +8,23 @@ import ( tcontext "github.com/pingcap/dumpling/v4/context" ) +func filterDatabases(tctx *tcontext.Context, conf *Config, databases []string) []string { + tctx.L().Debug("start to filter databases") + newDatabases := make([]string, 0, len(databases)) + ignoreDatabases := make([]string, 0, len(databases)) + for _, database := range databases { + if conf.TableFilter.MatchSchema(database) { + newDatabases = append(newDatabases, database) + } else { + ignoreDatabases = append(ignoreDatabases, database) + } + } + if len(ignoreDatabases) > 0 { + tctx.L().Debug("ignore database", zap.Strings("databases", ignoreDatabases)) + } + return newDatabases +} + func filterTables(tctx *tcontext.Context, conf *Config) { filterTablesFunc(tctx, conf, conf.TableFilter.MatchTable) } diff --git a/v4/export/block_allow_list_test.go b/v4/export/block_allow_list_test.go index fddc7716..845bcfba 100644 --- a/v4/export/block_allow_list_test.go +++ b/v4/export/block_allow_list_test.go @@ -17,6 +17,7 @@ var _ = Suite(&testBWListSuite{}) type testBWListSuite struct{} func (s *testBWListSuite) TestFilterTables(c *C) { + tctx := tcontext.Background().WithLogger(appLogger) dbTables := DatabaseTables{} expectedDBTables := DatabaseTables{} @@ -36,7 +37,11 @@ func (s *testBWListSuite) TestFilterTables(c *C) { TableFilter: tableFilter, } + databases := []string{filter.InformationSchemaName, filter.PerformanceSchemaName, "xxx", "yyy"} + c.Assert(filterDatabases(tctx, conf, databases), DeepEquals, databases) + conf.TableFilter = tf.NewSchemasFilter("xxx") + c.Assert(filterDatabases(tctx, conf, databases), DeepEquals, []string{"xxx"}) filterTables(tcontext.Background(), conf) c.Assert(conf.Tables, HasLen, 1) c.Assert(conf.Tables, DeepEquals, expectedDBTables) diff --git a/v4/export/dump.go b/v4/export/dump.go index 51a96fb2..f1191d65 100755 --- a/v4/export/dump.go +++ b/v4/export/dump.go @@ -865,7 +865,7 @@ func getListTableTypeByConf(conf *Config) listTableType { } func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) error { - databases, err := prepareDumpingDatabases(conf, db) + databases, err := prepareDumpingDatabases(tctx, conf, db) if err != nil { return err } diff --git a/v4/export/prepare.go b/v4/export/prepare.go index e995f8aa..2a6d7a14 100644 --- a/v4/export/prepare.go +++ b/v4/export/prepare.go @@ -10,6 +10,8 @@ import ( "text/template" "github.com/pingcap/errors" + + tcontext "github.com/pingcap/dumpling/v4/context" ) const ( @@ -75,10 +77,14 @@ func ParseOutputFileTemplate(text string) (*template.Template, error) { return template.Must(DefaultOutputFileTemplate.Clone()).Parse(text) } -func prepareDumpingDatabases(conf *Config, db *sql.Conn) ([]string, error) { +func prepareDumpingDatabases(tctx *tcontext.Context, conf *Config, db *sql.Conn) ([]string, error) { databases, err := ShowDatabases(db) + if err != nil { + return nil, err + } + databases = filterDatabases(tctx, conf, databases) if len(conf.Databases) == 0 { - return databases, err + return databases, nil } dbMap := make(map[string]interface{}, len(databases)) for _, database := range databases { diff --git a/v4/export/prepare_test.go b/v4/export/prepare_test.go index 04b806d1..aeb218ba 100644 --- a/v4/export/prepare_test.go +++ b/v4/export/prepare_test.go @@ -21,7 +21,8 @@ func (s *testPrepareSuite) TestPrepareDumpingDatabases(c *C) { db, mock, err := sqlmock.New() c.Assert(err, IsNil) defer db.Close() - conn, err := db.Conn(context.Background()) + tctx := tcontext.Background().WithLogger(appLogger) + conn, err := db.Conn(tctx) c.Assert(err, IsNil) rows := sqlmock.NewRows([]string{"Database"}). @@ -32,7 +33,7 @@ func (s *testPrepareSuite) TestPrepareDumpingDatabases(c *C) { mock.ExpectQuery("SHOW DATABASES").WillReturnRows(rows) conf := defaultConfigForTest(c) conf.Databases = []string{"db1", "db2", "db3"} - result, err := prepareDumpingDatabases(conf, conn) + result, err := prepareDumpingDatabases(tctx, conf, conn) c.Assert(err, IsNil) c.Assert(result, DeepEquals, []string{"db1", "db2", "db3"}) @@ -41,12 +42,12 @@ func (s *testPrepareSuite) TestPrepareDumpingDatabases(c *C) { AddRow("db1"). AddRow("db2") mock.ExpectQuery("SHOW DATABASES").WillReturnRows(rows) - result, err = prepareDumpingDatabases(conf, conn) + result, err = prepareDumpingDatabases(tctx, conf, conn) c.Assert(err, IsNil) c.Assert(result, DeepEquals, []string{"db1", "db2"}) mock.ExpectQuery("SHOW DATABASES").WillReturnError(fmt.Errorf("err")) - _, err = prepareDumpingDatabases(conf, conn) + _, err = prepareDumpingDatabases(tctx, conf, conn) c.Assert(err, NotNil) rows = sqlmock.NewRows([]string{"Database"}). @@ -56,7 +57,7 @@ func (s *testPrepareSuite) TestPrepareDumpingDatabases(c *C) { AddRow("db5") mock.ExpectQuery("SHOW DATABASES").WillReturnRows(rows) conf.Databases = []string{"db1", "db2", "db4", "db6"} - _, err = prepareDumpingDatabases(conf, conn) + _, err = prepareDumpingDatabases(tctx, conf, conn) c.Assert(err, ErrorMatches, `Unknown databases \[db4,db6\]`) c.Assert(mock.ExpectationsWereMet(), IsNil) }