From 0c4daf32e4c138b12d9687e03b8d49a4abe1e2bd Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Thu, 30 May 2024 16:27:40 +0800 Subject: [PATCH 1/8] meta: add ListTablesByFilter function to list table by filter Signed-off-by: dongmen <414110582@qq.com> --- pkg/meta/meta.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index 75d4bf1e1158f..c4e0972ddf7c6 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -1022,6 +1022,49 @@ func (m *Meta) ListTables(dbID int64) ([]*model.TableInfo, error) { return tables, nil } +// ListTablesByFilter return all tables that does not filtered by filter in a database. +func (m *Meta) ListTablesByFilter( + dbID int64, + filter func(info *model.TableNameInfo) bool) ([]*model.TableInfo, error) { + dbKey := m.dbKey(dbID) + if err := m.checkDBExists(dbKey); err != nil { + return nil, errors.Trace(err) + } + + res, err := m.txn.HGetAll(dbKey) + if err != nil { + return nil, errors.Trace(err) + } + + tables := make([]*model.TableInfo, 0, len(res)/2) + for _, r := range res { + // only handle table meta + tableKey := string(r.Field) + if !strings.HasPrefix(tableKey, mTablePrefix) { + continue + } + tbName := &model.TableNameInfo{} + err = json.Unmarshal(r.Value, tbName) + if err != nil { + return nil, errors.Trace(err) + } + if filter(tbName) { + continue + } + + tbInfo := &model.TableInfo{} + err = json.Unmarshal(r.Value, tbInfo) + if err != nil { + return nil, errors.Trace(err) + } + tbInfo.DBID = dbID + + tables = append(tables, tbInfo) + } + + return tables, nil +} + // ListSimpleTables shows all simple tables in database. func (m *Meta) ListSimpleTables(dbID int64) ([]*model.TableNameInfo, error) { dbKey := m.dbKey(dbID) From 04e158bdff5511086f30a46f17ce22c13da8428f Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Thu, 30 May 2024 16:37:40 +0800 Subject: [PATCH 2/8] meta: add unit test case Signed-off-by: dongmen <414110582@qq.com> --- pkg/meta/meta_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 31abb03fe7c66..394d430795eb0 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -293,6 +293,16 @@ func TestMeta(t *testing.T) { }) require.NoError(t, err) } + + // Test ListTableByFilter + filter := func(tableName *model.TableNameInfo) bool { + return tableName.Name != tblName.Name + } + filteredTables, err := m.ListTablesByFilter(1, filter) + require.NoError(t, err) + require.Len(t, filteredTables, 1) + require.Equal(t, tblName.Name, filteredTables[0].Name) + // Generate an auto id. n, err = m.GetAutoIDAccessors(1, 2).RowID().Inc(10) require.NoError(t, err) From 8930d0e147e97b557e832cfac7c50430e2ffc223 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Thu, 30 May 2024 16:54:00 +0800 Subject: [PATCH 3/8] meta: add comments Signed-off-by: dongmen <414110582@qq.com> --- pkg/meta/meta.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index c4e0972ddf7c6..57cab7847d827 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -1023,6 +1023,8 @@ func (m *Meta) ListTables(dbID int64) ([]*model.TableInfo, error) { } // ListTablesByFilter return all tables that does not filtered by filter in a database. +// Note(dongmen): This method is used by TiCDC to reduce the time of changefeed initialization. +// Ref: https://github.com/pingcap/tiflow/issues/11109 func (m *Meta) ListTablesByFilter( dbID int64, filter func(info *model.TableNameInfo) bool) ([]*model.TableInfo, error) { From 71186d7270749644c378ca33835109765333710e Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 31 May 2024 16:10:15 +0800 Subject: [PATCH 4/8] meta: add function GetDBMeta Signed-off-by: dongmen <414110582@qq.com> --- pkg/meta/meta.go | 58 +++++++------------------------------------ pkg/meta/meta_test.go | 9 ------- 2 files changed, 9 insertions(+), 58 deletions(-) diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index 57cab7847d827..380c55b8cffab 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -989,51 +989,24 @@ func (m *Meta) IterTables(dbID int64, fn func(info *model.TableInfo) error) erro return errors.Trace(err) } -// ListTables shows all tables in database. -func (m *Meta) ListTables(dbID int64) ([]*model.TableInfo, error) { +// GetDBMeta return all meta information of a database. +// Note(dongmen): This method is used by TiCDC to reduce the time of changefeed initialization. +// Ref: https://github.com/pingcap/tiflow/issues/11109 +func (m *Meta) GetDBMeta(dbID int64) ([]structure.HashPair, error) { dbKey := m.dbKey(dbID) if err := m.checkDBExists(dbKey); err != nil { return nil, errors.Trace(err) } - res, err := m.txn.HGetAll(dbKey) if err != nil { return nil, errors.Trace(err) } - - tables := make([]*model.TableInfo, 0, len(res)/2) - for _, r := range res { - // only handle table meta - tableKey := string(r.Field) - if !strings.HasPrefix(tableKey, mTablePrefix) { - continue - } - - tbInfo := &model.TableInfo{} - err = json.Unmarshal(r.Value, tbInfo) - if err != nil { - return nil, errors.Trace(err) - } - tbInfo.DBID = dbID - - tables = append(tables, tbInfo) - } - - return tables, nil + return res, nil } -// ListTablesByFilter return all tables that does not filtered by filter in a database. -// Note(dongmen): This method is used by TiCDC to reduce the time of changefeed initialization. -// Ref: https://github.com/pingcap/tiflow/issues/11109 -func (m *Meta) ListTablesByFilter( - dbID int64, - filter func(info *model.TableNameInfo) bool) ([]*model.TableInfo, error) { - dbKey := m.dbKey(dbID) - if err := m.checkDBExists(dbKey); err != nil { - return nil, errors.Trace(err) - } - - res, err := m.txn.HGetAll(dbKey) +// ListTables shows all tables in database. +func (m *Meta) ListTables(dbID int64) ([]*model.TableInfo, error) { + res, err := m.GetDBMeta(dbID) if err != nil { return nil, errors.Trace(err) } @@ -1045,14 +1018,6 @@ func (m *Meta) ListTablesByFilter( if !strings.HasPrefix(tableKey, mTablePrefix) { continue } - tbName := &model.TableNameInfo{} - err = json.Unmarshal(r.Value, tbName) - if err != nil { - return nil, errors.Trace(err) - } - if filter(tbName) { - continue - } tbInfo := &model.TableInfo{} err = json.Unmarshal(r.Value, tbInfo) @@ -1069,12 +1034,7 @@ func (m *Meta) ListTablesByFilter( // ListSimpleTables shows all simple tables in database. func (m *Meta) ListSimpleTables(dbID int64) ([]*model.TableNameInfo, error) { - dbKey := m.dbKey(dbID) - if err := m.checkDBExists(dbKey); err != nil { - return nil, errors.Trace(err) - } - - res, err := m.txn.HGetAll(dbKey) + res, err := m.GetDBMeta(dbID) if err != nil { return nil, errors.Trace(err) } diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 394d430795eb0..e118b74b0f3e6 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -294,15 +294,6 @@ func TestMeta(t *testing.T) { require.NoError(t, err) } - // Test ListTableByFilter - filter := func(tableName *model.TableNameInfo) bool { - return tableName.Name != tblName.Name - } - filteredTables, err := m.ListTablesByFilter(1, filter) - require.NoError(t, err) - require.Len(t, filteredTables, 1) - require.Equal(t, tblName.Name, filteredTables[0].Name) - // Generate an auto id. n, err = m.GetAutoIDAccessors(1, 2).RowID().Inc(10) require.NoError(t, err) From 7f62efc08090f5cc7247f3e2a52e6101ac6f73e0 Mon Sep 17 00:00:00 2001 From: dongmen <20351731+asddongmen@users.noreply.github.com> Date: Fri, 31 May 2024 17:03:40 +0800 Subject: [PATCH 5/8] Update pkg/meta/meta_test.go Co-authored-by: D3Hunter --- pkg/meta/meta_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index e118b74b0f3e6..31abb03fe7c66 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -293,7 +293,6 @@ func TestMeta(t *testing.T) { }) require.NoError(t, err) } - // Generate an auto id. n, err = m.GetAutoIDAccessors(1, 2).RowID().Inc(10) require.NoError(t, err) From a570b9d6f5c47ab51ae34a3708933236ef169fb8 Mon Sep 17 00:00:00 2001 From: dongmen <20351731+asddongmen@users.noreply.github.com> Date: Fri, 31 May 2024 17:03:47 +0800 Subject: [PATCH 6/8] Update pkg/meta/meta.go Co-authored-by: D3Hunter --- pkg/meta/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index 380c55b8cffab..c56e0a8ce0d0f 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -992,7 +992,7 @@ func (m *Meta) IterTables(dbID int64, fn func(info *model.TableInfo) error) erro // GetDBMeta return all meta information of a database. // Note(dongmen): This method is used by TiCDC to reduce the time of changefeed initialization. // Ref: https://github.com/pingcap/tiflow/issues/11109 -func (m *Meta) GetDBMeta(dbID int64) ([]structure.HashPair, error) { +func (m *Meta) GetMetasByDBID(dbID int64) ([]structure.HashPair, error) { dbKey := m.dbKey(dbID) if err := m.checkDBExists(dbKey); err != nil { return nil, errors.Trace(err) From edacd24b5e5d1957f4f1568da4bdee3d7cd1982b Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 31 May 2024 17:17:31 +0800 Subject: [PATCH 7/8] meta: fix make error Signed-off-by: dongmen <414110582@qq.com> --- pkg/meta/meta.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index c56e0a8ce0d0f..3c210b267fce1 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -1006,7 +1006,7 @@ func (m *Meta) GetMetasByDBID(dbID int64) ([]structure.HashPair, error) { // ListTables shows all tables in database. func (m *Meta) ListTables(dbID int64) ([]*model.TableInfo, error) { - res, err := m.GetDBMeta(dbID) + res, err := m.GetMetasByDBID(dbID) if err != nil { return nil, errors.Trace(err) } @@ -1034,7 +1034,7 @@ func (m *Meta) ListTables(dbID int64) ([]*model.TableInfo, error) { // ListSimpleTables shows all simple tables in database. func (m *Meta) ListSimpleTables(dbID int64) ([]*model.TableNameInfo, error) { - res, err := m.GetDBMeta(dbID) + res, err := m.GetMetasByDBID(dbID) if err != nil { return nil, errors.Trace(err) } From da2bcc73924f8f7a6fa5953e77b85e22af90b67d Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 4 Jun 2024 10:41:34 +0800 Subject: [PATCH 8/8] meta: fix make check error Signed-off-by: dongmen <414110582@qq.com> --- pkg/meta/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index 3c210b267fce1..7138b7c9fdcbe 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -989,7 +989,7 @@ func (m *Meta) IterTables(dbID int64, fn func(info *model.TableInfo) error) erro return errors.Trace(err) } -// GetDBMeta return all meta information of a database. +// GetMetasByDBID return all meta information of a database. // Note(dongmen): This method is used by TiCDC to reduce the time of changefeed initialization. // Ref: https://github.com/pingcap/tiflow/issues/11109 func (m *Meta) GetMetasByDBID(dbID int64) ([]structure.HashPair, error) {