From 8f544c9380a6e336a1d4bdbd046b9149e7d436e5 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Mon, 21 Aug 2023 19:50:18 +0800 Subject: [PATCH 1/8] br: fix wait tiflash replicas ready --- br/pkg/restore/client.go | 28 +++++++++++++++++++++++----- br/tests/br_tiflash/run.sh | 2 ++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go index 7ec9c868984dc..2e1d80370326e 100644 --- a/br/pkg/restore/client.go +++ b/br/pkg/restore/client.go @@ -1812,6 +1812,7 @@ func (rc *Client) GoWaitTiFlashReady(ctx context.Context, inCh <-chan *CreatedTa } } go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error { + tiFlashStoresMap := tiFlashStores if tbl.Table != nil && tbl.Table.TiFlashReplica == nil { log.Info("table has no tiflash replica", zap.Stringer("table", tbl.OldTable.Info.Name), @@ -1824,11 +1825,28 @@ func (rc *Client) GoWaitTiFlashReady(ctx context.Context, inCh <-chan *CreatedTa zap.Stringer("table", tbl.OldTable.Info.Name), zap.Stringer("db", tbl.OldTable.DB.Name)) for { - progress, err := infosync.CalculateTiFlashProgress(tbl.Table.ID, tbl.Table.TiFlashReplica.Count, tiFlashStores) - if err != nil { - log.Warn("failed to get tiflash replica progress, wait for next retry", zap.Error(err)) - time.Sleep(time.Second) - continue + var progress float64 + if pi := tbl.Table.GetPartitionInfo(); pi != nil && len(pi.Definitions) > 0 { + for _, p := range pi.Definitions { + progressOfPartition, err := infosync.MustGetTiFlashProgress(p.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStoresMap) + if err != nil { + log.Warn("failed to get progress for tiflash partition replica, retry it", + zap.Int64("tableID", tbl.Table.ID), zap.Int64("partitionID", p.ID), zap.Error(err)) + time.Sleep(time.Second) + continue + } + progress += progressOfPartition + } + progress = progress / float64(len(pi.Definitions)) + } else { + var err error + progress, err = infosync.MustGetTiFlashProgress(tbl.Table.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStoresMap) + if err != nil { + log.Warn("failed to get progress for tiflash replica, retry it", + zap.Int64("tableID", tbl.Table.ID), zap.Error(err)) + time.Sleep(time.Second) + continue + } } // check until progress is 1 if progress == 1 { diff --git a/br/tests/br_tiflash/run.sh b/br/tests/br_tiflash/run.sh index b416b05a74f58..81a35e20673a0 100644 --- a/br/tests/br_tiflash/run.sh +++ b/br/tests/br_tiflash/run.sh @@ -54,6 +54,8 @@ run_sql "DROP DATABASE $DB" run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --wait-tiflash-ready=true # check TiFlash sync +echo "wait 3 seconds for tiflash tick puller triggered" +sleep 3 if ! [ $(run_sql "select * from information_schema.tiflash_replica" | grep "PROGRESS" | sed "s/[^0-9]//g") -eq 1 ]; then echo "restore didn't wait tiflash synced after set --wait-tiflash-ready=true." exit 1 From 75412e2e29667aebae8fae354bf26298111939c7 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Tue, 22 Aug 2023 13:10:57 +0800 Subject: [PATCH 2/8] add partition test --- br/tests/br_tiflash/run.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/br/tests/br_tiflash/run.sh b/br/tests/br_tiflash/run.sh index 81a35e20673a0..78ff3fabb47a8 100644 --- a/br/tests/br_tiflash/run.sh +++ b/br/tests/br_tiflash/run.sh @@ -23,16 +23,30 @@ run_sql "CREATE DATABASE $DB" run_sql "CREATE TABLE $DB.kv(k varchar(256) primary key, v int)" +CREATE TABLE $DB.partition_kv( + id BIGINT, + data INT, + PRIMARY KEY(id) CLUSTERED +) PARTITION BY RANGE(id) ( + PARTITION p0 VALUES LESS THAN (200), + PARTITION p1 VALUES LESS THAN (400), + PARTITION p2 VALUES LESS THAN (600) +); + stmt="INSERT INTO $DB.kv(k, v) VALUES ('1-record', 1)" +parition_stmt="INSERT INTO $DB.partition_kv(k, v) VALUES (1, 1)" for i in $(seq 2 $RECORD_COUNT); do stmt="$stmt,('$i-record', $i)" + parition_stmt="$parition_stmt,($i, $i)" done run_sql "$stmt" +run_sql "$parition_stmt" if ! run_sql "ALTER TABLE $DB.kv SET TIFLASH REPLICA 1"; then # 10s should be enough for tiflash-proxy get started sleep 10 run_sql "ALTER TABLE $DB.kv SET TIFLASH REPLICA 1" + run_sql "ALTER TABLE $DB.partition_kv SET TIFLASH REPLICA 1" fi From 9cd667275bc1c437094823bebb1c41d8a0b49ef6 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Tue, 22 Aug 2023 15:31:29 +0800 Subject: [PATCH 3/8] update --- br/pkg/restore/client.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go index 2e1d80370326e..34e74c1039005 100644 --- a/br/pkg/restore/client.go +++ b/br/pkg/restore/client.go @@ -1812,7 +1812,6 @@ func (rc *Client) GoWaitTiFlashReady(ctx context.Context, inCh <-chan *CreatedTa } } go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error { - tiFlashStoresMap := tiFlashStores if tbl.Table != nil && tbl.Table.TiFlashReplica == nil { log.Info("table has no tiflash replica", zap.Stringer("table", tbl.OldTable.Info.Name), @@ -1828,7 +1827,7 @@ func (rc *Client) GoWaitTiFlashReady(ctx context.Context, inCh <-chan *CreatedTa var progress float64 if pi := tbl.Table.GetPartitionInfo(); pi != nil && len(pi.Definitions) > 0 { for _, p := range pi.Definitions { - progressOfPartition, err := infosync.MustGetTiFlashProgress(p.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStoresMap) + progressOfPartition, err := infosync.MustGetTiFlashProgress(p.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStores) if err != nil { log.Warn("failed to get progress for tiflash partition replica, retry it", zap.Int64("tableID", tbl.Table.ID), zap.Int64("partitionID", p.ID), zap.Error(err)) @@ -1840,7 +1839,7 @@ func (rc *Client) GoWaitTiFlashReady(ctx context.Context, inCh <-chan *CreatedTa progress = progress / float64(len(pi.Definitions)) } else { var err error - progress, err = infosync.MustGetTiFlashProgress(tbl.Table.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStoresMap) + progress, err = infosync.MustGetTiFlashProgress(tbl.Table.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStores) if err != nil { log.Warn("failed to get progress for tiflash replica, retry it", zap.Int64("tableID", tbl.Table.ID), zap.Error(err)) From a4e85dea98b5a003840ff6556c1827a608e6deab Mon Sep 17 00:00:00 2001 From: 3pointer Date: Tue, 22 Aug 2023 15:33:20 +0800 Subject: [PATCH 4/8] update --- br/tests/br_tiflash/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/br/tests/br_tiflash/run.sh b/br/tests/br_tiflash/run.sh index 78ff3fabb47a8..67b654b6c70e7 100644 --- a/br/tests/br_tiflash/run.sh +++ b/br/tests/br_tiflash/run.sh @@ -23,7 +23,7 @@ run_sql "CREATE DATABASE $DB" run_sql "CREATE TABLE $DB.kv(k varchar(256) primary key, v int)" -CREATE TABLE $DB.partition_kv( +run_sql "CREATE TABLE $DB.partition_kv( id BIGINT, data INT, PRIMARY KEY(id) CLUSTERED @@ -31,7 +31,7 @@ CREATE TABLE $DB.partition_kv( PARTITION p0 VALUES LESS THAN (200), PARTITION p1 VALUES LESS THAN (400), PARTITION p2 VALUES LESS THAN (600) -); +);" stmt="INSERT INTO $DB.kv(k, v) VALUES ('1-record', 1)" parition_stmt="INSERT INTO $DB.partition_kv(k, v) VALUES (1, 1)" From 36c4b9fdbf710576b4176965a5623158a970e55e Mon Sep 17 00:00:00 2001 From: 3pointer Date: Tue, 22 Aug 2023 16:59:11 +0800 Subject: [PATCH 5/8] fix test --- br/tests/br_tiflash/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/br/tests/br_tiflash/run.sh b/br/tests/br_tiflash/run.sh index 67b654b6c70e7..f89ea9c0bbc63 100644 --- a/br/tests/br_tiflash/run.sh +++ b/br/tests/br_tiflash/run.sh @@ -24,10 +24,10 @@ run_sql "CREATE DATABASE $DB" run_sql "CREATE TABLE $DB.kv(k varchar(256) primary key, v int)" run_sql "CREATE TABLE $DB.partition_kv( - id BIGINT, - data INT, - PRIMARY KEY(id) CLUSTERED -) PARTITION BY RANGE(id) ( + k BIGINT, + v INT, + PRIMARY KEY(k) CLUSTERED +) PARTITION BY RANGE(k) ( PARTITION p0 VALUES LESS THAN (200), PARTITION p1 VALUES LESS THAN (400), PARTITION p2 VALUES LESS THAN (600) From e6122365679c989c0ecaebe2f7bbb07526443137 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Tue, 22 Aug 2023 17:48:00 +0800 Subject: [PATCH 6/8] fix test --- br/tests/br_tiflash/run.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/br/tests/br_tiflash/run.sh b/br/tests/br_tiflash/run.sh index f89ea9c0bbc63..d0e145e2f234a 100644 --- a/br/tests/br_tiflash/run.sh +++ b/br/tests/br_tiflash/run.sh @@ -23,14 +23,14 @@ run_sql "CREATE DATABASE $DB" run_sql "CREATE TABLE $DB.kv(k varchar(256) primary key, v int)" -run_sql "CREATE TABLE $DB.partition_kv( - k BIGINT, - v INT, - PRIMARY KEY(k) CLUSTERED -) PARTITION BY RANGE(k) ( - PARTITION p0 VALUES LESS THAN (200), - PARTITION p1 VALUES LESS THAN (400), - PARTITION p2 VALUES LESS THAN (600) +run_sql "CREATE TABLE $DB.partition_kv( \ + k BIGINT, \ + v INT, \ + PRIMARY KEY(k) CLUSTERED \ +) PARTITION BY RANGE(k) ( \ + PARTITION p0 VALUES LESS THAN (200), \ + PARTITION p1 VALUES LESS THAN (400), \ + PARTITION p2 VALUES LESS THAN (600) \ );" stmt="INSERT INTO $DB.kv(k, v) VALUES ('1-record', 1)" From 34ac9f7755e6218abf0011eeaff04bdcff5ab7a5 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 23 Aug 2023 08:49:56 +0800 Subject: [PATCH 7/8] update --- br/tests/br_tiflash/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/br/tests/br_tiflash/run.sh b/br/tests/br_tiflash/run.sh index d0e145e2f234a..c8e5e274309c0 100644 --- a/br/tests/br_tiflash/run.sh +++ b/br/tests/br_tiflash/run.sh @@ -30,8 +30,7 @@ run_sql "CREATE TABLE $DB.partition_kv( \ ) PARTITION BY RANGE(k) ( \ PARTITION p0 VALUES LESS THAN (200), \ PARTITION p1 VALUES LESS THAN (400), \ - PARTITION p2 VALUES LESS THAN (600) \ -);" + PARTITION p2 VALUES LESS THAN (600))" stmt="INSERT INTO $DB.kv(k, v) VALUES ('1-record', 1)" parition_stmt="INSERT INTO $DB.partition_kv(k, v) VALUES (1, 1)" From e51652f4db3971ec9ac7fe559618cf6822fb76de Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 23 Aug 2023 11:19:48 +0800 Subject: [PATCH 8/8] update --- br/tests/br_tiflash/run.sh | 16 ++++++++-------- br/tests/run.sh | 5 +++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/br/tests/br_tiflash/run.sh b/br/tests/br_tiflash/run.sh index c8e5e274309c0..6161fed2eeec8 100644 --- a/br/tests/br_tiflash/run.sh +++ b/br/tests/br_tiflash/run.sh @@ -23,14 +23,14 @@ run_sql "CREATE DATABASE $DB" run_sql "CREATE TABLE $DB.kv(k varchar(256) primary key, v int)" -run_sql "CREATE TABLE $DB.partition_kv( \ - k BIGINT, \ - v INT, \ - PRIMARY KEY(k) CLUSTERED \ -) PARTITION BY RANGE(k) ( \ - PARTITION p0 VALUES LESS THAN (200), \ - PARTITION p1 VALUES LESS THAN (400), \ - PARTITION p2 VALUES LESS THAN (600))" +run_sql "CREATE TABLE $DB.partition_kv(\ + k INT, \ + v INT, \ + PRIMARY KEY(k) CLUSTERED \ +) PARTITION BY RANGE(k) (\ + PARTITION p0 VALUES LESS THAN (200), \ + PARTITION p1 VALUES LESS THAN (400), \ + PARTITION p2 VALUES LESS THAN MAXVALUE)" stmt="INSERT INTO $DB.kv(k, v) VALUES ('1-record', 1)" parition_stmt="INSERT INTO $DB.partition_kv(k, v) VALUES (1, 1)" diff --git a/br/tests/run.sh b/br/tests/run.sh index c1022371ac291..f3ff0c5dcbd33 100755 --- a/br/tests/run.sh +++ b/br/tests/run.sh @@ -21,6 +21,11 @@ export TEST_DIR=/tmp/backup_restore_test export COV_DIR="/tmp/group_cover" source $CUR/_utils/run_services +# Create COV_DIR if not exists +if [ -d "$COV_DIR" ]; then + mkdir -p $COV_DIR +fi + # Reset TEST_DIR rm -rf $TEST_DIR && mkdir -p $TEST_DIR