diff --git a/br/pkg/backup/push.go b/br/pkg/backup/push.go index 6adc2daaba920..4713509024a25 100644 --- a/br/pkg/backup/push.go +++ b/br/pkg/backup/push.go @@ -119,7 +119,6 @@ func (push *pushDown) pushBackup( close(push.respCh) }() - regionErrorIngestedOnce := false for { select { case respAndStore, ok := <-push.respCh: @@ -151,19 +150,16 @@ func (push *pushDown) pushBackup( } }) failpoint.Inject("tikv-region-error", func(val failpoint.Value) { - if !regionErrorIngestedOnce { - msg := val.(string) - logutil.CL(ctx).Debug("failpoint tikv-regionh-error injected.", zap.String("msg", msg)) - resp.Error = &backuppb.Error{ - // Msg: msg, - Detail: &backuppb.Error_RegionError{ - RegionError: &errorpb.Error{ - Message: msg, - }, + msg := val.(string) + logutil.CL(ctx).Debug("failpoint tikv-region-error injected.", zap.String("msg", msg)) + resp.Error = &backuppb.Error{ + // Msg: msg, + Detail: &backuppb.Error_RegionError{ + RegionError: &errorpb.Error{ + Message: msg, }, - } + }, } - regionErrorIngestedOnce = true }) if resp.GetError() == nil { // None error means range has been backuped successfully. diff --git a/br/pkg/rtree/rtree.go b/br/pkg/rtree/rtree.go index f17ebf38df510..27f2684d5482f 100644 --- a/br/pkg/rtree/rtree.go +++ b/br/pkg/rtree/rtree.go @@ -185,17 +185,19 @@ func (rangeTree *RangeTree) GetIncompleteRange( return []Range{} } incomplete := make([]Range, 0, 64) - requsetRange := Range{StartKey: startKey, EndKey: endKey} + requestRange := Range{StartKey: startKey, EndKey: endKey} lastEndKey := startKey pviot := &Range{StartKey: startKey} if first := rangeTree.Find(pviot); first != nil { pviot.StartKey = first.StartKey } + pviotNotFound := true rangeTree.AscendGreaterOrEqual(pviot, func(i btree.Item) bool { + pviotNotFound = false rg := i.(*Range) if bytes.Compare(lastEndKey, rg.StartKey) < 0 { start, end, isIntersect := - requsetRange.Intersect(lastEndKey, rg.StartKey) + requestRange.Intersect(lastEndKey, rg.StartKey) if isIntersect { // There is a gap between the last item and the current item. incomplete = @@ -207,9 +209,10 @@ func (rangeTree *RangeTree) GetIncompleteRange( }) // Check whether we need append the last range - if !bytes.Equal(lastEndKey, endKey) && len(lastEndKey) != 0 && - (len(endKey) == 0 || bytes.Compare(lastEndKey, endKey) < 0) { - start, end, isIntersect := requsetRange.Intersect(lastEndKey, endKey) + if pviotNotFound || + (!bytes.Equal(lastEndKey, endKey) && len(lastEndKey) != 0 && + (len(endKey) == 0 || bytes.Compare(lastEndKey, endKey) < 0)) { + start, end, isIntersect := requestRange.Intersect(lastEndKey, endKey) if isIntersect { incomplete = append(incomplete, Range{StartKey: start, EndKey: end}) diff --git a/br/pkg/rtree/rtree_test.go b/br/pkg/rtree/rtree_test.go index a5fce76080f52..92146e595ffaf 100644 --- a/br/pkg/rtree/rtree_test.go +++ b/br/pkg/rtree/rtree_test.go @@ -47,6 +47,10 @@ func TestRangeTree(t *testing.T) { } } + assertIncomplete([]byte(""), []byte("b"), []rtree.Range{{StartKey: []byte(""), EndKey: []byte("b")}}) + assertIncomplete([]byte(""), []byte(""), []rtree.Range{{StartKey: []byte(""), EndKey: []byte("")}}) + assertIncomplete([]byte("b"), []byte(""), []rtree.Range{{StartKey: []byte("b"), EndKey: []byte("")}}) + range0 := newRange([]byte(""), []byte("a")) rangeA := newRange([]byte("a"), []byte("b")) rangeB := newRange([]byte("b"), []byte("c")) @@ -61,6 +65,7 @@ func TestRangeTree(t *testing.T) { {StartKey: []byte(""), EndKey: []byte("a")}, {StartKey: []byte("b"), EndKey: []byte("")}, }) + assertIncomplete([]byte("b"), []byte(""), []rtree.Range{{StartKey: []byte("b"), EndKey: []byte("")}}) rangeTree.Update(*rangeC) require.Equal(t, 2, rangeTree.Len()) diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh index b32cca0f8e41f..de7e2b702868b 100755 --- a/br/tests/br_rawkv/run.sh +++ b/br/tests/br_rawkv/run.sh @@ -52,8 +52,7 @@ test_full_rawkv() { checksum_full=$(checksum $check_range_start $check_range_end) # backup current state of key-values - # raw backup is not working with range [nil, nil]. TODO: fix it. - run_br --pd $PD_ADDR backup raw -s "local://$BACKUP_FULL" --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" --start $check_range_start --format hex + run_br --pd $PD_ADDR backup raw -s "local://$BACKUP_FULL" --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" clean $check_range_start $check_range_end # Ensure the data is deleted @@ -63,7 +62,7 @@ test_full_rawkv() { fail_and_exit fi - run_br --pd $PD_ADDR restore raw -s "local://$BACKUP_FULL" --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" --start $check_range_start --format hex + run_br --pd $PD_ADDR restore raw -s "local://$BACKUP_FULL" --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" checksum_new=$(checksum $check_range_start $check_range_end) if [ "$checksum_new" != "$checksum_full" ];then echo "failed to restore" @@ -185,5 +184,7 @@ run_test() { run_test "" -# ingest "region error" to trigger fineGrainedBackup +# ingest "region error" to trigger fineGrainedBackup, only one region error. +run_test "github.com/pingcap/tidb/br/pkg/backup/tikv-region-error=1*return(\"region error\")" +# all regions failed. run_test "github.com/pingcap/tidb/br/pkg/backup/tikv-region-error=return(\"region error\")"