diff --git a/go.mod1 b/go.mod1 index 09d444277..cb1e5e923 100644 --- a/go.mod1 +++ b/go.mod1 @@ -16,7 +16,7 @@ require ( github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20201120081251-756b1447ba12 + github.com/pingcap/kvproto v0.0.0-20201124110645-494a2fb764b7 github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 github.com/pingcap/parser v0.0.0-20201123080035-8f4c6ab94e11 github.com/pingcap/tidb v1.1.0-beta.0.20201125030655-07ff41d0c43e diff --git a/go.sum1 b/go.sum1 index d3ad11d4b..2e777de6a 100644 --- a/go.sum1 +++ b/go.sum1 @@ -754,8 +754,9 @@ github.com/pingcap/kvproto v0.0.0-20200417092353-efbe03bcffbd/go.mod h1:IOdRDPLy github.com/pingcap/kvproto v0.0.0-20200420075417-e0c6e8842f22/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200810113304-6157337686b1/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20201113092725-08f2872278eb/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20201120081251-756b1447ba12 h1:f33y/pngBI525jqytoSZevpmmq43XiIHoeElx3BppNQ= github.com/pingcap/kvproto v0.0.0-20201120081251-756b1447ba12/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20201124110645-494a2fb764b7 h1:xi36VGntRELtpBBAUkvx2XrCxSLwrOVEnbLnGT/Kx5g= +github.com/pingcap/kvproto v0.0.0-20201124110645-494a2fb764b7/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 6dee5a5ec..776711001 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -687,14 +687,15 @@ func (bc *Client) fineGrainedBackup( } } -func onBackupResponse( +// OnBackupResponse checks the backup resp, decides whether to retry and generate the error. +func OnBackupResponse( storeID uint64, bo *tikv.Backoffer, backupTS uint64, lockResolver *tikv.LockResolver, resp *kvproto.BackupResponse, ) (*kvproto.BackupResponse, int, error) { - log.Debug("onBackupResponse", zap.Reflect("resp", resp)) + log.Debug("OnBackupResponse", zap.Reflect("resp", resp)) if resp.Error == nil { return resp, 0, nil } @@ -716,7 +717,7 @@ func onBackupResponse( } // Backup should not meet error other than KeyLocked. log.Error("unexpect kv error", zap.Reflect("KvError", v.KvError)) - return nil, backoffMs, errors.Annotatef(berrors.ErrKVUnknown, "storeID: %d onBackupResponse error %v", storeID, v) + return nil, backoffMs, errors.Annotatef(berrors.ErrKVUnknown, "storeID: %d OnBackupResponse error %v", storeID, v) case *kvproto.Error_RegionError: regionErr := v.RegionError @@ -726,9 +727,11 @@ func onBackupResponse( regionErr.RegionNotFound != nil || regionErr.ServerIsBusy != nil || regionErr.StaleCommand != nil || - regionErr.StoreNotMatch != nil) { + regionErr.StoreNotMatch != nil || + regionErr.ReadIndexNotReady != nil || + regionErr.ProposalInMergingMode != nil) { log.Error("unexpect region error", zap.Reflect("RegionError", regionErr)) - return nil, backoffMs, errors.Annotatef(berrors.ErrKVUnknown, "storeID: %d onBackupResponse error %v", storeID, v) + return nil, backoffMs, errors.Annotatef(berrors.ErrKVUnknown, "storeID: %d OnBackupResponse error %v", storeID, v) } log.Warn("backup occur region error", zap.Reflect("RegionError", regionErr), @@ -787,7 +790,7 @@ func (bc *Client) handleFineGrained( // Handle responses with the same backoffer. func(resp *kvproto.BackupResponse) error { response, backoffMs, err1 := - onBackupResponse(storeID, bo, backupTS, lockResolver, resp) + OnBackupResponse(storeID, bo, backupTS, lockResolver, resp) if err1 != nil { return err1 } diff --git a/pkg/backup/client_test.go b/pkg/backup/client_test.go index 8b6931c37..065de9e59 100644 --- a/pkg/backup/client_test.go +++ b/pkg/backup/client_test.go @@ -9,9 +9,12 @@ import ( "time" . "github.com/pingcap/check" + kvproto "github.com/pingcap/kvproto/pkg/backup" + "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/mockstore/mocktikv" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" @@ -140,3 +143,41 @@ func (r *testBackup) TestBuildTableRange(c *C) { {StartKey: tablecodec.EncodeRowKey(7, low), EndKey: tablecodec.EncodeRowKey(7, high)}, }) } + +func (r *testBackup) TestOnBackupRegionErrorResponse(c *C) { + type Case struct { + storeID uint64 + bo *tikv.Backoffer + backupTS uint64 + lockResolver *tikv.LockResolver + resp *kvproto.BackupResponse + exceptedBackoffMs int + exceptedErr bool + } + newBackupRegionErrorResp := func(regionError *errorpb.Error) *kvproto.BackupResponse { + return &kvproto.BackupResponse{Error: &kvproto.Error{Detail: &kvproto.Error_RegionError{RegionError: regionError}}} + } + + cases := []Case{ + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{NotLeader: &errorpb.NotLeader{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{KeyNotInRegion: &errorpb.KeyNotInRegion{}}), exceptedBackoffMs: 0, exceptedErr: true}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{StaleCommand: &errorpb.StaleCommand{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{StoreNotMatch: &errorpb.StoreNotMatch{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{RaftEntryTooLarge: &errorpb.RaftEntryTooLarge{}}), exceptedBackoffMs: 0, exceptedErr: true}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ReadIndexNotReady: &errorpb.ReadIndexNotReady{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ProposalInMergingMode: &errorpb.ProposalInMergingMode{}}), exceptedBackoffMs: 1000, exceptedErr: false}, + } + for _, cs := range cases { + c.Log(cs) + _, backoffMs, err := backup.OnBackupResponse(cs.storeID, cs.bo, cs.backupTS, cs.lockResolver, cs.resp) + c.Assert(backoffMs, Equals, cs.exceptedBackoffMs) + if cs.exceptedErr { + c.Assert(err, NotNil) + } else { + c.Assert(err, IsNil) + } + } +}