Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cdc,regionspan(ticdc): more logs about eventfeed retry rate limit #4008

Merged
merged 5 commits into from
Dec 26, 2021
Merged

cdc,regionspan(ticdc): more logs about eventfeed retry rate limit #4008

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7a15235
cdc,regionspan(ticdc): more logs about eventfeed retry rate limit
overvenus Dec 21, 2021
35105c6
address comments
overvenus Dec 22, 2021
254404a
log span
overvenus Dec 22, 2021
2bf51c6
address comments
overvenus Dec 22, 2021
30786cd
Merge branch 'master' into kv-log-rate-limited
ti-chi-bot Dec 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions cdc/kv/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,8 @@ func (s *eventFeedSession) eventFeed(ctx context.Context, ts uint64) error {
}
})

tableID, tableName := util.TableIDFromCtx(ctx)
cfID := util.ChangefeedIDFromCtx(ctx)
g.Go(func() error {
checkRateLimitInterval := 50 * time.Millisecond
timer := time.NewTimer(checkRateLimitInterval)
Expand All @@ -520,7 +522,13 @@ func (s *eventFeedSession) eventFeed(ctx context.Context, ts uint64) error {
s.errChSizeGauge.Dec()
allowed := s.checkRateLimit(errInfo.singleRegionInfo.verID.GetID())
if !allowed {
// rate limit triggers, add the error info to the rate limit queue
// rate limit triggers, add the error info to the rate limit queue.
log.Info("EventFeed retry rate limited",
zap.Uint64("regionID", errInfo.singleRegionInfo.verID.GetID()),
zap.Uint64("ts", errInfo.singleRegionInfo.ts),
zap.String("changefeed", cfID), zap.Stringer("span", errInfo.span),
zap.Int64("tableID", tableID), zap.String("tableName", tableName),
zap.String("addr", errInfo.singleRegionInfo.rpcCtx.Addr))
s.rateLimitQueue = append(s.rateLimitQueue, errInfo)
} else {
err := s.handleError(ctx, errInfo)
Expand Down Expand Up @@ -967,15 +975,11 @@ func (s *eventFeedSession) handleRateLimit(ctx context.Context) {
}

// checkRateLimit checks whether a region can be reconnected based on its rate limiter
func (s *eventFeedSession) checkRateLimit(regionID uint64) (allowed bool) {
func (s *eventFeedSession) checkRateLimit(regionID uint64) bool {
limiter := s.client.getRegionLimiter(regionID)
// use Limiter.Allow here since if exceed the rate limit, we skip this region
// and try it later.
allowed = limiter.Allow()
if !allowed {
log.Info("EventFeed retry rate limited", zap.Uint64("regionID", regionID))
}
return
return limiter.Allow()
}

// handleError handles error returned by a region. If some new EventFeed connection should be established, the region
Expand Down