Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixed ConsumerGroup flooding logs with client/metadata update req #1544 #1578

Merged
merged 3 commits into from
Jan 22, 2020
Merged

fixed ConsumerGroup flooding logs with client/metadata update req #1544 #1578

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bd430e6
fixed ConsumerGroup flooding logs with client/metadata update req #1544
antsbean Jan 16, 2020
fd421dc
solve call Consumer function agin, will generate more than loopCheckP…
antsbean Jan 16, 2020
0cce202
Merge remote-tracking branch 'shopify_sarama/master'
antsbean Jan 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions consumer_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler Co

// loop check topic partition numbers changed
// will trigger rebalance when any topic partitions number had changed
// avoid Consume function called again that will generate more than loopCheckPartitionNumbers coroutine
go c.loopCheckPartitionNumbers(topics, sess)

// Wait for session exit signal
Expand Down Expand Up @@ -448,7 +449,7 @@ func (c *consumerGroup) handleError(err error, topic string, partition int32) {
}

func (c *consumerGroup) loopCheckPartitionNumbers(topics []string, session *consumerGroupSession) {
pause := time.NewTicker(c.config.Consumer.Group.Heartbeat.Interval * 2)
pause := time.NewTicker(c.config.Metadata.RefreshFrequency)
defer session.cancel()
defer pause.Stop()
var oldTopicToPartitionNum map[string]int
Expand All @@ -468,17 +469,17 @@ func (c *consumerGroup) loopCheckPartitionNumbers(topics []string, session *cons
}
select {
case <-pause.C:
case <-session.ctx.Done():
Logger.Printf("loop check partition number coroutine will exit, topics %s", topics)
// if session closed by other, should be exited
return
case <-c.closed:
return
}
}
}

func (c *consumerGroup) topicToPartitionNumbers(topics []string) (map[string]int, error) {
if err := c.client.RefreshMetadata(topics...); err != nil {
Logger.Printf("Consumer Group refresh metadata failed %v", err)
return nil, err
}
topicToPartitionNum := make(map[string]int, len(topics))
for _, topic := range topics {
if partitionNum, err := c.client.Partitions(topic); err != nil {
Expand Down