Skip to content

Commit

Permalink
[release-19.0] Fix potential deadlock in health streamer (#17261) (#1…
Browse files Browse the repository at this point in the history
…7268)

Signed-off-by: Manan Gupta <manan@planetscale.com>
Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>
  • Loading branch information
vitess-bot[bot] authored Nov 21, 2024
1 parent 185b31e commit 36a3f9e
Showing 2 changed files with 5 additions and 2 deletions.
4 changes: 3 additions & 1 deletion go/vt/vttablet/tabletserver/health_streamer.go
Original file line number Diff line number Diff line change
@@ -317,8 +317,10 @@ func (hs *healthStreamer) SetUnhealthyThreshold(v time.Duration) {
// so it can read and write to the MySQL instance for schema-tracking.
func (hs *healthStreamer) MakePrimary(serving bool) {
hs.fieldsMu.Lock()
defer hs.fieldsMu.Unlock()
hs.isServingPrimary = serving
// We let go of the lock here because we don't want to hold the lock when calling RegisterNotifier.
// If we keep holding the lock, there is a potential deadlock that can happen.
hs.fieldsMu.Unlock()
// We register for notifications from the schema Engine only when schema tracking is enabled,
// and we are going to a serving primary state.
if serving && hs.signalWhenSchemaChange {
3 changes: 2 additions & 1 deletion go/vt/vttablet/tabletserver/health_streamer_test.go
Original file line number Diff line number Diff line change
@@ -587,13 +587,14 @@ func TestDeadlockBwCloseAndReload(t *testing.T) {

wg := sync.WaitGroup{}
wg.Add(2)
// Try running Close and reload in parallel multiple times.
// Try running Close & MakePrimary and reload in parallel multiple times.
// This reproduces the deadlock quite readily.
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
hs.Close()
hs.Open()
hs.MakePrimary(true)
}
}()

0 comments on commit 36a3f9e

Please sign in to comment.