Fix potential deadlock in health streamer (#17261)

Signed-off-by: Manan Gupta <manan@planetscale.com>
Signed-off-by: Renan Rangel <rrangel@slack-corp.com>

go/vt/vttablet/tabletserver/health_streamer.go

@@ -293,8 +293,10 @@ func (hs *healthStreamer) SetUnhealthyThreshold(v time.Duration) {
 // so it can read and write to the MySQL instance for schema-tracking.
 func (hs *healthStreamer) MakePrimary(serving bool) {
 	hs.fieldsMu.Lock()
-	defer hs.fieldsMu.Unlock()
 	hs.isServingPrimary = serving
+	// We let go of the lock here because we don't want to hold the lock when calling RegisterNotifier.
+	// If we keep holding the lock, there is a potential deadlock that can happen.
+	hs.fieldsMu.Unlock()
 	// We register for notifications from the schema Engine only when schema tracking is enabled,
 	// and we are going to a serving primary state.
 	if serving && hs.signalWhenSchemaChange {

go/vt/vttablet/tabletserver/health_streamer_test.go

@@ -592,13 +592,14 @@ func TestDeadlockBwCloseAndReload(t *testing.T) {
 
 	wg := sync.WaitGroup{}
 	wg.Add(2)
-	// Try running Close and reload in parallel multiple times.
+	// Try running Close & MakePrimary and reload in parallel multiple times.
 	// This reproduces the deadlock quite readily.
 	go func() {
 		defer wg.Done()
 		for i := 0; i < 100; i++ {
 			hs.Close()
 			hs.Open()
+			hs.MakePrimary(true)
 		}
 	}()