Merge pull request #6752 from dolthub/aaron/fix-cluster-setRole-deadl…

…ock-in-perms-replication

go/libraries/doltcore/sqle/cluster: Fix a possible deadlock in permissions replication.

go/libraries/doltcore/sqle/cluster/branch_control_replica.go

@@ -87,9 +87,20 @@ func (r *branchControlReplica) Run() {
 			r.wait()
 			continue
 		}
-		_, err := r.client.client.UpdateBranchControl(context.Background(), &replicationapi.UpdateBranchControlRequest{
-			SerializedContents: r.contents,
+		// We do not call into the client with the lock held here.
+		// Client interceptors could call
+		// `controller.setRoleAndEpoch()`, which will call back into
+		// this replica with the new role. We need to release this lock
+		// in order to avoid deadlock.
+		contents := r.contents
+		client := r.client.client
+		r.mu.Unlock()
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		_, err := client.UpdateBranchControl(ctx, &replicationapi.UpdateBranchControlRequest{
+			SerializedContents: contents,
 		})
+		cancel()
+		r.mu.Lock()
 		if err != nil {
 			r.lgr.Warnf("branchControlReplica[%s]: error replicating branch control permissions. backing off. %v", r.client.remote, err)
 			r.nextAttempt = time.Now().Add(r.backoff.NextBackOff())

go/libraries/doltcore/sqle/cluster/mysqldb_persister.go

@@ -96,9 +96,20 @@ func (r *mysqlDbReplica) Run() {
 			continue
 		}
 		if len(r.contents) > 0 {
-			_, err := r.client.client.UpdateUsersAndGrants(context.Background(), &replicationapi.UpdateUsersAndGrantsRequest{
-				SerializedContents: r.contents,
+			// We do not call into the client with the lock held
+			// here.  Client interceptors could call
+			// `controller.setRoleAndEpoch()`, which will call back
+			// into this replica with the new role. We need to
+			// release this lock in order to avoid deadlock.
+			contents := r.contents
+			client := r.client.client
+			r.mu.Unlock()
+			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+			_, err := client.UpdateUsersAndGrants(ctx, &replicationapi.UpdateUsersAndGrantsRequest{
+				SerializedContents: contents,
 			})
+			cancel()
+			r.mu.Lock()
 			if err != nil {
 				r.lgr.Warnf("mysqlDbReplica[%s]: error replicating users and grants. backing off. %v", r.client.remote, err)
 				r.nextAttempt = time.Now().Add(r.backoff.NextBackOff())