fix: a deadlock caused by bsc protocol handeshake timeout

eth/handler_bsc.go

@@ -27,7 +27,7 @@ func (h *bscHandler) RunPeer(peer *bsc.Peer, hand bsc.Handler) error {
 		ps.lock.Lock()
 		if wait, ok := ps.bscWait[id]; ok {
 			delete(ps.bscWait, id)
-			wait <- peer
+			wait <- nil
 		}
 		ps.lock.Unlock()
 		return err

eth/peerset.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/protocols/eth"
 	"github.com/ethereum/go-ethereum/eth/protocols/snap"
 	"github.com/ethereum/go-ethereum/eth/protocols/trust"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
 )
 
@@ -70,6 +71,7 @@ const (
 	// extensionWaitTimeout is the maximum allowed time for the extension wait to
 	// complete before dropping the connection as malicious.
 	extensionWaitTimeout = 10 * time.Second
+	tryWaitTimeout       = 100 * time.Millisecond
 )
 
 // peerSet represents the collection of active peers currently participating in
@@ -402,10 +404,27 @@ func (ps *peerSet) waitBscExtension(peer *eth.Peer) (*bsc.Peer, error) {
 		return peer, nil
 
 	case <-time.After(extensionWaitTimeout):
-		ps.lock.Lock()
-		delete(ps.bscWait, id)
-		ps.lock.Unlock()
-		return nil, errPeerWaitTimeout
+		// once timeout is reached, bsc protocol won't be enabled on this peer.
+		log.Warn("waitBscExtension", "peer", id, "bsc protocol timeout, it won't be enabled.")
+		// could be deadlock, so we use TryLock to avoid it.
+		if ps.lock.TryLock() {
+			delete(ps.bscWait, id)
+			ps.lock.Unlock()
+			return nil, errPeerWaitTimeout
+		}
+		// if TryLock failed, we wait for a while and try again.
+		for {
+			select {
+			case <-wait:
+				return nil, errPeerWaitTimeout
+			case <-time.After(tryWaitTimeout):
+				if ps.lock.TryLock() {
+					delete(ps.bscWait, id)
+					ps.lock.Unlock()
+					return nil, errPeerWaitTimeout
+				}
+			}
+		}
 	}
 }