From 462ba0a93fbe08e704f6aca510d42c21a3444972 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Mon, 19 Aug 2024 20:20:49 +0200 Subject: [PATCH] =?UTF-8?q?perf(consensus):=20Run=20broadcast=20routines?= =?UTF-8?q?=20out=20of=20process=20(backport=20#318=E2=80=A6=20(#135)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(consensus): Run broadcast routines out of process (backport #3180) (#3477) Run broadcast routines out of process. Right now each broadcast routine blocks the consensus mutex for roughly `num_peers * process_creation_time`, which is genuinely notable! This PR reduces the consensus blocking overhead to just be `process_creation_time`. On the latest osmosis branch with improvements, thats 20s of blocking time out of 140s (over the course of 1 hour. This 140s includes block execution!) ![image](https://github.com/cometbft/cometbft/assets/6440154/4c202988-a0d1-460e-89bc-7c1be11fd36f) Note that WAL write time should go significantly down with open PR's. For `HasVote`, this is a meaningful increase to consensus mutex lock time, so its worth reducing. --- #### PR checklist - [x] Tests written/updated - I can't think of any test to add - [x] Changelog entry added in `.changelog` (we use [unclog](https://github.com/informalsystems/unclog) to manage our changelog) - [x] Updated relevant documentation (`docs/` or `spec/`) and code comments - I don't know of any related docs here - [x] Title follows the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) spec
This is an automatic backport of pull request #3180 done by [Mergify](https://mergify.com). --------- Co-authored-by: Dev Ojha Co-authored-by: Anton Kaliaev * add back has vote message broadcast --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Anton Kaliaev --- consensus/reactor.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 3304159556..4be53d0272 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -441,10 +441,12 @@ func (conR *Reactor) unsubscribeFromBroadcastEvents() { func (conR *Reactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) { nrsMsg := makeRoundStepMessage(rs) - conR.Switch.Broadcast(p2p.Envelope{ - ChannelID: StateChannel, - Message: nrsMsg, - }) + go func() { + conR.Switch.Broadcast(p2p.Envelope{ + ChannelID: StateChannel, + Message: nrsMsg, + }) + }() } func (conR *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) { @@ -456,10 +458,12 @@ func (conR *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) { BlockParts: rs.ProposalBlockParts.BitArray().ToProto(), IsCommit: rs.Step == cstypes.RoundStepCommit, } - conR.Switch.Broadcast(p2p.Envelope{ - ChannelID: StateChannel, - Message: csMsg, - }) + go func() { + conR.Switch.Broadcast(p2p.Envelope{ + ChannelID: StateChannel, + Message: csMsg, + }) + }() } // Broadcasts HasVoteMessage to peers that care. @@ -470,10 +474,12 @@ func (conR *Reactor) broadcastHasVoteMessage(vote *types.Vote) { Type: vote.Type, Index: vote.ValidatorIndex, } - conR.Switch.Broadcast(p2p.Envelope{ - ChannelID: StateChannel, - Message: msg, - }) + go func() { + conR.Switch.Broadcast(p2p.Envelope{ + ChannelID: StateChannel, + Message: msg, + }) + }() /* // TODO: Make this broadcast more selective. for _, peer := range conR.Switch.Peers().List() {