From 157153264652ab01d9b538d3ec768cd8f2d66a34 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:43:54 +0400
Subject: [PATCH 1/2] perf(consensus): Run broadcast routines out of process
(backport #3180) (#3477)
Run broadcast routines out of process. Right now each broadcast routine
blocks the consensus mutex for roughly `num_peers *
process_creation_time`, which is genuinely notable! This PR reduces the
consensus blocking overhead to just be `process_creation_time`.
On the latest osmosis branch with improvements, thats 20s of blocking
time out of 140s (over the course of 1 hour. This 140s includes block
execution!)
![image](https://github.com/cometbft/cometbft/assets/6440154/4c202988-a0d1-460e-89bc-7c1be11fd36f)
Note that WAL write time should go significantly down with open PR's.
For `HasVote`, this is a meaningful increase to consensus mutex lock
time, so its worth reducing.
---
#### PR checklist
- [x] Tests written/updated - I can't think of any test to add
- [x] Changelog entry added in `.changelog` (we use
[unclog](https://github.com/informalsystems/unclog) to manage our
changelog)
- [x] Updated relevant documentation (`docs/` or `spec/`) and code
comments - I don't know of any related docs here
- [x] Title follows the [Conventional
Commits](https://www.conventionalcommits.org/en/v1.0.0/) spec
This is an automatic backport of pull request #3180 done by
[Mergify](https://mergify.com).
---------
Co-authored-by: Dev Ojha
Co-authored-by: Anton Kaliaev
---
consensus/reactor.go | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/consensus/reactor.go b/consensus/reactor.go
index 3304159556..5ac15e8cd9 100644
--- a/consensus/reactor.go
+++ b/consensus/reactor.go
@@ -441,10 +441,12 @@ func (conR *Reactor) unsubscribeFromBroadcastEvents() {
func (conR *Reactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) {
nrsMsg := makeRoundStepMessage(rs)
- conR.Switch.Broadcast(p2p.Envelope{
- ChannelID: StateChannel,
- Message: nrsMsg,
- })
+ go func() {
+ conR.Switch.Broadcast(p2p.Envelope{
+ ChannelID: StateChannel,
+ Message: nrsMsg,
+ })
+ }()
}
func (conR *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) {
@@ -456,10 +458,12 @@ func (conR *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) {
BlockParts: rs.ProposalBlockParts.BitArray().ToProto(),
IsCommit: rs.Step == cstypes.RoundStepCommit,
}
- conR.Switch.Broadcast(p2p.Envelope{
- ChannelID: StateChannel,
- Message: csMsg,
- })
+ go func() {
+ conR.Switch.Broadcast(p2p.Envelope{
+ ChannelID: StateChannel,
+ Message: csMsg,
+ })
+ }()
}
// Broadcasts HasVoteMessage to peers that care.
From c98101deaed623d4b1ca3a73a2c8cb2ecc748e77 Mon Sep 17 00:00:00 2001
From: Dev Ojha
Date: Mon, 19 Aug 2024 14:11:31 -0400
Subject: [PATCH 2/2] add back has vote message broadcast
---
consensus/reactor.go | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/consensus/reactor.go b/consensus/reactor.go
index 5ac15e8cd9..4be53d0272 100644
--- a/consensus/reactor.go
+++ b/consensus/reactor.go
@@ -474,10 +474,12 @@ func (conR *Reactor) broadcastHasVoteMessage(vote *types.Vote) {
Type: vote.Type,
Index: vote.ValidatorIndex,
}
- conR.Switch.Broadcast(p2p.Envelope{
- ChannelID: StateChannel,
- Message: msg,
- })
+ go func() {
+ conR.Switch.Broadcast(p2p.Envelope{
+ ChannelID: StateChannel,
+ Message: msg,
+ })
+ }()
/*
// TODO: Make this broadcast more selective.
for _, peer := range conR.Switch.Peers().List() {