From 016d979a8b6ed5fffe3cb9e6a77c5759b341504b Mon Sep 17 00:00:00 2001
From: Gheis Mohammadi <Gheis.Mohammadi@gmail.com>
Date: Fri, 30 Aug 2024 16:17:07 +0800
Subject: [PATCH 01/24] add connection manager high water mark flag to boot
 node (#4747)

---
 cmd/bootnode/main.go | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go
index 65250e2fb5..b438b47971 100644
--- a/cmd/bootnode/main.go
+++ b/cmd/bootnode/main.go
@@ -108,6 +108,7 @@ func main() {
 	logConn := flag.Bool("log_conn", false, "log incoming/outgoing connections")
 	maxConnPerIP := flag.Int("max_conn_per_ip", 10, "max connections number for same ip")
 	forceReachabilityPublic := flag.Bool("force_public", false, "forcing the local node to believe it is reachable externally")
+	connMgrHighWaterMark := flag.Int("cmg_high_watermark", 900, "connection manager trims excess connections when they pass the high watermark")
 	noTransportSecurity := flag.Bool("no_transport_security", false, "disable TLS encrypted transport")
 	muxer := flag.String("muxer", "mplex, yamux", "protocol muxer to mux per-protocol streams (mplex, yamux)")
 	userAgent := flag.String("user_agent", defUserAgent, "explicitly set the user-agent, so we can differentiate from other Go libp2p users")
@@ -142,18 +143,19 @@ func main() {
 	selfPeer := p2p.Peer{IP: *ip, Port: *port}
 
 	host, err := p2p.NewHost(p2p.HostConfig{
-		Self:                    &selfPeer,
-		BLSKey:                  privKey,
-		BootNodes:               nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined
-		DataStoreFile:           &dataStorePath,
-		MaxConnPerIP:            *maxConnPerIP,
-		ForceReachabilityPublic: *forceReachabilityPublic,
-		NoTransportSecurity:     *noTransportSecurity,
-		NAT:                     true,
-		UserAgent:               *userAgent,
-		DialTimeout:             time.Minute,
-		Muxer:                   *muxer,
-		NoRelay:                 *noRelay,
+		Self:                     &selfPeer,
+		BLSKey:                   privKey,
+		BootNodes:                nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined
+		DataStoreFile:            &dataStorePath,
+		MaxConnPerIP:             *maxConnPerIP,
+		ForceReachabilityPublic:  *forceReachabilityPublic,
+		ConnManagerHighWatermark: *connMgrHighWaterMark,
+		NoTransportSecurity:      *noTransportSecurity,
+		NAT:                      true,
+		UserAgent:                *userAgent,
+		DialTimeout:              time.Minute,
+		Muxer:                    *muxer,
+		NoRelay:                  *noRelay,
 	})
 	if err != nil {
 		utils.FatalErrMsg(err, "cannot initialize network")

From 4718a1cf749491db85cb91c4e331ba33032e422f Mon Sep 17 00:00:00 2001
From: Frozen <355847+Frozen@users.noreply.github.com>
Date: Mon, 2 Sep 2024 20:01:57 -0400
Subject: [PATCH 02/24] Lock free consensus methods (#4739)

* Lock free consensus methods.
---
 cmd/harmony/main.go                     |   1 -
 consensus/checks.go                     |   4 +-
 consensus/consensus.go                  |  16 +-
 consensus/consensus_msg_sender.go       |  10 +-
 consensus/consensus_service.go          |  33 ++--
 consensus/consensus_test.go             |   2 +-
 consensus/consensus_v2.go               |  11 +-
 consensus/debug.go                      |  12 +-
 consensus/double_sign.go                |   2 +-
 consensus/enums.go                      |   2 +-
 consensus/post_processing.go            | 232 ++++++++++++++++++++++++
 consensus/quorum/quorum.go              |   7 +-
 consensus/quorum/thread_safe_decider.go |   2 -
 consensus/view_change.go                |  49 ++---
 consensus/view_change_msg.go            |   4 +-
 consensus/view_change_test.go           |   5 +-
 node/node_cross_shard.go                | 114 ------------
 node/node_handler.go                    |  94 ----------
 node/node_handler_test.go               |   2 +-
 19 files changed, 304 insertions(+), 298 deletions(-)
 create mode 100644 consensus/post_processing.go

diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go
index 34cfd792b7..9629bee0aa 100644
--- a/cmd/harmony/main.go
+++ b/cmd/harmony/main.go
@@ -887,7 +887,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi
 		Msg("Init Blockchain")
 
 	currentNode.Consensus.Registry().SetNodeConfig(currentNode.NodeConfig)
-	currentConsensus.PostConsensusJob = currentNode.PostConsensusProcessing
 	// update consensus information based on the blockchain
 	currentConsensus.SetMode(currentConsensus.UpdateConsensusInformation())
 	currentConsensus.NextBlockDue = time.Now()
diff --git a/consensus/checks.go b/consensus/checks.go
index b4e1d12075..739c19dd6c 100644
--- a/consensus/checks.go
+++ b/consensus/checks.go
@@ -85,7 +85,7 @@ func (consensus *Consensus) onAnnounceSanityChecks(recvMsg *FBFTMessage) bool {
 				Str("logMsgSenderKey", logMsgs[0].SenderPubkeys[0].Bytes.Hex()).
 				Str("logMsgBlockHash", logMsgs[0].BlockHash.Hex()).
 				Str("recvMsg", recvMsg.String()).
-				Str("LeaderKey", consensus.LeaderPubKey.Bytes.Hex()).
+				Str("LeaderKey", consensus.getLeaderPubKey().Bytes.Hex()).
 				Msg("[OnAnnounce] Leader is malicious")
 			if consensus.isViewChangingMode() {
 				consensus.getLogger().Debug().Msg(
@@ -96,7 +96,7 @@ func (consensus *Consensus) onAnnounceSanityChecks(recvMsg *FBFTMessage) bool {
 			}
 		}
 		consensus.getLogger().Debug().
-			Str("leaderKey", consensus.LeaderPubKey.Bytes.Hex()).
+			Str("leaderKey", consensus.getLeaderPubKey().Bytes.Hex()).
 			Msg("[OnAnnounce] Announce message received again")
 	}
 	return consensus.isRightBlockNumCheck(recvMsg)
diff --git a/consensus/consensus.go b/consensus/consensus.go
index a281a8ae7c..1a649e5ee9 100644
--- a/consensus/consensus.go
+++ b/consensus/consensus.go
@@ -5,6 +5,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"github.com/harmony-one/abool"
 	bls_core "github.com/harmony-one/bls/ffi/go/bls"
@@ -84,7 +85,7 @@ type Consensus struct {
 	// private/public keys of current node
 	priKey multibls.PrivateKeys
 	// the publickey of leader
-	LeaderPubKey *bls.PublicKeyWrapper
+	leaderPubKey unsafe.Pointer //*bls.PublicKeyWrapper
 	// blockNum: the next blockNumber that FBFT is going to agree on,
 	// should be equal to the blockNumber of next block
 	blockNum uint64
@@ -104,9 +105,6 @@ type Consensus struct {
 	readySignal chan Proposal
 	// Channel to send full commit signatures to finish new block proposal
 	commitSigChannel chan []byte
-	// The post-consensus job func passed from Node object
-	// Called when consensus on a new block is done
-	PostConsensusJob func(*types.Block) error
 	// verified block to state sync broadcast
 	VerifiedNewBlock chan *types.Block
 	// Channel for DRG protocol to send pRnd (preimage of randomness resulting from combined vrf
@@ -230,17 +228,15 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper {
 }
 
 func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper {
-	return consensus.LeaderPubKey
+	return (*bls_cosi.PublicKeyWrapper)(atomic.LoadPointer(&consensus.leaderPubKey))
 }
 
 func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) {
-	consensus.mutex.Lock()
-	defer consensus.mutex.Unlock()
 	consensus.setLeaderPubKey(pub)
 }
 
 func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) {
-	consensus.LeaderPubKey = pub
+	atomic.StorePointer(&consensus.leaderPubKey, unsafe.Pointer(pub))
 }
 
 func (consensus *Consensus) GetPrivateKeys() multibls.PrivateKeys {
@@ -259,7 +255,7 @@ func (consensus *Consensus) getLeaderPrivateKey(leaderKey *bls_core.PublicKey) (
 
 // getConsensusLeaderPrivateKey returns consensus leader private key if node is the leader
 func (consensus *Consensus) getConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapper, error) {
-	return consensus.getLeaderPrivateKey(consensus.LeaderPubKey.Object)
+	return consensus.getLeaderPrivateKey(consensus.getLeaderPubKey().Object)
 }
 
 func (consensus *Consensus) IsBackup() bool {
@@ -287,7 +283,7 @@ func New(
 		ShardID:      shard,
 		fBFTLog:      NewFBFTLog(),
 		phase:        FBFTAnnounce,
-		current:      State{mode: Normal},
+		current:      NewState(Normal),
 		decider:      Decider,
 		registry:     registry,
 		MinPeers:     minPeers,
diff --git a/consensus/consensus_msg_sender.go b/consensus/consensus_msg_sender.go
index ad32b91ce3..b18d4e97ab 100644
--- a/consensus/consensus_msg_sender.go
+++ b/consensus/consensus_msg_sender.go
@@ -67,10 +67,7 @@ func (sender *MessageSender) SendWithRetry(blockNum uint64, msgType msg_pb.Messa
 			sender.Retry(&msgRetry)
 		}()
 	}
-	// MessageSender lays inside consensus, but internally calls consensus public api.
-	// Tt would be deadlock if run in current thread.
-	go sender.host.SendMessageToGroups(groups, p2pMsg)
-	return nil
+	return sender.host.SendMessageToGroups(groups, p2pMsg)
 }
 
 // DelayedSendWithRetry is similar to SendWithRetry but without the initial message sending but only retries.
@@ -89,10 +86,7 @@ func (sender *MessageSender) DelayedSendWithRetry(blockNum uint64, msgType msg_p
 
 // SendWithoutRetry sends message without retry logic.
 func (sender *MessageSender) SendWithoutRetry(groups []nodeconfig.GroupID, p2pMsg []byte) error {
-	// MessageSender lays inside consensus, but internally calls consensus public api.
-	// It would be deadlock if run in current thread.
-	go sender.host.SendMessageToGroups(groups, p2pMsg)
-	return nil
+	return sender.host.SendMessageToGroups(groups, p2pMsg)
 }
 
 // Retry will retry the consensus message for <RetryTimes> times.
diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go
index e92e242765..e64f3007ab 100644
--- a/consensus/consensus_service.go
+++ b/consensus/consensus_service.go
@@ -92,9 +92,9 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi
 
 	allKeys := consensus.decider.Participants()
 	if len(allKeys) != 0 {
-		consensus.LeaderPubKey = &allKeys[0]
+		consensus.setLeaderPubKey(&allKeys[0])
 		consensus.getLogger().Info().
-			Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("Setting leader as first validator, because provided new keys")
+			Str("info", consensus.getLeaderPubKey().Bytes.Hex()).Msg("Setting leader as first validator, because provided new keys")
 	} else {
 		consensus.getLogger().Error().
 			Msg("[UpdatePublicKeys] Participants is empty")
@@ -172,8 +172,6 @@ func (consensus *Consensus) resetState() {
 
 // IsValidatorInCommittee returns whether the given validator BLS address is part of my committee
 func (consensus *Consensus) IsValidatorInCommittee(pubKey bls.SerializedPublicKey) bool {
-	consensus.mutex.RLock()
-	defer consensus.mutex.RUnlock()
 	return consensus.isValidatorInCommittee(pubKey)
 }
 
@@ -207,9 +205,8 @@ func (consensus *Consensus) SetIsBackup(isBackup bool) {
 }
 
 // Mode returns the mode of consensus
+// Method is thread safe.
 func (consensus *Consensus) Mode() Mode {
-	consensus.mutex.RLock()
-	defer consensus.mutex.RUnlock()
 	return consensus.mode()
 }
 
@@ -239,11 +236,11 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error {
 		if !msg.HasSingleSender() {
 			return errors.New("Leader message can not have multiple sender keys")
 		}
-		consensus.LeaderPubKey = msg.SenderPubkeys[0]
+		consensus.setLeaderPubKey(msg.SenderPubkeys[0])
 		consensus.IgnoreViewIDCheck.UnSet()
 		consensus.consensusTimeout[timeoutConsensus].Start()
 		consensus.getLogger().Info().
-			Str("leaderKey", consensus.LeaderPubKey.Bytes.Hex()).
+			Str("leaderKey", consensus.getLeaderPubKey().Bytes.Hex()).
 			Msg("[checkViewID] Start consensus timer")
 		return nil
 	} else if msg.ViewID > consensus.getCurBlockViewID() {
@@ -399,7 +396,7 @@ func (consensus *Consensus) updateConsensusInformation() Mode {
 	}
 
 	// update public keys in the committee
-	oldLeader := consensus.LeaderPubKey
+	oldLeader := consensus.getLeaderPubKey()
 	pubKeys, _ := committeeToSet.BLSPublicKeys()
 
 	consensus.getLogger().Info().
@@ -436,7 +433,7 @@ func (consensus *Consensus) updateConsensusInformation() Mode {
 			consensus.getLogger().Info().
 				Str("leaderPubKey", leaderPubKey.Bytes.Hex()).
 				Msgf("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain, blocknum: %d", curHeader.NumberU64())
-			consensus.LeaderPubKey = leaderPubKey
+			consensus.setLeaderPubKey(leaderPubKey)
 		}
 	}
 
@@ -453,8 +450,8 @@ func (consensus *Consensus) updateConsensusInformation() Mode {
 			}
 
 			// If the leader changed and I myself become the leader
-			if (oldLeader != nil && consensus.LeaderPubKey != nil &&
-				!consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.isLeader() {
+			if (oldLeader != nil && consensus.getLeaderPubKey() != nil &&
+				!consensus.getLeaderPubKey().Object.IsEqual(oldLeader.Object)) && consensus.isLeader() {
 				go func() {
 					consensus.GetLogger().Info().
 						Str("myKey", myPubKeys.SerializeToHexStr()).
@@ -474,20 +471,19 @@ func (consensus *Consensus) updateConsensusInformation() Mode {
 
 // IsLeader check if the node is a leader or not by comparing the public key of
 // the node with the leader public key
+// Method is thread safe
 func (consensus *Consensus) IsLeader() bool {
-	consensus.mutex.RLock()
-	defer consensus.mutex.RUnlock()
-
 	return consensus.isLeader()
 }
 
 // isLeader check if the node is a leader or not by comparing the public key of
 // the node with the leader public key. This function assume it runs under lock.
 func (consensus *Consensus) isLeader() bool {
-	if consensus.LeaderPubKey == nil {
+	pub := consensus.getLeaderPubKey()
+	if pub == nil {
 		return false
 	}
-	obj := consensus.LeaderPubKey.Object
+	obj := pub.Object
 	for _, key := range consensus.priKey {
 		if key.Pub.Object.IsEqual(obj) {
 			return true
@@ -624,12 +620,11 @@ func (consensus *Consensus) selfCommit(payload []byte) error {
 }
 
 // NumSignaturesIncludedInBlock returns the number of signatures included in the block
+// Method is thread safe.
 func (consensus *Consensus) NumSignaturesIncludedInBlock(block *types.Block) uint32 {
 	count := uint32(0)
-	consensus.mutex.Lock()
 	members := consensus.decider.Participants()
 	pubKeys := consensus.getPublicKeys()
-	consensus.mutex.Unlock()
 
 	// TODO(audit): do not reconstruct the Mask
 	mask := bls.NewMask(members)
diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go
index 2fe524fdf8..0f8b0c724b 100644
--- a/consensus/consensus_test.go
+++ b/consensus/consensus_test.go
@@ -22,7 +22,7 @@ func TestConsensusInitialization(t *testing.T) {
 	assert.NoError(t, err)
 
 	messageSender := &MessageSender{host: host, retryTimes: int(phaseDuration.Seconds()) / RetryIntervalInSec}
-	state := State{mode: Normal}
+	state := NewState(Normal)
 
 	timeouts := createTimeout()
 	expectedTimeouts := make(map[TimeoutType]time.Duration)
diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go
index 6caed6e504..dbc8e8bf3d 100644
--- a/consensus/consensus_v2.go
+++ b/consensus/consensus_v2.go
@@ -40,10 +40,9 @@ const (
 	CommitSigSenderTimeout = 10 * time.Second
 )
 
-// IsViewChangingMode return true if curernt mode is viewchanging
+// IsViewChangingMode return true if current mode is viewchanging.
+// Method is thread safe.
 func (consensus *Consensus) IsViewChangingMode() bool {
-	consensus.mutex.RLock()
-	defer consensus.mutex.RUnlock()
 	return consensus.isViewChangingMode()
 }
 
@@ -68,7 +67,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, peer libp2p
 	// Do easier check before signature check
 	if msg.Type == msg_pb.MessageType_ANNOUNCE || msg.Type == msg_pb.MessageType_PREPARED || msg.Type == msg_pb.MessageType_COMMITTED {
 		// Only validator needs to check whether the message is from the correct leader
-		if !bytes.Equal(senderKey[:], consensus.LeaderPubKey.Bytes[:]) &&
+		if !bytes.Equal(senderKey[:], consensus.getLeaderPubKey().Bytes[:]) &&
 			consensus.current.Mode() == Normal && !consensus.IgnoreViewIDCheck.IsSet() {
 			return errSenderPubKeyNotLeader
 		}
@@ -666,9 +665,7 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess
 	}
 
 	consensus.FinishFinalityCount()
-	go func() {
-		consensus.PostConsensusJob(blk)
-	}()
+	consensus.PostConsensusProcessing(blk)
 	consensus.setupForNewConsensus(blk, committedMsg)
 	utils.Logger().Info().Uint64("blockNum", blk.NumberU64()).
 		Str("hash", blk.Header().Hash().Hex()).
diff --git a/consensus/debug.go b/consensus/debug.go
index a323e04ed6..c195305809 100644
--- a/consensus/debug.go
+++ b/consensus/debug.go
@@ -14,15 +14,12 @@ func (consensus *Consensus) getConsensusPhase() string {
 
 // GetConsensusMode returns the current mode of the consensus
 func (consensus *Consensus) GetConsensusMode() string {
-	consensus.mutex.RLock()
-	defer consensus.mutex.RUnlock()
-	return consensus.current.mode.String()
+	return consensus.current.Mode().String()
 }
 
 // GetCurBlockViewID returns the current view ID of the consensus
+// Method is thread safe.
 func (consensus *Consensus) GetCurBlockViewID() uint64 {
-	consensus.mutex.RLock()
-	defer consensus.mutex.RUnlock()
 	return consensus.getCurBlockViewID()
 }
 
@@ -31,10 +28,9 @@ func (consensus *Consensus) getCurBlockViewID() uint64 {
 	return consensus.current.GetCurBlockViewID()
 }
 
-// GetViewChangingID returns the current view changing ID of the consensus
+// GetViewChangingID returns the current view changing ID of the consensus.
+// Method is thread safe.
 func (consensus *Consensus) GetViewChangingID() uint64 {
-	consensus.mutex.RLock()
-	defer consensus.mutex.RUnlock()
 	return consensus.current.GetViewChangingID()
 }
 
diff --git a/consensus/double_sign.go b/consensus/double_sign.go
index 144c67bff7..6ad2efa0f8 100644
--- a/consensus/double_sign.go
+++ b/consensus/double_sign.go
@@ -71,7 +71,7 @@ func (consensus *Consensus) checkDoubleSign(recvMsg *FBFTMessage) bool {
 									break
 								}
 
-								leaderAddr, err := subComm.AddressForBLSKey(consensus.LeaderPubKey.Bytes)
+								leaderAddr, err := subComm.AddressForBLSKey(consensus.getLeaderPubKey().Bytes)
 								if err != nil {
 									consensus.getLogger().Err(err).Str("msg", recvMsg.String()).
 										Msg("could not find address for leader bls key")
diff --git a/consensus/enums.go b/consensus/enums.go
index 41eafba863..7c4b28be25 100644
--- a/consensus/enums.go
+++ b/consensus/enums.go
@@ -3,7 +3,7 @@ package consensus
 import "fmt"
 
 // Mode is the current
-type Mode byte
+type Mode uint32
 
 const (
 	// Normal ..
diff --git a/consensus/post_processing.go b/consensus/post_processing.go
new file mode 100644
index 0000000000..384546f713
--- /dev/null
+++ b/consensus/post_processing.go
@@ -0,0 +1,232 @@
+package consensus
+
+import (
+	"math/rand"
+
+	proto_node "github.com/harmony-one/harmony/api/proto/node"
+	"github.com/harmony-one/harmony/block"
+	"github.com/harmony-one/harmony/core"
+	"github.com/harmony-one/harmony/core/types"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/p2p"
+	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/availability"
+	"github.com/harmony-one/harmony/webhooks"
+)
+
+// PostConsensusProcessing is called by consensus participants, after consensus is done, to:
+// 1. [leader] send new block to the client
+// 2. [leader] send cross shard tx receipts to destination shard
+func (consensus *Consensus) PostConsensusProcessing(newBlock *types.Block) error {
+	if consensus.IsLeader() {
+		if IsRunningBeaconChain(consensus) {
+			// TODO: consider removing this and letting other nodes broadcast new blocks.
+			// But need to make sure there is at least 1 node that will do the job.
+			BroadcastNewBlock(consensus.host, newBlock, consensus.registry.GetNodeConfig())
+		}
+		BroadcastCXReceipts(newBlock, consensus)
+	} else {
+		if mode := consensus.mode(); mode != Listening {
+			numSignatures := consensus.NumSignaturesIncludedInBlock(newBlock)
+			utils.Logger().Info().
+				Uint64("blockNum", newBlock.NumberU64()).
+				Uint64("epochNum", newBlock.Epoch().Uint64()).
+				Uint64("ViewId", newBlock.Header().ViewID().Uint64()).
+				Str("blockHash", newBlock.Hash().String()).
+				Int("numTxns", len(newBlock.Transactions())).
+				Int("numStakingTxns", len(newBlock.StakingTransactions())).
+				Uint32("numSignatures", numSignatures).
+				Str("mode", mode.String()).
+				Msg("BINGO !!! Reached Consensus")
+			if consensus.mode() == Syncing {
+				mode = consensus.updateConsensusInformation()
+				utils.Logger().Info().Msgf("Switching to mode %s", mode)
+				consensus.setMode(mode)
+			}
+
+			consensus.UpdateValidatorMetrics(float64(numSignatures), float64(newBlock.NumberU64()))
+
+			// 1% of the validator also need to do broadcasting
+			rnd := rand.Intn(100)
+			if rnd < 1 {
+				// Beacon validators also broadcast new blocks to make sure beacon sync is strong.
+				if IsRunningBeaconChain(consensus) {
+					BroadcastNewBlock(consensus.host, newBlock, consensus.registry.GetNodeConfig())
+				}
+				BroadcastCXReceipts(newBlock, consensus)
+			}
+		}
+	}
+
+	// Broadcast client requested missing cross shard receipts if there is any
+	BroadcastMissingCXReceipts(consensus)
+
+	if h := consensus.registry.GetNodeConfig().WebHooks.Hooks; h != nil {
+		if h.Availability != nil {
+			shardState, err := consensus.Blockchain().ReadShardState(newBlock.Epoch())
+			if err != nil {
+				utils.Logger().Error().Err(err).
+					Int64("epoch", newBlock.Epoch().Int64()).
+					Uint32("shard-id", consensus.ShardID).
+					Msg("failed to read shard state")
+				return err
+			}
+
+			for _, addr := range consensus.Registry().GetAddressToBLSKey().GetAddresses(consensus.getPublicKeys(), shardState, newBlock.Epoch()) {
+				wrapper, err := consensus.Beaconchain().ReadValidatorInformation(addr)
+				if err != nil {
+					utils.Logger().Err(err).Str("addr", addr.Hex()).Msg("failed reaching validator info")
+					return nil
+				}
+				snapshot, err := consensus.Beaconchain().ReadValidatorSnapshot(addr)
+				if err != nil {
+					utils.Logger().Err(err).Str("addr", addr.Hex()).Msg("failed reaching validator snapshot")
+					return nil
+				}
+				computed := availability.ComputeCurrentSigning(
+					snapshot.Validator, wrapper,
+				)
+				lastBlockOfEpoch := shard.Schedule.EpochLastBlock(consensus.Beaconchain().CurrentBlock().Header().Epoch().Uint64())
+
+				computed.BlocksLeftInEpoch = lastBlockOfEpoch - consensus.Beaconchain().CurrentBlock().Header().Number().Uint64()
+
+				if err != nil && computed.IsBelowThreshold {
+					url := h.Availability.OnDroppedBelowThreshold
+					go func() {
+						webhooks.DoPost(url, computed)
+					}()
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// BroadcastNewBlock is called by consensus leader to sync new blocks with other clients/nodes.
+// NOTE: For now, just send to the client (basically not broadcasting)
+// TODO (lc): broadcast the new blocks to new nodes doing state sync
+func BroadcastNewBlock(host p2p.Host, newBlock *types.Block, nodeConfig *nodeconfig.ConfigType) {
+	groups := []nodeconfig.GroupID{nodeConfig.GetClientGroupID()}
+	utils.Logger().Info().
+		Msgf(
+			"broadcasting new block %d, group %s", newBlock.NumberU64(), groups[0],
+		)
+	msg := p2p.ConstructMessage(
+		proto_node.ConstructBlocksSyncMessage([]*types.Block{newBlock}),
+	)
+	if err := host.SendMessageToGroups(groups, msg); err != nil {
+		utils.Logger().Warn().Err(err).Msg("cannot broadcast new block")
+	}
+}
+
+func IsRunningBeaconChain(c *Consensus) bool {
+	return c.ShardID == shard.BeaconChainShardID
+}
+
+// BroadcastCXReceipts broadcasts cross shard receipts to correspoding
+// destination shards
+func BroadcastCXReceipts(newBlock *types.Block, consensus *Consensus) {
+	commitSigAndBitmap := newBlock.GetCurrentCommitSig()
+	//#### Read payload data from committed msg
+	if len(commitSigAndBitmap) <= 96 {
+		utils.Logger().Debug().Int("commitSigAndBitmapLen", len(commitSigAndBitmap)).Msg("[BroadcastCXReceipts] commitSigAndBitmap Not Enough Length")
+		return
+	}
+	commitSig := make([]byte, 96)
+	commitBitmap := make([]byte, len(commitSigAndBitmap)-96)
+	offset := 0
+	copy(commitSig[:], commitSigAndBitmap[offset:offset+96])
+	offset += 96
+	copy(commitBitmap[:], commitSigAndBitmap[offset:])
+	//#### END Read payload data from committed msg
+
+	epoch := newBlock.Header().Epoch()
+	shardingConfig := shard.Schedule.InstanceForEpoch(epoch)
+	shardNum := int(shardingConfig.NumShards())
+	myShardID := consensus.ShardID
+	utils.Logger().Info().Int("shardNum", shardNum).Uint32("myShardID", myShardID).Uint64("blockNum", newBlock.NumberU64()).Msg("[BroadcastCXReceipts]")
+
+	for i := 0; i < shardNum; i++ {
+		if i == int(myShardID) {
+			continue
+		}
+		BroadcastCXReceiptsWithShardID(newBlock.Header(), commitSig, commitBitmap, uint32(i), consensus)
+	}
+}
+
+// BroadcastCXReceiptsWithShardID broadcasts cross shard receipts to given ToShardID
+func BroadcastCXReceiptsWithShardID(block *block.Header, commitSig []byte, commitBitmap []byte, toShardID uint32, consensus *Consensus) {
+	myShardID := consensus.ShardID
+	utils.Logger().Debug().
+		Uint32("toShardID", toShardID).
+		Uint32("myShardID", myShardID).
+		Uint64("blockNum", block.NumberU64()).
+		Msg("[BroadcastCXReceiptsWithShardID]")
+
+	cxReceipts, err := consensus.Blockchain().ReadCXReceipts(toShardID, block.NumberU64(), block.Hash())
+	if err != nil || len(cxReceipts) == 0 {
+		utils.Logger().Debug().Uint32("ToShardID", toShardID).
+			Int("numCXReceipts", len(cxReceipts)).
+			Msg("[CXMerkleProof] No receipts found for the destination shard")
+		return
+	}
+
+	merkleProof, err := consensus.Blockchain().CXMerkleProof(toShardID, block)
+	if err != nil {
+		utils.Logger().Warn().
+			Uint32("ToShardID", toShardID).
+			Msg("[BroadcastCXReceiptsWithShardID] Unable to get merkleProof")
+		return
+	}
+
+	cxReceiptsProof := &types.CXReceiptsProof{
+		Receipts:     cxReceipts,
+		MerkleProof:  merkleProof,
+		Header:       block,
+		CommitSig:    commitSig,
+		CommitBitmap: commitBitmap,
+	}
+
+	groupID := nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(toShardID))
+	utils.Logger().Info().Uint32("ToShardID", toShardID).
+		Str("GroupID", string(groupID)).
+		Interface("cxp", cxReceiptsProof).
+		Msg("[BroadcastCXReceiptsWithShardID] ReadCXReceipts and MerkleProof ready. Sending CX receipts...")
+	// TODO ek – limit concurrency
+	consensus.GetHost().SendMessageToGroups([]nodeconfig.GroupID{groupID},
+		p2p.ConstructMessage(proto_node.ConstructCXReceiptsProof(cxReceiptsProof)),
+	)
+}
+
+// BroadcastMissingCXReceipts broadcasts missing cross shard receipts per request
+func BroadcastMissingCXReceipts(c *Consensus) {
+	var (
+		sendNextTime = make([]core.CxEntry, 0)
+		cxPool       = c.Registry().GetCxPool()
+		blockchain   = c.Blockchain()
+	)
+	it := cxPool.Pool().Iterator()
+	for entry := range it.C {
+		cxEntry := entry.(core.CxEntry)
+		toShardID := cxEntry.ToShardID
+		blk := blockchain.GetBlockByHash(cxEntry.BlockHash)
+		if blk == nil {
+			continue
+		}
+		blockNum := blk.NumberU64()
+		nextHeader := blockchain.GetHeaderByNumber(blockNum + 1)
+		if nextHeader == nil {
+			sendNextTime = append(sendNextTime, cxEntry)
+			continue
+		}
+		sig := nextHeader.LastCommitSignature()
+		bitmap := nextHeader.LastCommitBitmap()
+		BroadcastCXReceiptsWithShardID(blk.Header(), sig[:], bitmap, toShardID, c)
+	}
+	cxPool.Clear()
+	// this should not happen or maybe happen for impatient user
+	for _, entry := range sendNextTime {
+		cxPool.Add(entry)
+	}
+}
diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go
index 148c8cb4a9..070a85e029 100644
--- a/consensus/quorum/quorum.go
+++ b/consensus/quorum/quorum.go
@@ -3,6 +3,7 @@ package quorum
 import (
 	"fmt"
 	"math/big"
+	"sync/atomic"
 
 	"github.com/harmony-one/harmony/crypto/bls"
 
@@ -153,7 +154,8 @@ type cIdentities struct {
 	commit      *votepower.Round
 	// viewIDSigs: every validator
 	// sign on |viewID|blockHash| in view changing message
-	viewChange *votepower.Round
+	viewChange        *votepower.Round
+	participantsCount int64
 }
 
 func (s *cIdentities) AggregateVotes(p Phase) *bls_core.Sign {
@@ -286,10 +288,11 @@ func (s *cIdentities) UpdateParticipants(pubKeys, allowlist []bls.PublicKeyWrapp
 	}
 	s.publicKeys = pubKeys
 	s.keyIndexMap = keyIndexMap
+	atomic.StoreInt64(&s.participantsCount, int64(len(s.publicKeys)))
 }
 
 func (s *cIdentities) ParticipantsCount() int64 {
-	return int64(len(s.publicKeys))
+	return atomic.LoadInt64(&s.participantsCount)
 }
 
 func (s *cIdentities) SignersCount(p Phase) int64 {
diff --git a/consensus/quorum/thread_safe_decider.go b/consensus/quorum/thread_safe_decider.go
index 9df0ec03f1..6b6279e53e 100644
--- a/consensus/quorum/thread_safe_decider.go
+++ b/consensus/quorum/thread_safe_decider.go
@@ -47,8 +47,6 @@ func (a threadSafeDeciderImpl) IndexOf(key bls.SerializedPublicKey) int {
 }
 
 func (a threadSafeDeciderImpl) ParticipantsCount() int64 {
-	a.mu.Lock()
-	defer a.mu.Unlock()
 	return a.decider.ParticipantsCount()
 }
 
diff --git a/consensus/view_change.go b/consensus/view_change.go
index a0ac6c0f79..2752270cba 100644
--- a/consensus/view_change.go
+++ b/consensus/view_change.go
@@ -2,15 +2,14 @@ package consensus
 
 import (
 	"math/big"
+	"sync/atomic"
 	"time"
 
-	"github.com/harmony-one/harmony/internal/chain"
-
-	"github.com/harmony-one/harmony/crypto/bls"
-
 	"github.com/ethereum/go-ethereum/common"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/consensus/quorum"
+	"github.com/harmony-one/harmony/crypto/bls"
+	"github.com/harmony-one/harmony/internal/chain"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/p2p"
@@ -24,7 +23,7 @@ const MaxViewIDDiff = 249
 
 // State contains current mode and current viewID
 type State struct {
-	mode Mode
+	mode uint32
 
 	// current view id in normal mode
 	// it changes per successful consensus
@@ -35,43 +34,49 @@ type State struct {
 	viewChangingID uint64
 }
 
+func NewState(mode Mode) State {
+	return State{
+		mode: uint32(mode),
+	}
+}
+
 // Mode return the current node mode
 func (pm *State) Mode() Mode {
-	return pm.mode
+	return Mode(atomic.LoadUint32(&pm.mode))
 }
 
 // SetMode set the node mode as required
 func (pm *State) SetMode(s Mode) {
-	pm.mode = s
+	atomic.StoreUint32(&pm.mode, uint32(s))
 }
 
 // GetCurBlockViewID return the current view id
 func (pm *State) GetCurBlockViewID() uint64 {
-	return pm.blockViewID
+	return atomic.LoadUint64(&pm.blockViewID)
 }
 
 // SetCurBlockViewID sets the current view id
 func (pm *State) SetCurBlockViewID(viewID uint64) uint64 {
-	pm.blockViewID = viewID
-	return pm.blockViewID
+	atomic.StoreUint64(&pm.blockViewID, viewID)
+	return viewID
 }
 
 // GetViewChangingID return the current view changing id
 // It is meaningful during view change mode
 func (pm *State) GetViewChangingID() uint64 {
-	return pm.viewChangingID
+	return atomic.LoadUint64(&pm.viewChangingID)
 }
 
 // SetViewChangingID set the current view changing id
 // It is meaningful during view change mode
 func (pm *State) SetViewChangingID(id uint64) {
-	pm.viewChangingID = id
+	atomic.StoreUint64(&pm.viewChangingID, id)
 }
 
 // GetViewChangeDuraion return the duration of the current view change
 // It increase in the power of difference betweeen view changing ID and current view ID
 func (pm *State) GetViewChangeDuraion() time.Duration {
-	diff := int64(pm.viewChangingID - pm.blockViewID)
+	diff := int64(pm.GetViewChangingID() - pm.GetCurBlockViewID())
 	return time.Duration(diff * diff * int64(viewChangeDuration))
 }
 
@@ -152,12 +157,12 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64, committee *shard.Com
 	epoch := big.NewInt(0)
 	if blockchain == nil {
 		consensus.getLogger().Error().Msg("[getNextLeaderKey] Blockchain is nil. Use consensus.LeaderPubKey")
-		lastLeaderPubKey = consensus.LeaderPubKey
+		lastLeaderPubKey = consensus.getLeaderPubKey()
 	} else {
 		curHeader := blockchain.CurrentHeader()
 		if curHeader == nil {
 			consensus.getLogger().Error().Msg("[getNextLeaderKey] Failed to get current header from blockchain")
-			lastLeaderPubKey = consensus.LeaderPubKey
+			lastLeaderPubKey = consensus.getLeaderPubKey()
 		} else {
 			stuckBlockViewID := curHeader.ViewID().Uint64() + 1
 			gap = int(viewID - stuckBlockViewID)
@@ -166,7 +171,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64, committee *shard.Com
 			if err != nil || lastLeaderPubKey == nil {
 				consensus.getLogger().Error().Err(err).
 					Msg("[getNextLeaderKey] Unable to get leaderPubKey from coinbase. Set it to consensus.LeaderPubKey")
-				lastLeaderPubKey = consensus.LeaderPubKey
+				lastLeaderPubKey = consensus.getLeaderPubKey()
 			}
 			epoch = curHeader.Epoch()
 			// viewchange happened at the first block of new epoch
@@ -183,7 +188,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64, committee *shard.Com
 	}
 	consensus.getLogger().Info().
 		Str("lastLeaderPubKey", lastLeaderPubKey.Bytes.Hex()).
-		Str("leaderPubKey", consensus.LeaderPubKey.Bytes.Hex()).
+		Str("leaderPubKey", consensus.getLeaderPubKey().Bytes.Hex()).
 		Int("gap", gap).
 		Uint64("newViewID", viewID).
 		Uint64("myCurBlockViewID", consensus.getCurBlockViewID()).
@@ -212,7 +217,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64, committee *shard.Com
 	}
 	if !wasFound {
 		consensus.getLogger().Warn().
-			Str("key", consensus.LeaderPubKey.Bytes.Hex()).
+			Str("key", consensus.getLeaderPubKey().Bytes.Hex()).
 			Msg("[getNextLeaderKey] currentLeaderKey not found")
 	}
 	consensus.getLogger().Info().
@@ -257,13 +262,13 @@ func (consensus *Consensus) startViewChange() {
 	// aganist the consensus.LeaderPubKey variable.
 	// Ideally, we shall use another variable to keep track of the
 	// leader pubkey in viewchange mode
-	consensus.LeaderPubKey = consensus.getNextLeaderKey(nextViewID, committee)
+	consensus.setLeaderPubKey(consensus.getNextLeaderKey(nextViewID, committee))
 
 	consensus.getLogger().Warn().
 		Uint64("nextViewID", nextViewID).
 		Uint64("viewChangingID", consensus.getViewChangingID()).
 		Dur("timeoutDuration", duration).
-		Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()).
+		Str("NextLeader", consensus.getLeaderPubKey().Bytes.Hex()).
 		Msg("[startViewChange]")
 	consensusVCCounterVec.With(prometheus.Labels{"viewchange": "started"}).Inc()
 
@@ -540,7 +545,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) {
 
 	// newView message verified success, override my state
 	consensus.setViewIDs(recvMsg.ViewID)
-	consensus.LeaderPubKey = senderKey
+	consensus.setLeaderPubKey(senderKey)
 	consensus.resetViewChangeState()
 
 	consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE)
@@ -554,7 +559,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) {
 		consensus.getLogger().Info().Msg("onNewView === announce")
 	}
 	consensus.getLogger().Info().
-		Str("newLeaderKey", consensus.LeaderPubKey.Bytes.Hex()).
+		Str("newLeaderKey", consensus.getLeaderPubKey().Bytes.Hex()).
 		Msg("new leader changed")
 	consensus.consensusTimeout[timeoutConsensus].Start()
 	consensusVCCounterVec.With(prometheus.Labels{"viewchange": "finished"}).Inc()
diff --git a/consensus/view_change_msg.go b/consensus/view_change_msg.go
index 6c4b080055..1ba9c76063 100644
--- a/consensus/view_change_msg.go
+++ b/consensus/view_change_msg.go
@@ -27,7 +27,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra
 				BlockNum:     consensus.getBlockNum(),
 				ShardId:      consensus.ShardID,
 				SenderPubkey: priKey.Pub.Bytes[:],
-				LeaderPubkey: consensus.LeaderPubKey.Bytes[:],
+				LeaderPubkey: consensus.getLeaderPubKey().Bytes[:],
 			},
 		},
 	}
@@ -71,7 +71,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra
 
 	consensus.getLogger().Info().
 		Hex("m1Payload", vcMsg.Payload).
-		Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()).
+		Str("NextLeader", consensus.getLeaderPubKey().Bytes.Hex()).
 		Str("SenderPubKey", priKey.Pub.Bytes.Hex()).
 		Msg("[constructViewChangeMessage]")
 
diff --git a/consensus/view_change_test.go b/consensus/view_change_test.go
index bbc6999445..42b44f7c78 100644
--- a/consensus/view_change_test.go
+++ b/consensus/view_change_test.go
@@ -14,14 +14,13 @@ func TestBasicViewChanging(t *testing.T) {
 	_, _, consensus, _, err := GenerateConsensusForTesting()
 	assert.NoError(t, err)
 
-	state := State{mode: Normal}
+	state := NewState(Normal)
 
 	// Change Mode
 	assert.Equal(t, state.mode, consensus.current.mode)
 	assert.Equal(t, state.Mode(), consensus.current.Mode())
 
 	consensus.current.SetMode(ViewChanging)
-	assert.Equal(t, ViewChanging, consensus.current.mode)
 	assert.Equal(t, ViewChanging, consensus.current.Mode())
 
 	// Change ViewID
@@ -114,7 +113,7 @@ func TestGetNextLeaderKeyShouldSucceed(t *testing.T) {
 	consensus.Decider().UpdateParticipants(wrappedBLSKeys, []bls.PublicKeyWrapper{})
 	assert.Equal(t, keyCount, consensus.Decider().ParticipantsCount())
 
-	consensus.LeaderPubKey = &wrappedBLSKeys[0]
+	consensus.setLeaderPubKey(&wrappedBLSKeys[0])
 	nextKey := consensus.getNextLeaderKey(uint64(1), nil)
 
 	assert.Equal(t, nextKey, &wrappedBLSKeys[1])
diff --git a/node/node_cross_shard.go b/node/node_cross_shard.go
index e516394834..484c7926a8 100644
--- a/node/node_cross_shard.go
+++ b/node/node_cross_shard.go
@@ -2,124 +2,10 @@ package node
 
 import (
 	"github.com/ethereum/go-ethereum/rlp"
-	proto_node "github.com/harmony-one/harmony/api/proto/node"
-	"github.com/harmony-one/harmony/block"
-	"github.com/harmony-one/harmony/consensus"
-	"github.com/harmony-one/harmony/core"
 	"github.com/harmony-one/harmony/core/types"
-	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
-	"github.com/harmony-one/harmony/p2p"
-	"github.com/harmony-one/harmony/shard"
 )
 
-// BroadcastCXReceipts broadcasts cross shard receipts to correspoding
-// destination shards
-func BroadcastCXReceipts(newBlock *types.Block, consensus *consensus.Consensus) {
-	commitSigAndBitmap := newBlock.GetCurrentCommitSig()
-	//#### Read payload data from committed msg
-	if len(commitSigAndBitmap) <= 96 {
-		utils.Logger().Debug().Int("commitSigAndBitmapLen", len(commitSigAndBitmap)).Msg("[BroadcastCXReceipts] commitSigAndBitmap Not Enough Length")
-		return
-	}
-	commitSig := make([]byte, 96)
-	commitBitmap := make([]byte, len(commitSigAndBitmap)-96)
-	offset := 0
-	copy(commitSig[:], commitSigAndBitmap[offset:offset+96])
-	offset += 96
-	copy(commitBitmap[:], commitSigAndBitmap[offset:])
-	//#### END Read payload data from committed msg
-
-	epoch := newBlock.Header().Epoch()
-	shardingConfig := shard.Schedule.InstanceForEpoch(epoch)
-	shardNum := int(shardingConfig.NumShards())
-	myShardID := consensus.ShardID
-	utils.Logger().Info().Int("shardNum", shardNum).Uint32("myShardID", myShardID).Uint64("blockNum", newBlock.NumberU64()).Msg("[BroadcastCXReceipts]")
-
-	for i := 0; i < shardNum; i++ {
-		if i == int(myShardID) {
-			continue
-		}
-		BroadcastCXReceiptsWithShardID(newBlock.Header(), commitSig, commitBitmap, uint32(i), consensus)
-	}
-}
-
-// BroadcastCXReceiptsWithShardID broadcasts cross shard receipts to given ToShardID
-func BroadcastCXReceiptsWithShardID(block *block.Header, commitSig []byte, commitBitmap []byte, toShardID uint32, consensus *consensus.Consensus) {
-	myShardID := consensus.ShardID
-	utils.Logger().Debug().
-		Uint32("toShardID", toShardID).
-		Uint32("myShardID", myShardID).
-		Uint64("blockNum", block.NumberU64()).
-		Msg("[BroadcastCXReceiptsWithShardID]")
-
-	cxReceipts, err := consensus.Blockchain().ReadCXReceipts(toShardID, block.NumberU64(), block.Hash())
-	if err != nil || len(cxReceipts) == 0 {
-		utils.Logger().Debug().Uint32("ToShardID", toShardID).
-			Int("numCXReceipts", len(cxReceipts)).
-			Msg("[CXMerkleProof] No receipts found for the destination shard")
-		return
-	}
-
-	merkleProof, err := consensus.Blockchain().CXMerkleProof(toShardID, block)
-	if err != nil {
-		utils.Logger().Warn().
-			Uint32("ToShardID", toShardID).
-			Msg("[BroadcastCXReceiptsWithShardID] Unable to get merkleProof")
-		return
-	}
-
-	cxReceiptsProof := &types.CXReceiptsProof{
-		Receipts:     cxReceipts,
-		MerkleProof:  merkleProof,
-		Header:       block,
-		CommitSig:    commitSig,
-		CommitBitmap: commitBitmap,
-	}
-
-	groupID := nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(toShardID))
-	utils.Logger().Info().Uint32("ToShardID", toShardID).
-		Str("GroupID", string(groupID)).
-		Interface("cxp", cxReceiptsProof).
-		Msg("[BroadcastCXReceiptsWithShardID] ReadCXReceipts and MerkleProof ready. Sending CX receipts...")
-	// TODO ek – limit concurrency
-	go consensus.GetHost().SendMessageToGroups([]nodeconfig.GroupID{groupID},
-		p2p.ConstructMessage(proto_node.ConstructCXReceiptsProof(cxReceiptsProof)),
-	)
-}
-
-// BroadcastMissingCXReceipts broadcasts missing cross shard receipts per request
-func BroadcastMissingCXReceipts(c *consensus.Consensus) {
-	var (
-		sendNextTime = make([]core.CxEntry, 0)
-		cxPool       = c.Registry().GetCxPool()
-		blockchain   = c.Blockchain()
-	)
-	it := cxPool.Pool().Iterator()
-	for entry := range it.C {
-		cxEntry := entry.(core.CxEntry)
-		toShardID := cxEntry.ToShardID
-		blk := blockchain.GetBlockByHash(cxEntry.BlockHash)
-		if blk == nil {
-			continue
-		}
-		blockNum := blk.NumberU64()
-		nextHeader := blockchain.GetHeaderByNumber(blockNum + 1)
-		if nextHeader == nil {
-			sendNextTime = append(sendNextTime, cxEntry)
-			continue
-		}
-		sig := nextHeader.LastCommitSignature()
-		bitmap := nextHeader.LastCommitBitmap()
-		BroadcastCXReceiptsWithShardID(blk.Header(), sig[:], bitmap, toShardID, c)
-	}
-	cxPool.Clear()
-	// this should not happen or maybe happen for impatient user
-	for _, entry := range sendNextTime {
-		cxPool.Add(entry)
-	}
-}
-
 // ProcessReceiptMessage store the receipts and merkle proof in local data store
 func (node *Node) ProcessReceiptMessage(msgPayload []byte) {
 	cxp := types.CXReceiptsProof{}
diff --git a/node/node_handler.go b/node/node_handler.go
index 7aa87100bb..02c6f30884 100644
--- a/node/node_handler.go
+++ b/node/node_handler.go
@@ -14,17 +14,14 @@ import (
 	"github.com/harmony-one/harmony/api/proto"
 	proto_node "github.com/harmony-one/harmony/api/proto/node"
 	"github.com/harmony-one/harmony/block"
-	"github.com/harmony-one/harmony/consensus"
 	"github.com/harmony-one/harmony/core"
 	"github.com/harmony-one/harmony/core/types"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/shard"
-	"github.com/harmony-one/harmony/staking/availability"
 	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
-	"github.com/harmony-one/harmony/webhooks"
 )
 
 const p2pMsgPrefixSize = 5
@@ -325,97 +322,6 @@ func getCrosslinkHeadersForShards(shardChain core.BlockChain, curBlock *types.Bl
 	return headers, nil
 }
 
-// PostConsensusProcessing is called by consensus participants, after consensus is done, to:
-// 1. [leader] send new block to the client
-// 2. [leader] send cross shard tx receipts to destination shard
-func (node *Node) PostConsensusProcessing(newBlock *types.Block) error {
-	if node.Consensus.IsLeader() {
-		if IsRunningBeaconChain(node.Consensus) {
-			// TODO: consider removing this and letting other nodes broadcast new blocks.
-			// But need to make sure there is at least 1 node that will do the job.
-			BroadcastNewBlock(node.host, newBlock, node.NodeConfig)
-		}
-		BroadcastCXReceipts(newBlock, node.Consensus)
-	} else {
-		if mode := node.Consensus.Mode(); mode != consensus.Listening {
-			numSignatures := node.Consensus.NumSignaturesIncludedInBlock(newBlock)
-			utils.Logger().Info().
-				Uint64("blockNum", newBlock.NumberU64()).
-				Uint64("epochNum", newBlock.Epoch().Uint64()).
-				Uint64("ViewId", newBlock.Header().ViewID().Uint64()).
-				Str("blockHash", newBlock.Hash().String()).
-				Int("numTxns", len(newBlock.Transactions())).
-				Int("numStakingTxns", len(newBlock.StakingTransactions())).
-				Uint32("numSignatures", numSignatures).
-				Str("mode", mode.String()).
-				Msg("BINGO !!! Reached Consensus")
-			if node.Consensus.Mode() == consensus.Syncing {
-				mode = node.Consensus.UpdateConsensusInformation()
-				utils.Logger().Info().Msgf("Switching to mode %s", mode)
-				node.Consensus.SetMode(mode)
-			}
-
-			node.Consensus.UpdateValidatorMetrics(float64(numSignatures), float64(newBlock.NumberU64()))
-
-			// 1% of the validator also need to do broadcasting
-			rnd := rand.Intn(100)
-			if rnd < 1 {
-				// Beacon validators also broadcast new blocks to make sure beacon sync is strong.
-				if IsRunningBeaconChain(node.Consensus) {
-					BroadcastNewBlock(node.host, newBlock, node.NodeConfig)
-				}
-				BroadcastCXReceipts(newBlock, node.Consensus)
-			}
-		}
-	}
-
-	// Broadcast client requested missing cross shard receipts if there is any
-	BroadcastMissingCXReceipts(node.Consensus)
-
-	if h := node.NodeConfig.WebHooks.Hooks; h != nil {
-		if h.Availability != nil {
-			shardState, err := node.Blockchain().ReadShardState(newBlock.Epoch())
-			if err != nil {
-				utils.Logger().Error().Err(err).
-					Int64("epoch", newBlock.Epoch().Int64()).
-					Uint32("shard-id", node.Consensus.ShardID).
-					Msg("failed to read shard state")
-				return err
-			}
-			for _, addr := range node.Consensus.Registry().GetAddressToBLSKey().GetAddresses(node.Consensus.GetPublicKeys(), shardState, newBlock.Epoch()) {
-				wrapper, err := node.Beaconchain().ReadValidatorInformation(addr)
-				if err != nil {
-					utils.Logger().Err(err).Str("addr", addr.Hex()).Msg("failed reaching validator info")
-					return nil
-				}
-				snapshot, err := node.Beaconchain().ReadValidatorSnapshot(addr)
-				if err != nil {
-					utils.Logger().Err(err).Str("addr", addr.Hex()).Msg("failed reaching validator snapshot")
-					return nil
-				}
-				computed := availability.ComputeCurrentSigning(
-					snapshot.Validator, wrapper,
-				)
-				lastBlockOfEpoch := shard.Schedule.EpochLastBlock(node.Beaconchain().CurrentBlock().Header().Epoch().Uint64())
-
-				computed.BlocksLeftInEpoch = lastBlockOfEpoch - node.Beaconchain().CurrentBlock().Header().Number().Uint64()
-
-				if err != nil && computed.IsBelowThreshold {
-					url := h.Availability.OnDroppedBelowThreshold
-					go func() {
-						webhooks.DoPost(url, computed)
-					}()
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func IsRunningBeaconChain(c *consensus.Consensus) bool {
-	return c.ShardID == shard.BeaconChainShardID
-}
-
 // BootstrapConsensus is a goroutine to check number of peers and start the consensus
 func (node *Node) BootstrapConsensus() error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
diff --git a/node/node_handler_test.go b/node/node_handler_test.go
index 867a9616dc..65c960e028 100644
--- a/node/node_handler_test.go
+++ b/node/node_handler_test.go
@@ -203,7 +203,7 @@ func TestVerifyVRF(t *testing.T) {
 	shardState.Epoch = big.NewInt(1)
 	shardState.Shards = append(shardState.Shards, com)
 
-	node.Consensus.LeaderPubKey = &bls.PublicKeyWrapper{Bytes: spKey, Object: pubKey}
+	node.Consensus.SetLeaderPubKey(&bls.PublicKeyWrapper{Bytes: spKey, Object: pubKey})
 	node.Worker.GetCurrentHeader().SetEpoch(big.NewInt(1))
 	node.Consensus.GenerateVrfAndProof(node.Worker.GetCurrentHeader())
 	block, _ := node.Worker.FinalizeNewBlock(

From 583e38a0b80046a7f1e734707a7f2799c6cad815 Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Wed, 28 Aug 2024 16:14:06 +0300
Subject: [PATCH 03/24] chore(CI) - point tests to the master

---
 .travis.yml                       | 4 +---
 scripts/travis_rosetta_checker.sh | 2 --
 scripts/travis_rpc_checker.sh     | 2 --
 3 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 96af4ab61c..1f58faf9e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,11 +24,9 @@ install:
   #   /home/travis/gopath/src/github.com/harmony-one/harmony
   # https://docs.travis-ci.com/user/languages/go/#go-import-path
   - echo $TRAVIS_PULL_REQUEST_BRANCH
-    #TODO: return back to master
-  - TEST_REPO_BRANCH="feature/go1.22"
+  - TEST_REPO_BRANCH="master"
   - git clone https://github.com/harmony-one/mcl.git $GOPATH/src/github.com/harmony-one/mcl
   - git clone https://github.com/harmony-one/bls.git $GOPATH/src/github.com/harmony-one/bls
-    #TODO: return back harmony instead of personal fork
   - git clone --branch $TEST_REPO_BRANCH https://github.com/harmony-one/harmony-test.git $GOPATH/src/github.com/harmony-one/harmony-test
   - (cd $GOPATH/src/github.com/harmony-one/mcl; make -j4)
   - (cd $GOPATH/src/github.com/harmony-one/bls; make BLS_SWAP_G=1 -j4)
diff --git a/scripts/travis_rosetta_checker.sh b/scripts/travis_rosetta_checker.sh
index 00cc67014f..a2a20bc1a1 100755
--- a/scripts/travis_rosetta_checker.sh
+++ b/scripts/travis_rosetta_checker.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-#TODO: return back to master
-TEST_REPO_BRANCH="feature/go1.22"
 TEST_REPO_BRANCH=${TEST_REPO_BRANCH:-master}
 # handle for the Travis build run:
 # * uses TRAVIS_PULL_REQUEST_BRANCH for RP branch
diff --git a/scripts/travis_rpc_checker.sh b/scripts/travis_rpc_checker.sh
index 13c64061fb..2d9bd93eba 100755
--- a/scripts/travis_rpc_checker.sh
+++ b/scripts/travis_rpc_checker.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-#TODO: return back to master
-TEST_REPO_BRANCH="feature/go1.22"
 TEST_REPO_BRANCH=${TEST_REPO_BRANCH:-master}
 # handle for the Travis build run:
 # * uses TRAVIS_PULL_REQUEST_BRANCH for RP branch

From e1b0ebc2e2f1b6941be2bbe3ee1f1cf2a51c8c78 Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Wed, 28 Aug 2024 16:18:08 +0300
Subject: [PATCH 04/24] feat(CI) - cover corner case for the PR from forked
 repo

---
 scripts/travis_rosetta_checker.sh | 13 ++++++++++++-
 scripts/travis_rpc_checker.sh     | 13 ++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/scripts/travis_rosetta_checker.sh b/scripts/travis_rosetta_checker.sh
index a2a20bc1a1..a2cce624ad 100755
--- a/scripts/travis_rosetta_checker.sh
+++ b/scripts/travis_rosetta_checker.sh
@@ -3,8 +3,16 @@ set -e
 
 TEST_REPO_BRANCH=${TEST_REPO_BRANCH:-master}
 # handle for the Travis build run:
+# * uses TRAVIS_PULL_REQUEST_SLUG if PR is done from fork
 # * uses TRAVIS_PULL_REQUEST_BRANCH for RP branch
 # * uses TRAVIS_BRANCH for simple branch builds
+if [[ -z ${TRAVIS_PULL_REQUEST_SLUG} ]]; then
+    MAIN_REPO_ORG='harmony-one'
+else
+    MAIN_REPO_ORG=${TRAVIS_PULL_REQUEST_SLUG%/*}
+    echo "[WARN] - working on the fork - ${MAIN_REPO_ORG}"
+fi
+
 MAIN_REPO_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}
 # handle for the local run, covers:
 # * branch exist on remote - will use it in the tests
@@ -32,6 +40,9 @@ git fetch origin "${TEST_REPO_BRANCH}"
 git checkout "${TEST_REPO_BRANCH}"
 git pull --rebase=true
 cd localnet
-docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" -t harmonyone/localnet-test .
+# we don't care about build logs - if wonna debug it - go to harmony-test repo
+#  and run local debug
+docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" \
+    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test . > /dev/null 2>&1
 # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo
 docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -r
diff --git a/scripts/travis_rpc_checker.sh b/scripts/travis_rpc_checker.sh
index 2d9bd93eba..e24c9fc337 100755
--- a/scripts/travis_rpc_checker.sh
+++ b/scripts/travis_rpc_checker.sh
@@ -3,8 +3,16 @@ set -e
 
 TEST_REPO_BRANCH=${TEST_REPO_BRANCH:-master}
 # handle for the Travis build run:
+# * uses TRAVIS_PULL_REQUEST_SLUG if PR is done from fork
 # * uses TRAVIS_PULL_REQUEST_BRANCH for RP branch
 # * uses TRAVIS_BRANCH for simple branch builds
+if [[ -z ${TRAVIS_PULL_REQUEST_SLUG} ]]; then
+    MAIN_REPO_ORG='harmony-one'
+else
+    MAIN_REPO_ORG=${TRAVIS_PULL_REQUEST_SLUG%/*}
+    echo "[WARN] - working on the fork - ${MAIN_REPO_ORG}"
+fi
+
 MAIN_REPO_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}
 # handle for the local run, covers:
 # * branch exist on remote - will use it in the tests
@@ -32,6 +40,9 @@ git fetch origin "${TEST_REPO_BRANCH}"
 git checkout "${TEST_REPO_BRANCH}"
 git pull --rebase=true
 cd localnet
-docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" -t harmonyone/localnet-test .
+# we don't care about build logs - if wonna debug it - go to harmony-test repo 
+#  and run local debug
+docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" \
+    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test . > /dev/null 2>&1
 # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo
 docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -n

From f90304a7634c4a5fe82921e4dfdc0c522b2963f2 Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Fri, 30 Aug 2024 14:56:22 +0300
Subject: [PATCH 05/24] chore(CI) - tell Docker to silence it output to minimal
 or it will break our CI

---
 scripts/travis_rosetta_checker.sh | 6 ++----
 scripts/travis_rpc_checker.sh     | 6 ++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/scripts/travis_rosetta_checker.sh b/scripts/travis_rosetta_checker.sh
index a2cce624ad..868c673ce2 100755
--- a/scripts/travis_rosetta_checker.sh
+++ b/scripts/travis_rosetta_checker.sh
@@ -40,9 +40,7 @@ git fetch origin "${TEST_REPO_BRANCH}"
 git checkout "${TEST_REPO_BRANCH}"
 git pull --rebase=true
 cd localnet
-# we don't care about build logs - if wonna debug it - go to harmony-test repo
-#  and run local debug
-docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" \
-    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test . > /dev/null 2>&1
+docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \
+    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test .
 # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo
 docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -r
diff --git a/scripts/travis_rpc_checker.sh b/scripts/travis_rpc_checker.sh
index e24c9fc337..20e470120d 100755
--- a/scripts/travis_rpc_checker.sh
+++ b/scripts/travis_rpc_checker.sh
@@ -40,9 +40,7 @@ git fetch origin "${TEST_REPO_BRANCH}"
 git checkout "${TEST_REPO_BRANCH}"
 git pull --rebase=true
 cd localnet
-# we don't care about build logs - if wonna debug it - go to harmony-test repo 
-#  and run local debug
-docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" \
-    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test . > /dev/null 2>&1
+docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \
+    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test .
 # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo
 docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -n

From 3a2349cf969e6f529b2bdbbd0b4f61a1b3b52434 Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Thu, 5 Sep 2024 14:20:17 +0300
Subject: [PATCH 06/24] fix(CI) - move harmony folder to the GOPATH instead of
 default folder

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 1f58faf9e5..aa3c5ced23 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,6 +34,8 @@ install:
   - make go-get
   - go install golang.org/x/tools/cmd/goimports@latest
   - go install github.com/harmony-ek/gencodec@latest
+  - echo "[WARN] - workaround for the GOPATH:"
+  - mv /home/travis/build/harmony-one/harmony $GOPATH/src/github.com/harmony-one/
 script:
   - ${TEST}
 after_success:

From 647d20b65352ff50f84b84096d7e6ade22b5c861 Mon Sep 17 00:00:00 2001
From: "Nita Neou (Soph)" <soph@harmony.one>
Date: Tue, 10 Sep 2024 13:44:20 +0700
Subject: [PATCH 07/24] run stable localnet with external node

---
 Makefile                                        | 8 +++++++-
 test/configs/local-resharding-with-external.txt | 4 ++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 4dbcf61cde..ce00f80d08 100644
--- a/Makefile
+++ b/Makefile
@@ -79,7 +79,13 @@ debug-kill:
 	bash ./test/kill_node.sh
 
 debug-ext:
-	bash ./test/debug-external.sh
+	# update localnet block per epoch to ensure a stable localnet
+	sed -i 's/localnetBlocksPerEpoch\s*=\s*[0-9]*/localnetBlocksPerEpoch = 64/' internal/configs/sharding/localnet.go
+	sed -i 's/localnetBlocksPerEpochV2\s*=\s*[0-9]*/localnetBlocksPerEpochV2 = 64/' internal/configs/sharding/localnet.go
+	bash ./test/debug-external.sh &
+	echo sleep 10s before creating the external validator
+	sleep 10
+	bash ./test/build-localnet-validator.sh
 
 clean:
 	rm -rf ./tmp_log*
diff --git a/test/configs/local-resharding-with-external.txt b/test/configs/local-resharding-with-external.txt
index 318a08b9bc..bb12da4c49 100644
--- a/test/configs/local-resharding-with-external.txt
+++ b/test/configs/local-resharding-with-external.txt
@@ -13,7 +13,7 @@
 # fn node
 127.0.0.1 9050 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key
 127.0.0.1 9052 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key
-#127.0.0.1 9054 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key
+# 127.0.0.1 9054 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key
 # explorer node
 127.0.0.1 9098 explorer null 0
 
@@ -32,6 +32,6 @@
 # fn node
 127.0.0.1 9150 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key
 127.0.0.1 9152 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key
-#127.0.0.1 9154 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key
+# 127.0.0.1 9154 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key
 # explorer node
 127.0.0.1 9096 explorer null 1

From 952814aa46e4adfa98ec594ecb2a33e8bb745fc4 Mon Sep 17 00:00:00 2001
From: "Nita Neou (Soph)" <soph@harmony.one>
Date: Tue, 10 Sep 2024 18:12:05 +0700
Subject: [PATCH 08/24] improve comment detection in deploy.sh

---
 test/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/deploy.sh b/test/deploy.sh
index a87514b689..470a4b4f13 100755
--- a/test/deploy.sh
+++ b/test/deploy.sh
@@ -83,7 +83,7 @@ function launch_localnet() {
     # Read config for i-th node form config file
     IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}"
     args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false")
-    if [[ -z "$ip" || -z "$port" || "$ip" == "#" ]]; then
+    if [[ -z "$ip" || -z "$port" || "${ip:0:1}" == "#" ]]; then
       echo "skip empty line or node or comment"
       continue
     fi

From 6d9b09da8cffa7e5f24bc9705682ed38f4e580a3 Mon Sep 17 00:00:00 2001
From: Gheis Mohammadi <Gheis.Mohammadi@gmail.com>
Date: Wed, 11 Sep 2024 11:01:15 +0800
Subject: [PATCH 09/24] Add RPC for Boot Nodes (#4735)

* initial commit for boot rpc feature
* add start rpc to kick off boot node rpc server
* fix boot node tests, goimports
* remove unused configs and functions from node/boot, cleanup boot metrics, improve boot node test
* fix the issue of get default harmony config for boot node
* set boot node version, fix dependencies name
* goimports, fix boot format
* remove downloader from hmy_boot
* remove tracer from hmy_boot
* remove unused RPCs from hmy_boot
* remove unused functions from hmy_boot
* fix the issue of nil config for new node/boot
* fix getBootAPIs config issue, update boot log tag
* fix boot node invalid address issue for local net
* fix version flag issue, move version meta data to main
* fix boot version info
* disable boot rpc
* disable Start HHTP and WS endpoints to find the build issue
* disable boot apis
* remove RpcMethodFilter for boot servers
* remove StartServers for boot servers
* remove StartServers for boot servers
* enable RpcMethodFilter
* check build issue by uncommenting a part of code
* uncomment startServers for boot host
* add http and ws ports flags for boot node RPCs
* enable http server for boot
* enable ws server for boot
* remove unused functions for boot services, add boot node descriptions, improve boot node comments
* update HTTP modules and ws modules array for boot rpc
* todo for enable services
* goimports for cmd/main
* update version year for boot node rpc
* remove unnecessary configs from boot node RPC configs
* remove unnecessary configs from boot node RPC configs
* remove group configs from boot node RPC configs
* fix the boot node RPC config test
* remove a few more unnecessary configs from boot node RPC configs
* remove shard configs from boot RPC configs
* remove network from boot config
* remove auth ports from boot api
* remove transaction error from bootnode api
* remove extra fields from boot node tests
* fix boot node test
* remove a few extra functions and tests from internal config for bootn ode
* separate structure for boot node meta data
* refactor peer info in boot api
* set peer id for node configs in boot node api
* remove shard-id from boot node metadata, add network to boot node meta data
* fix goimports
---
 cmd/bootnode/main.go                          |  51 ++++-
 cmd/{harmony => config}/config.go             | 131 +++++++++++-
 cmd/{harmony => config}/config_migrations.go  |   6 +-
 .../config_migrations_test.go                 |   6 +-
 cmd/{harmony => config}/config_test.go        |  14 +-
 cmd/{harmony => config}/default.go            |  49 +++--
 cmd/{harmony => config}/dumpdb.go             |   2 +-
 cmd/{harmony => config}/flags.go              |  18 +-
 cmd/{harmony => config}/flags_test.go         |  20 +-
 cmd/{harmony => config}/inspectdb.go          |   2 +-
 cmd/{harmony => config}/version.go            |  24 +--
 cmd/harmony/main.go                           | 158 +++------------
 consensus/consensus_block_proposing.go        |   2 +-
 core_test/shardchain_test.go                  |   2 +-
 hmy/blockchain.go                             |   2 +-
 hmy/hmy.go                                    |   2 +-
 hmy/net.go                                    |   2 +-
 hmy/staking.go                                |   2 +-
 hmy_boot/hmy_boot.go                          |  64 ++++++
 hmy_boot/net.go                               |  22 ++
 internal/configs/bootnode/bootnode.go         | 186 +++++++++++++++++
 internal/configs/bootnode/bootnode_test.go    |  71 +++++++
 internal/configs/bootnode/config.go           | 191 ++++++++++++++++++
 internal/registry/registry.go                 |   2 +-
 node/boot/README.md                           |   5 +
 node/boot/api.go                              | 138 +++++++++++++
 node/boot/bootnode.go                         |  94 +++++++++
 node/boot/metrics.go                          |  32 +++
 node/boot/node_test.go                        |  33 +++
 node/boot/service_setup.go                    |  20 ++
 node/{ => harmony}/README.md                  |   0
 node/{ => harmony}/addresses.go               |   0
 node/{ => harmony}/api.go                     |   6 +-
 node/{ => harmony}/double_signing.go          |   0
 node/{ => harmony}/metrics.go                 |   0
 node/{ => harmony}/node.go                    |   2 +-
 node/{ => harmony}/node.md                    |   0
 node/{ => harmony}/node_cross_link.go         |   0
 node/{ => harmony}/node_cross_shard.go        |   0
 node/{ => harmony}/node_explorer.go           |   0
 node/{ => harmony}/node_handler.go            |   0
 node/{ => harmony}/node_handler_test.go       |   0
 node/{ => harmony}/node_newblock.go           |   0
 node/{ => harmony}/node_newblock_test.go      |   0
 node/{ => harmony}/node_syncing.go            |   0
 node/{ => harmony}/node_test.go               |   0
 node/{ => harmony}/service_setup.go           |   0
 node/{ => harmony}/worker/types.go            |   0
 node/{ => harmony}/worker/worker.go           |   0
 node/{ => harmony}/worker/worker_test.go      |   0
 rosetta/common/operations.go                  |   2 +-
 rosetta/services/account.go                   |   2 +-
 rosetta/services/block.go                     |   2 +-
 rosetta/services/call_service.go              |   2 +-
 rosetta/services/construction_check.go        |   2 +-
 rosetta/services/network.go                   |   2 +-
 rosetta/services/network_test.go              |   2 +-
 rosetta/services/tx_operation.go              |   2 +-
 rpc/boot/boot.go                              |  58 ++++++
 rpc/boot/common/types.go                      |  40 ++++
 rpc/boot/error.go                             |  18 ++
 rpc/boot/rpc.go                               | 167 +++++++++++++++
 rpc/boot/types.go                             |  27 +++
 rpc/{ => harmony}/blockchain.go               |   8 +-
 rpc/{ => harmony}/common/block.go             |   0
 rpc/{ => harmony}/common/hacks.go             |   0
 rpc/{ => harmony}/common/types.go             |   0
 rpc/{ => harmony}/contract.go                 |   0
 rpc/{ => harmony}/error.go                    |   0
 rpc/{ => harmony}/eth/block.go                |   2 +-
 rpc/{ => harmony}/eth/rpc.go                  |   0
 rpc/{ => harmony}/eth/types.go                |   0
 rpc/{ => harmony}/filters/api.go              |   4 +-
 rpc/{ => harmony}/filters/filter.go           |   0
 rpc/{ => harmony}/filters/filter_criteria.go  |   0
 rpc/{ => harmony}/filters/filter_system.go    |   0
 rpc/{ => harmony}/harmony.go                  |   0
 rpc/{ => harmony}/metrics.go                  |   0
 rpc/{ => harmony}/net.go                      |   0
 rpc/{ => harmony}/pool.go                     |   6 +-
 rpc/{ => harmony}/preimages.go                |   0
 rpc/{ => harmony}/private_debug.go            |   0
 rpc/{ => harmony}/public_debug.go             |   0
 rpc/{ => harmony}/rpc.go                      |   6 +-
 rpc/{ => harmony}/staking.go                  |   0
 rpc/{ => harmony}/tracer.go                   |   0
 rpc/{ => harmony}/tracerParity.go             |   0
 rpc/{ => harmony}/transaction.go              |   6 +-
 rpc/{ => harmony}/types.go                    |   2 +-
 rpc/{ => harmony}/types_test.go               |   0
 rpc/{ => harmony}/utils/utils.go              |   0
 rpc/{ => harmony}/v1/block.go                 |   2 +-
 rpc/{ => harmony}/v1/legacy.go                |   0
 rpc/{ => harmony}/v1/types.go                 |   0
 rpc/{ => harmony}/v2/block.go                 |   2 +-
 rpc/{ => harmony}/v2/legacy.go                |   0
 rpc/{ => harmony}/v2/types.go                 |   0
 rpc/{ => harmony}/web3.go                     |   0
 test/chain/main.go                            |   2 +-
 test/deploy.sh                                |   2 +-
 test/helpers/transaction.go                   |   2 +-
 101 files changed, 1478 insertions(+), 251 deletions(-)
 rename cmd/{harmony => config}/config.go (68%)
 rename cmd/{harmony => config}/config_migrations.go (99%)
 rename cmd/{harmony => config}/config_migrations_test.go (98%)
 rename cmd/{harmony => config}/config_test.go (93%)
 rename cmd/{harmony => config}/default.go (89%)
 rename cmd/{harmony => config}/dumpdb.go (99%)
 rename cmd/{harmony => config}/flags.go (99%)
 rename cmd/{harmony => config}/flags_test.go (99%)
 rename cmd/{harmony => config}/inspectdb.go (98%)
 rename cmd/{harmony => config}/version.go (62%)
 create mode 100644 hmy_boot/hmy_boot.go
 create mode 100644 hmy_boot/net.go
 create mode 100644 internal/configs/bootnode/bootnode.go
 create mode 100644 internal/configs/bootnode/bootnode_test.go
 create mode 100644 internal/configs/bootnode/config.go
 create mode 100644 node/boot/README.md
 create mode 100644 node/boot/api.go
 create mode 100644 node/boot/bootnode.go
 create mode 100644 node/boot/metrics.go
 create mode 100644 node/boot/node_test.go
 create mode 100644 node/boot/service_setup.go
 rename node/{ => harmony}/README.md (100%)
 rename node/{ => harmony}/addresses.go (100%)
 rename node/{ => harmony}/api.go (97%)
 rename node/{ => harmony}/double_signing.go (100%)
 rename node/{ => harmony}/metrics.go (100%)
 rename node/{ => harmony}/node.go (99%)
 rename node/{ => harmony}/node.md (100%)
 rename node/{ => harmony}/node_cross_link.go (100%)
 rename node/{ => harmony}/node_cross_shard.go (100%)
 rename node/{ => harmony}/node_explorer.go (100%)
 rename node/{ => harmony}/node_handler.go (100%)
 rename node/{ => harmony}/node_handler_test.go (100%)
 rename node/{ => harmony}/node_newblock.go (100%)
 rename node/{ => harmony}/node_newblock_test.go (100%)
 rename node/{ => harmony}/node_syncing.go (100%)
 rename node/{ => harmony}/node_test.go (100%)
 rename node/{ => harmony}/service_setup.go (100%)
 rename node/{ => harmony}/worker/types.go (100%)
 rename node/{ => harmony}/worker/worker.go (100%)
 rename node/{ => harmony}/worker/worker_test.go (100%)
 create mode 100644 rpc/boot/boot.go
 create mode 100644 rpc/boot/common/types.go
 create mode 100644 rpc/boot/error.go
 create mode 100644 rpc/boot/rpc.go
 create mode 100644 rpc/boot/types.go
 rename rpc/{ => harmony}/blockchain.go (99%)
 rename rpc/{ => harmony}/common/block.go (100%)
 rename rpc/{ => harmony}/common/hacks.go (100%)
 rename rpc/{ => harmony}/common/types.go (100%)
 rename rpc/{ => harmony}/contract.go (100%)
 rename rpc/{ => harmony}/error.go (100%)
 rename rpc/{ => harmony}/eth/block.go (96%)
 rename rpc/{ => harmony}/eth/rpc.go (100%)
 rename rpc/{ => harmony}/eth/types.go (100%)
 rename rpc/{ => harmony}/filters/api.go (99%)
 rename rpc/{ => harmony}/filters/filter.go (100%)
 rename rpc/{ => harmony}/filters/filter_criteria.go (100%)
 rename rpc/{ => harmony}/filters/filter_system.go (100%)
 rename rpc/{ => harmony}/harmony.go (100%)
 rename rpc/{ => harmony}/metrics.go (100%)
 rename rpc/{ => harmony}/net.go (100%)
 rename rpc/{ => harmony}/pool.go (98%)
 rename rpc/{ => harmony}/preimages.go (100%)
 rename rpc/{ => harmony}/private_debug.go (100%)
 rename rpc/{ => harmony}/public_debug.go (100%)
 rename rpc/{ => harmony}/rpc.go (98%)
 rename rpc/{ => harmony}/staking.go (100%)
 rename rpc/{ => harmony}/tracer.go (100%)
 rename rpc/{ => harmony}/tracerParity.go (100%)
 rename rpc/{ => harmony}/transaction.go (99%)
 rename rpc/{ => harmony}/types.go (100%)
 rename rpc/{ => harmony}/types_test.go (100%)
 rename rpc/{ => harmony}/utils/utils.go (100%)
 rename rpc/{ => harmony}/v1/block.go (96%)
 rename rpc/{ => harmony}/v1/legacy.go (100%)
 rename rpc/{ => harmony}/v1/types.go (100%)
 rename rpc/{ => harmony}/v2/block.go (96%)
 rename rpc/{ => harmony}/v2/legacy.go (100%)
 rename rpc/{ => harmony}/v2/types.go (100%)
 rename rpc/{ => harmony}/web3.go (100%)

diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go
index b438b47971..5926ddc6b1 100644
--- a/cmd/bootnode/main.go
+++ b/cmd/bootnode/main.go
@@ -12,7 +12,11 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
+	//cmdharmony "github.com/harmony-one/harmony/cmd/harmony"
+	harmonyConfigs "github.com/harmony-one/harmony/cmd/config"
+	nodeConfigs "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
+	bootnode "github.com/harmony-one/harmony/node/boot"
 	"github.com/harmony-one/harmony/p2p"
 	net "github.com/libp2p/go-libp2p/core/network"
 	ma "github.com/multiformats/go-multiaddr"
@@ -87,8 +91,7 @@ var (
 )
 
 func printVersion(me string) {
-	fmt.Fprintf(os.Stderr, "Harmony (C) 2019. %v, version %v-%v (%v %v)\n", path.Base(me), version, commit, builtBy, builtAt)
-	os.Exit(0)
+	fmt.Fprintf(os.Stderr, "Harmony (C) 2024. %v, version %v-%v (%v %v)\n", path.Base(me), version, commit, builtBy, builtAt)
 }
 
 func main() {
@@ -97,6 +100,8 @@ func main() {
 
 	ip := flag.String("ip", "127.0.0.1", "IP of the node")
 	port := flag.String("port", "9876", "port of the node.")
+	httpPort := flag.Int("rpc_http_port", 9500, "port of the rpc http")
+	wsPort := flag.Int("rpc_ws_port", 9800, "port of the rpc ws")
 	console := flag.Bool("console_only", false, "Output to console only")
 	logFolder := flag.String("log_folder", "latest", "the folder collecting the logs of this execution")
 	logMaxSize := flag.Int("log_max_size", 100, "the max size in megabytes of the log file before it gets rotated")
@@ -113,7 +118,7 @@ func main() {
 	muxer := flag.String("muxer", "mplex, yamux", "protocol muxer to mux per-protocol streams (mplex, yamux)")
 	userAgent := flag.String("user_agent", defUserAgent, "explicitly set the user-agent, so we can differentiate from other Go libp2p users")
 	noRelay := flag.Bool("no_relay", true, "no relay services, direct connections between peers only")
-
+	networkType := flag.String("network", "mainnet", "network type (mainnet, testnet, pangaea, partner, stressnet, devnet, localnet)")
 	pprof := flag.Bool("pprof", false, "enabled pprof")
 	pprofAddr := flag.String("pprof.addr", "127.0.0.1:6060", "http pprof address")
 	//keyFile := flag.String("pprof.profile.names", "", "the private key file of the bootnode")
@@ -124,12 +129,13 @@ func main() {
 
 	if *versionFlag {
 		printVersion(os.Args[0])
+		os.Exit(0)
 	}
 
 	// Logging setup
 	utils.SetLogContext(*port, *ip)
 	utils.SetLogVerbosity(log.Lvl(*verbosity))
-	if *console != true {
+	if !*console {
 		utils.AddLogFile(fmt.Sprintf("%v/bootnode-%v-%v.log", *logFolder, *ip, *port), *logMaxSize, *logRotateCount, *logRotateMaxAge)
 	}
 
@@ -165,8 +171,19 @@ func main() {
 		fmt.Sprintf("/ip4/%s/tcp/%s/p2p/%s\n", *ip, *port, host.GetID().String()),
 	)
 
+	nt := nodeConfigs.NetworkType(*networkType)
+	nodeConfigs.SetNetworkType(nt)
+	harmonyConfigs.VersionMetaData = append(harmonyConfigs.VersionMetaData, path.Base(os.Args[0]), version, commit, builtBy, builtAt)
+	nodeConfigs.SetVersion(harmonyConfigs.GetHarmonyVersion())
+	nodeConfigs.SetPeerID(host.GetID())
+	hc := harmonyConfigs.GetDefaultConfigCopy()
+
 	host.Start()
 
+	utils.Logger().Info().
+		Interface("network", nt).
+		Msg("boot node host started")
+
 	if *logConn {
 		host.GetP2PHost().Network().Notify(NewConnLogger(utils.GetLogInstance()))
 	}
@@ -176,5 +193,31 @@ func main() {
 		http.ListenAndServe(*pprofAddr, nil)
 	}
 
+	currentBootNode := bootnode.New(host, &hc)
+	rpcConfigs := currentBootNode.GetRPCServerConfig()
+	rpcConfigs.HTTPPort = *httpPort
+	rpcConfigs.WSPort = *wsPort
+
+	// TODO: enable boot services
+	/*
+		if err := currentBootNode.StartServices(); err != nil {
+			fmt.Fprint(os.Stderr, err.Error())
+			os.Exit(-1)
+		}
+	*/
+
+	if err := currentBootNode.StartRPC(); err != nil {
+		utils.Logger().Error().
+			Err(err).
+			Msg("StartRPC failed")
+	}
+
+	utils.Logger().Info().
+		Interface("network", nt).
+		Interface("ip", currentBootNode.SelfPeer.IP).
+		Interface("port", currentBootNode.SelfPeer.Port).
+		Interface("PeerID", currentBootNode.SelfPeer.PeerID).
+		Msg("boot node RPC started")
+
 	select {}
 }
diff --git a/cmd/harmony/config.go b/cmd/config/config.go
similarity index 68%
rename from cmd/harmony/config.go
rename to cmd/config/config.go
index 0d2632d8aa..5cb14e4d77 100644
--- a/cmd/harmony/config.go
+++ b/cmd/config/config.go
@@ -1,8 +1,9 @@
-package main
+package config
 
 import (
 	"errors"
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 	"time"
@@ -19,7 +20,7 @@ func validateHarmonyConfig(config harmonyconfig.HarmonyConfig) error {
 	var accepts []string
 
 	nodeType := config.General.NodeType
-	accepts = []string{nodeTypeValidator, nodeTypeExplorer}
+	accepts = []string{NodeTypeValidator, NodeTypeExplorer}
 	if err := checkStringAccepted("--run", nodeType, accepts); err != nil {
 		return err
 	}
@@ -42,7 +43,7 @@ func validateHarmonyConfig(config harmonyconfig.HarmonyConfig) error {
 		return err
 	}
 
-	if config.General.NodeType == nodeTypeExplorer && config.General.ShardID < 0 {
+	if config.General.NodeType == NodeTypeExplorer && config.General.ShardID < 0 {
 		return errors.New("flag --run.shard must be specified for explorer node")
 	}
 
@@ -76,7 +77,7 @@ func checkStringAccepted(flag string, val string, accepts []string) error {
 	return fmt.Errorf("unknown arg for %s: %s (%v)", flag, val, acceptsStr)
 }
 
-func getDefaultDNSSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.DnsSync {
+func GetDefaultDNSSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.DnsSync {
 	zone := nodeconfig.GetDefaultDNSZone(nt)
 	port := nodeconfig.GetDefaultDNSPort(nt)
 	dnsSync := harmonyconfig.DnsSync{
@@ -101,7 +102,7 @@ func getDefaultDNSSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.DnsSync {
 	return dnsSync
 }
 
-func getDefaultNetworkConfig(nt nodeconfig.NetworkType) harmonyconfig.NetworkConfig {
+func GetDefaultNetworkConfig(nt nodeconfig.NetworkType) harmonyconfig.NetworkConfig {
 	bn := nodeconfig.GetDefaultBootNodes(nt)
 	return harmonyconfig.NetworkConfig{
 		NetworkType: string(nt),
@@ -130,7 +131,7 @@ func parseNetworkType(nt string) nodeconfig.NetworkType {
 	}
 }
 
-func getDefaultSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.SyncConfig {
+func GetDefaultSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.SyncConfig {
 	switch nt {
 	case nodeconfig.Mainnet:
 		return defaultMainnetSyncConfig
@@ -145,7 +146,7 @@ func getDefaultSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.SyncConfig {
 	}
 }
 
-func getDefaultCacheConfig(nt nodeconfig.NetworkType) harmonyconfig.CacheConfig {
+func GetDefaultCacheConfig(nt nodeconfig.NetworkType) harmonyconfig.CacheConfig {
 	cacheConfig := harmonyconfig.CacheConfig{
 		Disabled:        defaultCacheConfig.Disabled,
 		TrieNodeLimit:   defaultCacheConfig.TrieNodeLimit,
@@ -203,7 +204,7 @@ var updateConfigCmd = &cobra.Command{
 
 func dumpConfig(cmd *cobra.Command, args []string) {
 	nt := getNetworkType(cmd)
-	config := getDefaultHmyConfigCopy(nt)
+	config := GetDefaultHmyConfigCopy(nt)
 
 	if err := writeHarmonyConfigToFile(config, args[0]); err != nil {
 		fmt.Println(err)
@@ -296,3 +297,117 @@ func writeHarmonyConfigToFile(config harmonyconfig.HarmonyConfig, file string) e
 	}
 	return os.WriteFile(file, b, 0644)
 }
+
+var configFlag = cli.StringFlag{
+	Name:      "config",
+	Usage:     "load node config from the config toml file.",
+	Shorthand: "c",
+	DefValue:  "",
+}
+
+func applyRootFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
+	// Misc flags shall be applied first since legacy ip / port is overwritten
+	// by new ip / port flags
+	applyLegacyMiscFlags(cmd, config)
+	applyGeneralFlags(cmd, config)
+	applyNetworkFlags(cmd, config)
+	applyDNSSyncFlags(cmd, config)
+	applyP2PFlags(cmd, config)
+	applyHTTPFlags(cmd, config)
+	applyWSFlags(cmd, config)
+	applyRPCOptFlags(cmd, config)
+	applyBLSFlags(cmd, config)
+	applyConsensusFlags(cmd, config)
+	applyTxPoolFlags(cmd, config)
+	applyPprofFlags(cmd, config)
+	applyLogFlags(cmd, config)
+	applySysFlags(cmd, config)
+	applyDevnetFlags(cmd, config)
+	applyRevertFlags(cmd, config)
+	applyPreimageFlags(cmd, config)
+	applyPrometheusFlags(cmd, config)
+	applySyncFlags(cmd, config)
+	applyShardDataFlags(cmd, config)
+	applyGPOFlags(cmd, config)
+	applyCacheFlags(cmd, config)
+}
+
+func registerRootCmdFlags(rootCmd *cobra.Command) error {
+	flags := getRootFlags()
+
+	return cli.RegisterFlags(rootCmd, flags)
+}
+
+func GetHarmonyConfig(cmd *cobra.Command) (harmonyconfig.HarmonyConfig, error) {
+	var (
+		config         harmonyconfig.HarmonyConfig
+		err            error
+		migratedFrom   string
+		configFile     string
+		isUsingDefault bool
+	)
+	if cli.IsFlagChanged(cmd, configFlag) {
+		configFile = cli.GetStringFlagValue(cmd, configFlag)
+		config, migratedFrom, err = loadHarmonyConfig(configFile)
+	} else {
+		nt := getNetworkType(cmd)
+		config = GetDefaultHmyConfigCopy(nt)
+		isUsingDefault = true
+	}
+	if err != nil {
+		return harmonyconfig.HarmonyConfig{}, err
+	}
+	if migratedFrom != defaultConfig.Version && !isUsingDefault {
+		fmt.Printf("Old config version detected %s\n",
+			migratedFrom)
+		stat, _ := os.Stdin.Stat()
+		// Ask to update if only using terminal
+		if stat.Mode()&os.ModeCharDevice != 0 {
+			if promptConfigUpdate() {
+				err := updateConfigFile(configFile)
+				if err != nil {
+					fmt.Printf("Could not update config - %s", err.Error())
+					fmt.Println("Update config manually with `./harmony config update [config_file]`")
+				}
+			}
+
+		} else {
+			fmt.Println("Update saved config with `./harmony config update [config_file]`")
+		}
+	}
+
+	applyRootFlags(cmd, &config)
+
+	if err := validateHarmonyConfig(config); err != nil {
+		return harmonyconfig.HarmonyConfig{}, err
+	}
+	sanityFixHarmonyConfig(&config)
+	return config, nil
+}
+
+func Init(rootCmd *cobra.Command) {
+	rand.Seed(time.Now().UnixNano())
+	cli.SetParseErrorHandle(func(err error) {
+		os.Exit(128) // 128 - invalid command line arguments
+	})
+	configCmd.AddCommand(dumpConfigCmd)
+	configCmd.AddCommand(updateConfigCmd)
+	rootCmd.AddCommand(configCmd)
+	rootCmd.AddCommand(versionCmd)
+	rootCmd.AddCommand(dumpConfigLegacyCmd)
+	rootCmd.AddCommand(dumpDBCmd)
+	rootCmd.AddCommand(inspectDBCmd)
+
+	if err := registerRootCmdFlags(rootCmd); err != nil {
+		os.Exit(2)
+	}
+	if err := registerDumpConfigFlags(); err != nil {
+		os.Exit(2)
+	}
+	if err := registerDumpDBFlags(); err != nil {
+		os.Exit(2)
+	}
+	if err := registerInspectionFlags(); err != nil {
+		os.Exit(2)
+	}
+}
diff --git a/cmd/harmony/config_migrations.go b/cmd/config/config_migrations.go
similarity index 99%
rename from cmd/harmony/config_migrations.go
rename to cmd/config/config_migrations.go
index c8a59c722d..69e2b4d236 100644
--- a/cmd/harmony/config_migrations.go
+++ b/cmd/config/config_migrations.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"errors"
@@ -83,8 +83,8 @@ func init() {
 		ntStr := confTree.Get("Network.NetworkType").(string)
 		nt := parseNetworkType(ntStr)
 
-		defDNSSyncConf := getDefaultDNSSyncConfig(nt)
-		defSyncConfig := getDefaultSyncConfig(nt)
+		defDNSSyncConf := GetDefaultDNSSyncConfig(nt)
+		defSyncConfig := GetDefaultSyncConfig(nt)
 
 		// Legacy conf missing fields
 		if confTree.Get("Sync") == nil {
diff --git a/cmd/harmony/config_migrations_test.go b/cmd/config/config_migrations_test.go
similarity index 98%
rename from cmd/harmony/config_migrations_test.go
rename to cmd/config/config_migrations_test.go
index e52c7347b2..edceaa42e4 100644
--- a/cmd/harmony/config_migrations_test.go
+++ b/cmd/config/config_migrations_test.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"testing"
@@ -343,8 +343,8 @@ Version = "1.0.4"
 )
 
 func Test_migrateConf(t *testing.T) {
-	defConf := getDefaultHmyConfigCopy(nodeconfig.Mainnet)
-	legacyDefConf := getDefaultHmyConfigCopy(nodeconfig.Mainnet)
+	defConf := GetDefaultHmyConfigCopy(nodeconfig.Mainnet)
+	legacyDefConf := GetDefaultHmyConfigCopy(nodeconfig.Mainnet)
 	// Versions prior to 1.0.3 use different BootNodes
 	legacyDefConf.Network.BootNodes = []string{
 		"/ip4/100.26.90.187/tcp/9874/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv",
diff --git a/cmd/harmony/config_test.go b/cmd/config/config_test.go
similarity index 93%
rename from cmd/harmony/config_test.go
rename to cmd/config/config_test.go
index 2711041de8..302f4b7840 100644
--- a/cmd/harmony/config_test.go
+++ b/cmd/config/config_test.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"fmt"
@@ -16,7 +16,7 @@ import (
 type testCfgOpt func(config *harmonyconfig.HarmonyConfig)
 
 func makeTestConfig(nt nodeconfig.NetworkType, opt testCfgOpt) harmonyconfig.HarmonyConfig {
-	cfg := getDefaultHmyConfigCopy(nt)
+	cfg := GetDefaultHmyConfigCopy(nt)
 	if opt != nil {
 		opt(&cfg)
 	}
@@ -127,7 +127,7 @@ Version = "1.0.4"
 	if err != nil {
 		t.Fatal(err)
 	}
-	defConf := getDefaultHmyConfigCopy(nodeconfig.Mainnet)
+	defConf := GetDefaultHmyConfigCopy(nodeconfig.Mainnet)
 	if config.HTTP.RosettaEnabled {
 		t.Errorf("Expected rosetta http server to be disabled when loading old config")
 	}
@@ -160,13 +160,13 @@ func TestPersistConfig(t *testing.T) {
 		},
 		{
 			config: makeTestConfig("mainnet", func(cfg *harmonyconfig.HarmonyConfig) {
-				consensus := getDefaultConsensusConfigCopy()
+				consensus := GetDefaultConsensusConfigCopy()
 				cfg.Consensus = &consensus
 
-				devnet := getDefaultDevnetConfigCopy()
+				devnet := GetDefaultDevnetConfigCopy()
 				cfg.Devnet = &devnet
 
-				revert := getDefaultRevertConfigCopy()
+				revert := GetDefaultRevertConfigCopy()
 				cfg.Revert = &revert
 
 				webHook := "web hook"
@@ -175,7 +175,7 @@ func TestPersistConfig(t *testing.T) {
 					TPBroadcastInvalidTxn: &trueBool,
 				}
 
-				logCtx := getDefaultLogContextCopy()
+				logCtx := GetDefaultLogContextCopy()
 				cfg.Log.Context = &logCtx
 			}),
 		},
diff --git a/cmd/harmony/default.go b/cmd/config/default.go
similarity index 89%
rename from cmd/harmony/default.go
rename to cmd/config/default.go
index 5d8349d72b..7d6cb0e06f 100644
--- a/cmd/harmony/default.go
+++ b/cmd/config/default.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"time"
@@ -27,7 +27,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
 		DataDir:          "./",
 		TraceEnable:      false,
 	},
-	Network: getDefaultNetworkConfig(defNetworkType),
+	Network: GetDefaultNetworkConfig(defNetworkType),
 	P2P: harmonyconfig.P2pConfig{
 		Port:                     nodeconfig.DefaultP2PPort,
 		IP:                       nodeconfig.DefaultPublicListenIP,
@@ -100,7 +100,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
 		PriceLimit:        harmonyconfig.PriceLimit(core.DefaultTxPoolConfig.PriceLimit),
 		PriceBump:         core.DefaultTxPoolConfig.PriceBump,
 	},
-	Sync: getDefaultSyncConfig(defNetworkType),
+	Sync: GetDefaultSyncConfig(defNetworkType),
 	Pprof: harmonyconfig.PprofConfig{
 		Enabled:            false,
 		ListenAddr:         "127.0.0.1:6060",
@@ -121,7 +121,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
 			Config: true,
 		},
 	},
-	DNSSync: getDefaultDNSSyncConfig(defNetworkType),
+	DNSSync: GetDefaultDNSSyncConfig(defNetworkType),
 	ShardData: harmonyconfig.ShardDataConfig{
 		EnableShardData: false,
 		DiskCount:       8,
@@ -138,7 +138,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
 		LowUsageThreshold: hmy.DefaultGPOConfig.LowUsageThreshold,
 		BlockGasLimit:     hmy.DefaultGPOConfig.BlockGasLimit,
 	},
-	Cache: getDefaultCacheConfig(defNetworkType),
+	Cache: GetDefaultCacheConfig(defNetworkType),
 }
 
 var defaultSysConfig = harmonyconfig.SysConfig{
@@ -291,67 +291,72 @@ var defaultCacheConfig = harmonyconfig.CacheConfig{
 }
 
 const (
-	defaultBroadcastInvalidTx = false
+	DefaultBroadcastInvalidTx = false
 )
 
-func getDefaultHmyConfigCopy(nt nodeconfig.NetworkType) harmonyconfig.HarmonyConfig {
+func GetDefaultConfigCopy() harmonyconfig.HarmonyConfig {
+	config := defaultConfig
+	return config
+}
+
+func GetDefaultHmyConfigCopy(nt nodeconfig.NetworkType) harmonyconfig.HarmonyConfig {
 	config := defaultConfig
 
-	config.Network = getDefaultNetworkConfig(nt)
+	config.Network = GetDefaultNetworkConfig(nt)
 	if nt == nodeconfig.Devnet {
-		devnet := getDefaultDevnetConfigCopy()
+		devnet := GetDefaultDevnetConfigCopy()
 		config.Devnet = &devnet
 	}
-	config.Sync = getDefaultSyncConfig(nt)
-	config.DNSSync = getDefaultDNSSyncConfig(nt)
-	config.Cache = getDefaultCacheConfig(nt)
+	config.Sync = GetDefaultSyncConfig(nt)
+	config.DNSSync = GetDefaultDNSSyncConfig(nt)
+	config.Cache = GetDefaultCacheConfig(nt)
 
 	return config
 }
 
-func getDefaultSysConfigCopy() harmonyconfig.SysConfig {
+func GetDefaultSysConfigCopy() harmonyconfig.SysConfig {
 	config := defaultSysConfig
 	return config
 }
 
-func getDefaultDevnetConfigCopy() harmonyconfig.DevnetConfig {
+func GetDefaultDevnetConfigCopy() harmonyconfig.DevnetConfig {
 	config := defaultDevnetConfig
 	return config
 }
 
-func getDefaultRevertConfigCopy() harmonyconfig.RevertConfig {
+func GetDefaultRevertConfigCopy() harmonyconfig.RevertConfig {
 	config := defaultRevertConfig
 	return config
 }
 
-func getDefaultPreimageConfigCopy() harmonyconfig.PreimageConfig {
+func GetDefaultPreimageConfigCopy() harmonyconfig.PreimageConfig {
 	config := defaultPreimageConfig
 	return config
 }
 
-func getDefaultLogContextCopy() harmonyconfig.LogContext {
+func GetDefaultLogContextCopy() harmonyconfig.LogContext {
 	config := defaultLogContext
 	return config
 }
 
-func getDefaultConsensusConfigCopy() harmonyconfig.ConsensusConfig {
+func GetDefaultConsensusConfigCopy() harmonyconfig.ConsensusConfig {
 	config := defaultConsensusConfig
 	return config
 }
 
-func getDefaultPrometheusConfigCopy() harmonyconfig.PrometheusConfig {
+func GetDefaultPrometheusConfigCopy() harmonyconfig.PrometheusConfig {
 	config := defaultPrometheusConfig
 	return config
 }
 
-func getDefaultCacheConfigCopy() harmonyconfig.CacheConfig {
+func GetDefaultCacheConfigCopy() harmonyconfig.CacheConfig {
 	config := defaultCacheConfig
 	return config
 }
 
 const (
-	nodeTypeValidator = "validator"
-	nodeTypeExplorer  = "explorer"
+	NodeTypeValidator = "validator"
+	NodeTypeExplorer  = "explorer"
 )
 
 const (
diff --git a/cmd/harmony/dumpdb.go b/cmd/config/dumpdb.go
similarity index 99%
rename from cmd/harmony/dumpdb.go
rename to cmd/config/dumpdb.go
index fd506ee05b..7e8a87f278 100644
--- a/cmd/harmony/dumpdb.go
+++ b/cmd/config/dumpdb.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"fmt"
diff --git a/cmd/harmony/flags.go b/cmd/config/flags.go
similarity index 99%
rename from cmd/harmony/flags.go
rename to cmd/config/flags.go
index 255520b356..95ab3f8c8e 100644
--- a/cmd/harmony/flags.go
+++ b/cmd/config/flags.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"fmt"
@@ -1215,7 +1215,7 @@ var (
 
 func applyConsensusFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	if config.Consensus == nil && cli.HasFlagsChanged(cmd, consensusValidFlags) {
-		cfg := getDefaultConsensusConfigCopy()
+		cfg := GetDefaultConsensusConfigCopy()
 		config.Consensus = &cfg
 	}
 
@@ -1544,7 +1544,7 @@ func applyLogFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	}
 
 	if cli.HasFlagsChanged(cmd, []cli.Flag{logContextIPFlag, logContextPortFlag}) {
-		ctx := getDefaultLogContextCopy()
+		ctx := GetDefaultLogContextCopy()
 		config.Log.Context = &ctx
 
 		if cli.IsFlagChanged(cmd, logContextIPFlag) {
@@ -1567,7 +1567,7 @@ var (
 
 func applySysFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	if cli.HasFlagsChanged(cmd, sysFlags) || config.Sys == nil {
-		cfg := getDefaultSysConfigCopy()
+		cfg := GetDefaultSysConfigCopy()
 		config.Sys = &cfg
 	}
 
@@ -1617,7 +1617,7 @@ var (
 
 func applyDevnetFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	if cli.HasFlagsChanged(cmd, devnetFlags) && config.Devnet == nil {
-		cfg := getDefaultDevnetConfigCopy()
+		cfg := GetDefaultDevnetConfigCopy()
 		config.Devnet = &cfg
 	}
 
@@ -1692,7 +1692,7 @@ var (
 
 func applyRevertFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	if cli.HasFlagsChanged(cmd, revertFlags) {
-		cfg := getDefaultRevertConfigCopy()
+		cfg := GetDefaultRevertConfigCopy()
 		config.Revert = &cfg
 	}
 
@@ -1750,7 +1750,7 @@ var (
 
 func applyPreimageFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	if cli.HasFlagsChanged(cmd, preimageFlags) {
-		cfg := getDefaultPreimageConfigCopy()
+		cfg := GetDefaultPreimageConfigCopy()
 		config.Preimage = &cfg
 	}
 	if cli.IsFlagChanged(cmd, preimageImportFlag) {
@@ -1795,7 +1795,7 @@ var (
 	legacyTPBroadcastInvalidTxFlag = cli.BoolFlag{
 		Name:       "broadcast_invalid_tx",
 		Usage:      "broadcast invalid transactions to sync pool state",
-		DefValue:   defaultBroadcastInvalidTx,
+		DefValue:   DefaultBroadcastInvalidTx,
 		Deprecated: "use --txpool.broadcast-invalid-tx",
 	}
 )
@@ -1889,7 +1889,7 @@ var (
 
 func applyPrometheusFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	if config.Prometheus == nil {
-		cfg := getDefaultPrometheusConfigCopy()
+		cfg := GetDefaultPrometheusConfigCopy()
 		config.Prometheus = &cfg
 		// enable pushgateway for mainnet nodes by default
 		if config.Network.NetworkType == "mainnet" {
diff --git a/cmd/harmony/flags_test.go b/cmd/config/flags_test.go
similarity index 99%
rename from cmd/harmony/flags_test.go
rename to cmd/config/flags_test.go
index 980ddbafd9..abcc74b65e 100644
--- a/cmd/harmony/flags_test.go
+++ b/cmd/config/flags_test.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"fmt"
@@ -314,7 +314,7 @@ func TestNetworkFlags(t *testing.T) {
 					NetworkType: defNetworkType,
 					BootNodes:   nodeconfig.GetDefaultBootNodes(defNetworkType),
 				},
-				DNSSync: getDefaultDNSSyncConfig(defNetworkType)},
+				DNSSync: GetDefaultDNSSyncConfig(defNetworkType)},
 		},
 		{
 			args: []string{"-n", "stn"},
@@ -323,7 +323,7 @@ func TestNetworkFlags(t *testing.T) {
 					NetworkType: nodeconfig.Stressnet,
 					BootNodes:   nodeconfig.GetDefaultBootNodes(nodeconfig.Stressnet),
 				},
-				DNSSync: getDefaultDNSSyncConfig(nodeconfig.Stressnet),
+				DNSSync: GetDefaultDNSSyncConfig(nodeconfig.Stressnet),
 			},
 		},
 		{
@@ -366,8 +366,8 @@ func TestNetworkFlags(t *testing.T) {
 		ts := newFlagTestSuite(t, neededFlags, func(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 			// This is the network related logic in function getharmonyconfig.HarmonyConfig
 			nt := getNetworkType(cmd)
-			config.Network = getDefaultNetworkConfig(nt)
-			config.DNSSync = getDefaultDNSSyncConfig(nt)
+			config.Network = GetDefaultNetworkConfig(nt)
+			config.DNSSync = GetDefaultDNSSyncConfig(nt)
 			applyNetworkFlags(cmd, config)
 			applyDNSSyncFlags(cmd, config)
 		})
@@ -1826,18 +1826,18 @@ func TestDNSSyncFlags(t *testing.T) {
 		{
 			args:      []string{},
 			network:   "mainnet",
-			expConfig: getDefaultDNSSyncConfig(nodeconfig.Mainnet),
+			expConfig: GetDefaultDNSSyncConfig(nodeconfig.Mainnet),
 		},
 		{
 			args:      []string{"--sync.legacy.server", "--sync.legacy.client"},
 			network:   "mainnet",
-			expConfig: getDefaultDNSSyncConfig(nodeconfig.Mainnet),
+			expConfig: GetDefaultDNSSyncConfig(nodeconfig.Mainnet),
 		},
 		{
 			args:    []string{"--sync.legacy.server", "--sync.legacy.client"},
 			network: "testnet",
 			expConfig: func() harmonyconfig.DnsSync {
-				cfg := getDefaultDNSSyncConfig(nodeconfig.Mainnet)
+				cfg := GetDefaultDNSSyncConfig(nodeconfig.Mainnet)
 				cfg.Client = true
 				cfg.Server = true
 				return cfg
@@ -1846,7 +1846,7 @@ func TestDNSSyncFlags(t *testing.T) {
 		{
 			args:      []string{"--dns.server", "--dns.client"},
 			network:   "mainnet",
-			expConfig: getDefaultDNSSyncConfig(nodeconfig.Mainnet),
+			expConfig: GetDefaultDNSSyncConfig(nodeconfig.Mainnet),
 		},
 	}
 
@@ -1976,7 +1976,7 @@ type flagTestSuite struct {
 func newFlagTestSuite(t *testing.T, flags []cli.Flag, applyFlags func(*cobra.Command, *harmonyconfig.HarmonyConfig)) *flagTestSuite {
 	cli.SetParseErrorHandle(func(err error) { t.Fatal(err) })
 
-	ts := &flagTestSuite{hc: getDefaultHmyConfigCopy(defNetworkType)}
+	ts := &flagTestSuite{hc: GetDefaultHmyConfigCopy(defNetworkType)}
 	ts.cmd = makeTestCommand(func(cmd *cobra.Command, args []string) {
 		applyFlags(cmd, &ts.hc)
 	})
diff --git a/cmd/harmony/inspectdb.go b/cmd/config/inspectdb.go
similarity index 98%
rename from cmd/harmony/inspectdb.go
rename to cmd/config/inspectdb.go
index 89f8c28271..f08e7872dc 100644
--- a/cmd/harmony/inspectdb.go
+++ b/cmd/config/inspectdb.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"fmt"
diff --git a/cmd/harmony/version.go b/cmd/config/version.go
similarity index 62%
rename from cmd/harmony/version.go
rename to cmd/config/version.go
index 4f9d3521e6..49b7b1bfbe 100644
--- a/cmd/harmony/version.go
+++ b/cmd/config/version.go
@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"fmt"
@@ -12,13 +12,7 @@ const (
 	versionFormat = "Harmony (C) 2023. %v, version %v-%v (%v %v)"
 )
 
-// Version string variables
-var (
-	version string
-	builtBy string
-	builtAt string
-	commit  string
-)
+var VersionMetaData []interface{}
 
 var versionFlag = cli.BoolFlag{
 	Name:      "version",
@@ -32,15 +26,19 @@ var versionCmd = &cobra.Command{
 	Long:  "print version of the harmony binary",
 	Args:  cobra.NoArgs,
 	Run: func(cmd *cobra.Command, args []string) {
-		printVersion()
+		PrintVersion()
 		os.Exit(0)
 	},
 }
 
-func getHarmonyVersion() string {
-	return fmt.Sprintf(versionFormat, "harmony", version, commit, builtBy, builtAt)
+func VersionFlag() cli.BoolFlag {
+	return versionFlag
+}
+
+func GetHarmonyVersion() string {
+	return fmt.Sprintf(versionFormat, VersionMetaData[:5]...) // "harmony", version, commit, builtBy, builtAt
 }
 
-func printVersion() {
-	fmt.Println(getHarmonyVersion())
+func PrintVersion() {
+	fmt.Println(GetHarmonyVersion())
 }
diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go
index 9629bee0aa..c0c07641f1 100644
--- a/cmd/harmony/main.go
+++ b/cmd/harmony/main.go
@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"math/big"
-	"math/rand"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
@@ -24,6 +23,7 @@ import (
 	"github.com/harmony-one/harmony/api/service/prometheus"
 	"github.com/harmony-one/harmony/api/service/stagedstreamsync"
 	"github.com/harmony-one/harmony/api/service/synchronize"
+	harmonyConfigs "github.com/harmony-one/harmony/cmd/config"
 	"github.com/harmony-one/harmony/common/fdlimit"
 	"github.com/harmony-one/harmony/common/ntp"
 	"github.com/harmony-one/harmony/consensus"
@@ -45,17 +45,25 @@ import (
 	"github.com/harmony-one/harmony/internal/tikv/statedb_cache"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/multibls"
-	"github.com/harmony-one/harmony/node"
+	node "github.com/harmony-one/harmony/node/harmony"
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/p2p"
 	rosetta_common "github.com/harmony-one/harmony/rosetta/common"
-	rpc_common "github.com/harmony-one/harmony/rpc/common"
+	rpc_common "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/webhooks"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
+// Version string variables
+var (
+	version string
+	builtBy string
+	builtAt string
+	commit  string
+)
+
 // Host
 var (
 	myHost          p2p.Host
@@ -87,53 +95,18 @@ Examples usage:
 	Run: runHarmonyNode,
 }
 
-var configFlag = cli.StringFlag{
-	Name:      "config",
-	Usage:     "load node config from the config toml file.",
-	Shorthand: "c",
-	DefValue:  "",
-}
-
 func init() {
-	rand.Seed(time.Now().UnixNano())
-	cli.SetParseErrorHandle(func(err error) {
-		os.Exit(128) // 128 - invalid command line arguments
-	})
-	configCmd.AddCommand(dumpConfigCmd)
-	configCmd.AddCommand(updateConfigCmd)
-	rootCmd.AddCommand(configCmd)
-	rootCmd.AddCommand(versionCmd)
-	rootCmd.AddCommand(dumpConfigLegacyCmd)
-	rootCmd.AddCommand(dumpDBCmd)
-	rootCmd.AddCommand(inspectDBCmd)
-
-	if err := registerRootCmdFlags(); err != nil {
-		os.Exit(2)
-	}
-	if err := registerDumpConfigFlags(); err != nil {
-		os.Exit(2)
-	}
-	if err := registerDumpDBFlags(); err != nil {
-		os.Exit(2)
-	}
-	if err := registerInspectionFlags(); err != nil {
-		os.Exit(2)
-	}
+	harmonyConfigs.VersionMetaData = append(harmonyConfigs.VersionMetaData, "harmony", version, commit, builtBy, builtAt)
+	harmonyConfigs.Init(rootCmd)
 }
 
 func main() {
 	rootCmd.Execute()
 }
 
-func registerRootCmdFlags() error {
-	flags := getRootFlags()
-
-	return cli.RegisterFlags(rootCmd, flags)
-}
-
 func runHarmonyNode(cmd *cobra.Command, args []string) {
-	if cli.GetBoolFlagValue(cmd, versionFlag) {
-		printVersion()
+	if cli.GetBoolFlagValue(cmd, harmonyConfigs.VersionFlag()) {
+		harmonyConfigs.PrintVersion()
 		os.Exit(0)
 	}
 
@@ -141,7 +114,7 @@ func runHarmonyNode(cmd *cobra.Command, args []string) {
 		fmt.Fprint(os.Stderr, err)
 		os.Exit(128)
 	}
-	cfg, err := getHarmonyConfig(cmd)
+	cfg, err := harmonyConfigs.GetHarmonyConfig(cmd)
 	if err != nil {
 		fmt.Fprint(os.Stderr, err)
 		os.Exit(128)
@@ -174,80 +147,6 @@ func raiseFdLimits() error {
 	return nil
 }
 
-func getHarmonyConfig(cmd *cobra.Command) (harmonyconfig.HarmonyConfig, error) {
-	var (
-		config         harmonyconfig.HarmonyConfig
-		err            error
-		migratedFrom   string
-		configFile     string
-		isUsingDefault bool
-	)
-	if cli.IsFlagChanged(cmd, configFlag) {
-		configFile = cli.GetStringFlagValue(cmd, configFlag)
-		config, migratedFrom, err = loadHarmonyConfig(configFile)
-	} else {
-		nt := getNetworkType(cmd)
-		config = getDefaultHmyConfigCopy(nt)
-		isUsingDefault = true
-	}
-	if err != nil {
-		return harmonyconfig.HarmonyConfig{}, err
-	}
-	if migratedFrom != defaultConfig.Version && !isUsingDefault {
-		fmt.Printf("Old config version detected %s\n",
-			migratedFrom)
-		stat, _ := os.Stdin.Stat()
-		// Ask to update if only using terminal
-		if stat.Mode()&os.ModeCharDevice != 0 {
-			if promptConfigUpdate() {
-				err := updateConfigFile(configFile)
-				if err != nil {
-					fmt.Printf("Could not update config - %s", err.Error())
-					fmt.Println("Update config manually with `./harmony config update [config_file]`")
-				}
-			}
-
-		} else {
-			fmt.Println("Update saved config with `./harmony config update [config_file]`")
-		}
-	}
-
-	applyRootFlags(cmd, &config)
-
-	if err := validateHarmonyConfig(config); err != nil {
-		return harmonyconfig.HarmonyConfig{}, err
-	}
-	sanityFixHarmonyConfig(&config)
-	return config, nil
-}
-
-func applyRootFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
-	// Misc flags shall be applied first since legacy ip / port is overwritten
-	// by new ip / port flags
-	applyLegacyMiscFlags(cmd, config)
-	applyGeneralFlags(cmd, config)
-	applyNetworkFlags(cmd, config)
-	applyDNSSyncFlags(cmd, config)
-	applyP2PFlags(cmd, config)
-	applyHTTPFlags(cmd, config)
-	applyWSFlags(cmd, config)
-	applyRPCOptFlags(cmd, config)
-	applyBLSFlags(cmd, config)
-	applyConsensusFlags(cmd, config)
-	applyTxPoolFlags(cmd, config)
-	applyPprofFlags(cmd, config)
-	applyLogFlags(cmd, config)
-	applySysFlags(cmd, config)
-	applyDevnetFlags(cmd, config)
-	applyRevertFlags(cmd, config)
-	applyPreimageFlags(cmd, config)
-	applyPrometheusFlags(cmd, config)
-	applySyncFlags(cmd, config)
-	applyShardDataFlags(cmd, config)
-	applyGPOFlags(cmd, config)
-	applyCacheFlags(cmd, config)
-}
-
 func setupNodeLog(config harmonyconfig.HarmonyConfig) {
 	logPath := filepath.Join(config.Log.Folder, config.Log.FileName)
 	verbosity := config.Log.Verbosity
@@ -292,7 +191,7 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) {
 
 	nodeconfigSetShardSchedule(hc)
 	nodeconfig.SetShardingSchedule(shard.Schedule)
-	nodeconfig.SetVersion(getHarmonyVersion())
+	nodeconfig.SetVersion(harmonyConfigs.GetHarmonyVersion())
 
 	if hc.General.NodeType == "validator" {
 		var err error
@@ -430,7 +329,7 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) {
 	}
 
 	startMsg := "==== New Harmony Node ===="
-	if hc.General.NodeType == nodeTypeExplorer {
+	if hc.General.NodeType == harmonyConfigs.NodeTypeExplorer {
 		startMsg = "==== New Explorer Node ===="
 	}
 
@@ -441,7 +340,7 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) {
 		Str("BeaconGroupID", nodeConfig.GetBeaconGroupID().String()).
 		Str("ClientGroupID", nodeConfig.GetClientGroupID().String()).
 		Str("Role", currentNode.NodeConfig.Role().String()).
-		Str("Version", getHarmonyVersion()).
+		Str("Version", harmonyConfigs.GetHarmonyVersion()).
 		Str("multiaddress",
 			fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", hc.P2P.IP, hc.P2P.Port, myHost.GetID().String()),
 		).
@@ -555,7 +454,7 @@ func nodeconfigSetShardSchedule(config harmonyconfig.HarmonyConfig) {
 		if config.Devnet != nil {
 			dnConfig = *config.Devnet
 		} else {
-			dnConfig = getDefaultDevnetConfigCopy()
+			dnConfig = harmonyConfigs.GetDefaultDevnetConfigCopy()
 		}
 
 		devnetConfig, err := shardingconfig.NewInstance(
@@ -645,7 +544,7 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType,
 		initialAccounts = append(initialAccounts, &genesis.DeployAccount{ShardID: uint32(hc.General.ShardID)})
 	}
 	nodeConfig := nodeconfig.GetShardConfig(initialAccounts[0].ShardID)
-	if hc.General.NodeType == nodeTypeValidator {
+	if hc.General.NodeType == harmonyConfigs.NodeTypeValidator {
 		// Set up consensus keys.
 		setupConsensusKeys(hc, nodeConfig)
 	} else {
@@ -802,6 +701,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi
 		minPeers = hc.Consensus.MinPeers
 		aggregateSig = hc.Consensus.AggregateSig
 	} else {
+		defaultConsensusConfig := harmonyConfigs.GetDefaultConsensusConfigCopy()
 		minPeers = defaultConsensusConfig.MinPeers
 		aggregateSig = defaultConsensusConfig.AggregateSig
 	}
@@ -842,7 +742,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi
 	if hc.Legacy != nil && hc.Legacy.TPBroadcastInvalidTxn != nil {
 		currentNode.BroadcastInvalidTx = *hc.Legacy.TPBroadcastInvalidTxn
 	} else {
-		currentNode.BroadcastInvalidTx = defaultBroadcastInvalidTx
+		currentNode.BroadcastInvalidTx = harmonyConfigs.DefaultBroadcastInvalidTx
 	}
 
 	// Syncing provider is provided by following rules:
@@ -917,11 +817,11 @@ func setupTiKV(hc harmonyconfig.HarmonyConfig) shardchain.DBFactory {
 
 func processNodeType(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType) {
 	switch hc.General.NodeType {
-	case nodeTypeExplorer:
+	case harmonyConfigs.NodeTypeExplorer:
 		nodeconfig.SetDefaultRole(nodeconfig.ExplorerNode)
 		nodeConfig.SetRole(nodeconfig.ExplorerNode)
 
-	case nodeTypeValidator:
+	case harmonyConfigs.NodeTypeValidator:
 		nodeconfig.SetDefaultRole(nodeconfig.Validator)
 		nodeConfig.SetRole(nodeconfig.Validator)
 	}
@@ -929,9 +829,9 @@ func processNodeType(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.Conf
 
 func isBackup(hc harmonyconfig.HarmonyConfig) (isBackup bool) {
 	switch hc.General.NodeType {
-	case nodeTypeExplorer:
+	case harmonyConfigs.NodeTypeExplorer:
 
-	case nodeTypeValidator:
+	case harmonyConfigs.NodeTypeValidator:
 		return hc.General.IsBackup
 	}
 	return false
@@ -1002,7 +902,7 @@ func setupSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyCo
 	node.RegisterService(service.Synchronize, s)
 
 	d := s.Downloaders.GetShardDownloader(node.Blockchain().ShardID())
-	if hc.Sync.Downloader && hc.General.NodeType != nodeTypeExplorer {
+	if hc.Sync.Downloader && hc.General.NodeType != harmonyConfigs.NodeTypeExplorer {
 		node.Consensus.SetDownloader(d) // Set downloader when stream client is active
 	}
 }
@@ -1044,7 +944,7 @@ func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.Har
 	node.RegisterService(service.StagedStreamSync, s)
 
 	d := s.Downloaders.GetShardDownloader(node.Blockchain().ShardID())
-	if hc.Sync.Downloader && hc.General.NodeType != nodeTypeExplorer {
+	if hc.Sync.Downloader && hc.General.NodeType != harmonyConfigs.NodeTypeExplorer {
 		node.Consensus.SetDownloader(d) // Set downloader when stream client is active
 	}
 }
diff --git a/consensus/consensus_block_proposing.go b/consensus/consensus_block_proposing.go
index aeb892aaeb..927b707e6d 100644
--- a/consensus/consensus_block_proposing.go
+++ b/consensus/consensus_block_proposing.go
@@ -11,7 +11,7 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/utils"
-	"github.com/harmony-one/harmony/node/worker"
+	"github.com/harmony-one/harmony/node/harmony/worker"
 	"github.com/harmony-one/harmony/shard"
 	staking "github.com/harmony-one/harmony/staking/types"
 	"github.com/pkg/errors"
diff --git a/core_test/shardchain_test.go b/core_test/shardchain_test.go
index a6a9238bab..ef4cc04cdb 100644
--- a/core_test/shardchain_test.go
+++ b/core_test/shardchain_test.go
@@ -15,7 +15,7 @@ import (
 	"github.com/harmony-one/harmony/internal/shardchain"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/multibls"
-	"github.com/harmony-one/harmony/node"
+	node "github.com/harmony-one/harmony/node/harmony"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/stretchr/testify/require"
diff --git a/hmy/blockchain.go b/hmy/blockchain.go
index 25a6d3aa7f..558d2a821d 100644
--- a/hmy/blockchain.go
+++ b/hmy/blockchain.go
@@ -6,6 +6,7 @@ import (
 	"math/big"
 
 	v3 "github.com/harmony-one/harmony/block/v3"
+	"github.com/harmony-one/harmony/eth/rpc"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/bloombits"
@@ -17,7 +18,6 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/crypto/bls"
 	internal_bls "github.com/harmony-one/harmony/crypto/bls"
-	"github.com/harmony-one/harmony/eth/rpc"
 	internal_common "github.com/harmony-one/harmony/internal/common"
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/internal/utils"
diff --git a/hmy/hmy.go b/hmy/hmy.go
index 097e597d02..12c3378844 100644
--- a/hmy/hmy.go
+++ b/hmy/hmy.go
@@ -18,7 +18,7 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
-	commonRPC "github.com/harmony-one/harmony/rpc/common"
+	commonRPC "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/harmony-one/harmony/shard"
 	staking "github.com/harmony-one/harmony/staking/types"
 	lru "github.com/hashicorp/golang-lru"
diff --git a/hmy/net.go b/hmy/net.go
index 754195d76d..c97e79c33c 100644
--- a/hmy/net.go
+++ b/hmy/net.go
@@ -2,7 +2,7 @@ package hmy
 
 import (
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
-	commonRPC "github.com/harmony-one/harmony/rpc/common"
+	commonRPC "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/harmony-one/harmony/staking/network"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
diff --git a/hmy/staking.go b/hmy/staking.go
index 83e800544c..f80f355886 100644
--- a/hmy/staking.go
+++ b/hmy/staking.go
@@ -16,7 +16,7 @@ import (
 	"github.com/harmony-one/harmony/internal/chain"
 	internalCommon "github.com/harmony-one/harmony/internal/common"
 	"github.com/harmony-one/harmony/numeric"
-	commonRPC "github.com/harmony-one/harmony/rpc/common"
+	commonRPC "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
 	"github.com/harmony-one/harmony/staking/availability"
diff --git a/hmy_boot/hmy_boot.go b/hmy_boot/hmy_boot.go
new file mode 100644
index 0000000000..474ea44456
--- /dev/null
+++ b/hmy_boot/hmy_boot.go
@@ -0,0 +1,64 @@
+package hmy_boot
+
+import (
+	"github.com/harmony-one/harmony/api/proto"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	rpc_common "github.com/harmony-one/harmony/rpc/boot/common"
+	"github.com/libp2p/go-libp2p/core/peer"
+)
+
+// BootService implements the BootService full node service.
+type BootService struct {
+	// Channel for shutting down the service
+	ShutdownChan chan bool // Channel for shutting down the BootService
+	// Boot node API
+	BootNodeAPI BootNodeAPI
+	// Shard ID
+	ShardID uint32
+}
+
+// BootNodeAPI is the list of functions from node used to call rpc apis.
+type BootNodeAPI interface {
+	GetNodeBootTime() int64
+	PeerID() peer.ID
+	PeerConnectivity() (int, int, int)
+	ListKnownPeers() peer.IDSlice
+	ListConnectedPeers() []peer.ID
+	ListPeer(topic string) []peer.ID
+	ListTopic() []string
+	ListBlockedPeer() []peer.ID
+	GetConfig() rpc_common.Config
+	ShutDown()
+}
+
+// New creates a new BootService object (including the
+// initialisation of the common BootService object)
+func New(nodeAPI BootNodeAPI) *BootService {
+	backend := &BootService{
+		ShutdownChan: make(chan bool),
+
+		BootNodeAPI: nodeAPI,
+	}
+
+	return backend
+}
+
+// ProtocolVersion ...
+func (hmyboot *BootService) ProtocolVersion() int {
+	return proto.ProtocolVersion
+}
+
+// GetNodeMetadata returns the node metadata.
+func (hmyboot *BootService) GetNodeMetadata() rpc_common.BootNodeMetadata {
+	var c rpc_common.C
+
+	c.TotalKnownPeers, c.Connected, c.NotConnected = hmyboot.BootNodeAPI.PeerConnectivity()
+
+	return rpc_common.BootNodeMetadata{
+		Version:      nodeconfig.GetVersion(),
+		Network:      string(nodeconfig.GetDefaultConfig().GetNetworkType()),
+		NodeBootTime: hmyboot.BootNodeAPI.GetNodeBootTime(),
+		PeerID:       nodeconfig.GetPeerID(),
+		C:            c,
+	}
+}
diff --git a/hmy_boot/net.go b/hmy_boot/net.go
new file mode 100644
index 0000000000..c0db8c2b0d
--- /dev/null
+++ b/hmy_boot/net.go
@@ -0,0 +1,22 @@
+package hmy_boot
+
+import (
+	commonRPC "github.com/harmony-one/harmony/rpc/boot/common"
+)
+
+// GetPeerInfo returns the peer info to the node, including blocked peer, connected peer, number of peers
+func (hmyboot *BootService) GetPeerInfo() commonRPC.BootNodePeerInfo {
+
+	var c commonRPC.C
+	c.TotalKnownPeers, c.Connected, c.NotConnected = hmyboot.BootNodeAPI.PeerConnectivity()
+
+	knownPeers := hmyboot.BootNodeAPI.ListKnownPeers()
+	connectedPeers := hmyboot.BootNodeAPI.ListConnectedPeers()
+
+	return commonRPC.BootNodePeerInfo{
+		PeerID:         hmyboot.BootNodeAPI.PeerID(),
+		KnownPeers:     knownPeers,
+		ConnectedPeers: connectedPeers,
+		C:              c,
+	}
+}
diff --git a/internal/configs/bootnode/bootnode.go b/internal/configs/bootnode/bootnode.go
new file mode 100644
index 0000000000..6d0abfe821
--- /dev/null
+++ b/internal/configs/bootnode/bootnode.go
@@ -0,0 +1,186 @@
+package bootnode
+
+import (
+	"reflect"
+	"strings"
+	"time"
+
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/internal/utils"
+)
+
+// BootNodeConfig contains all the configs user can set for running harmony binary. Served as the bridge
+// from user set flags to internal node configs. Also user can persist this structure to a toml file
+// to avoid inputting all arguments.
+type BootNodeConfig struct {
+	Version string
+	General GeneralConfig
+	P2P     P2pConfig
+	HTTP    HttpConfig
+	WS      WsConfig
+	RPCOpt  RpcOptConfig
+	Pprof   PprofConfig
+	Log     LogConfig
+	Sys     *SysConfig `toml:",omitempty"`
+}
+
+func (bnc BootNodeConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig {
+	readTimeout, err := time.ParseDuration(bnc.HTTP.ReadTimeout)
+	if err != nil {
+		readTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutRead)
+		utils.Logger().Warn().
+			Str("provided", bnc.HTTP.ReadTimeout).
+			Dur("updated", readTimeout).
+			Msg("Sanitizing invalid http read timeout")
+	}
+	writeTimeout, err := time.ParseDuration(bnc.HTTP.WriteTimeout)
+	if err != nil {
+		writeTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutWrite)
+		utils.Logger().Warn().
+			Str("provided", bnc.HTTP.WriteTimeout).
+			Dur("updated", writeTimeout).
+			Msg("Sanitizing invalid http write timeout")
+	}
+	idleTimeout, err := time.ParseDuration(bnc.HTTP.IdleTimeout)
+	if err != nil {
+		idleTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutIdle)
+		utils.Logger().Warn().
+			Str("provided", bnc.HTTP.IdleTimeout).
+			Dur("updated", idleTimeout).
+			Msg("Sanitizing invalid http idle timeout")
+	}
+	return nodeconfig.RPCServerConfig{
+		HTTPEnabled:        bnc.HTTP.Enabled,
+		HTTPIp:             bnc.HTTP.IP,
+		HTTPPort:           bnc.HTTP.Port,
+		HTTPTimeoutRead:    readTimeout,
+		HTTPTimeoutWrite:   writeTimeout,
+		HTTPTimeoutIdle:    idleTimeout,
+		WSEnabled:          bnc.WS.Enabled,
+		WSIp:               bnc.WS.IP,
+		WSPort:             bnc.WS.Port,
+		DebugEnabled:       bnc.RPCOpt.DebugEnabled,
+		EthRPCsEnabled:     bnc.RPCOpt.EthRPCsEnabled,
+		LegacyRPCsEnabled:  bnc.RPCOpt.LegacyRPCsEnabled,
+		RpcFilterFile:      bnc.RPCOpt.RpcFilterFile,
+		RateLimiterEnabled: bnc.RPCOpt.RateLimterEnabled,
+		RequestsPerSecond:  bnc.RPCOpt.RequestsPerSecond,
+	}
+}
+
+type P2pConfig struct {
+	Port                 int
+	IP                   string
+	KeyFile              string
+	DHTDataStore         *string `toml:",omitempty"`
+	DiscConcurrency      int     // Discovery Concurrency value
+	MaxConnsPerIP        int
+	DisablePrivateIPScan bool
+	MaxPeers             int64
+	// In order to disable Connection Manager, it only needs to
+	// set both the high and low watermarks to zero. In this way,
+	// using Connection Manager will be an optional feature.
+	ConnManagerLowWatermark  int
+	ConnManagerHighWatermark int
+	WaitForEachPeerToConnect bool
+	// to disable p2p security (tls and noise)
+	NoTransportSecurity bool
+	// enable p2p NAT. NAT Manager takes care of setting NAT port mappings, and discovering external addresses
+	NAT bool
+	// custom user agent; explicitly set the user-agent, so we can differentiate from other Go libp2p users
+	UserAgent string
+	// p2p dial timeout
+	DialTimeout time.Duration
+	// P2P multiplexer type, should be comma separated (mplex, Yamux)
+	Muxer string
+	// No relay services, direct connections between peers only
+	NoRelay bool
+}
+
+type GeneralConfig struct {
+	NodeType    string
+	ShardID     int
+	TraceEnable bool
+}
+
+type PprofConfig struct {
+	Enabled            bool
+	ListenAddr         string
+	Folder             string
+	ProfileNames       []string
+	ProfileIntervals   []int
+	ProfileDebugValues []int
+}
+
+type LogConfig struct {
+	Console       bool
+	Folder        string
+	FileName      string
+	RotateSize    int
+	RotateCount   int
+	RotateMaxAge  int
+	Verbosity     int
+	VerbosePrints LogVerbosePrints
+	Context       *LogContext `toml:",omitempty"`
+}
+
+type LogVerbosePrints struct {
+	Config bool
+}
+
+func FlagSliceToLogVerbosePrints(verbosePrintsFlagSlice []string) LogVerbosePrints {
+	verbosePrints := LogVerbosePrints{}
+	verbosePrintsReflect := reflect.Indirect(reflect.ValueOf(&verbosePrints))
+	for _, verbosePrint := range verbosePrintsFlagSlice {
+		verbosePrint = strings.Title(verbosePrint)
+		field := verbosePrintsReflect.FieldByName(verbosePrint)
+		if field.IsValid() && field.CanSet() {
+			field.SetBool(true)
+		}
+	}
+
+	return verbosePrints
+}
+
+type LogContext struct {
+	IP   string
+	Port int
+}
+
+type SysConfig struct {
+	NtpServer string
+}
+
+type HttpConfig struct {
+	Enabled      bool
+	IP           string
+	Port         int
+	AuthPort     int
+	ReadTimeout  string
+	WriteTimeout string
+	IdleTimeout  string
+}
+
+type WsConfig struct {
+	Enabled  bool
+	IP       string
+	Port     int
+	AuthPort int
+}
+
+type RpcOptConfig struct {
+	DebugEnabled      bool   // Enables PrivateDebugService APIs, including the EVM tracer
+	EthRPCsEnabled    bool   // Expose Eth RPCs
+	LegacyRPCsEnabled bool   // Expose Legacy RPCs
+	RpcFilterFile     string // Define filters to enable/disable RPC exposure
+	RateLimterEnabled bool   // Enable Rate limiter for RPC
+	RequestsPerSecond int    // for RPC rate limiter
+}
+
+type PrometheusConfig struct {
+	Enabled    bool
+	IP         string
+	Port       int
+	EnablePush bool
+	Gateway    string
+}
diff --git a/internal/configs/bootnode/bootnode_test.go b/internal/configs/bootnode/bootnode_test.go
new file mode 100644
index 0000000000..5e99de78e4
--- /dev/null
+++ b/internal/configs/bootnode/bootnode_test.go
@@ -0,0 +1,71 @@
+package bootnode
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestToRPCServerConfig(t *testing.T) {
+	tests := []struct {
+		input  BootNodeConfig
+		output nodeconfig.RPCServerConfig
+	}{
+		{
+			input: BootNodeConfig{
+				HTTP: HttpConfig{
+					Enabled:      true,
+					IP:           "127.0.0.1",
+					Port:         nodeconfig.DefaultRPCPort,
+					ReadTimeout:  "-1",
+					WriteTimeout: "-2",
+					IdleTimeout:  "-3",
+				},
+				WS: WsConfig{
+					Enabled: true,
+					IP:      "127.0.0.1",
+					Port:    nodeconfig.DefaultWSPort,
+				},
+				RPCOpt: RpcOptConfig{
+					DebugEnabled:      false,
+					EthRPCsEnabled:    true,
+					LegacyRPCsEnabled: true,
+					RpcFilterFile:     "./.hmy/rpc_filter.txt",
+					RateLimterEnabled: true,
+					RequestsPerSecond: nodeconfig.DefaultRPCRateLimit,
+				},
+			},
+			output: nodeconfig.RPCServerConfig{
+				HTTPEnabled:        true,
+				HTTPIp:             "127.0.0.1",
+				HTTPPort:           nodeconfig.DefaultRPCPort,
+				HTTPTimeoutRead:    30 * time.Second,
+				HTTPTimeoutWrite:   30 * time.Second,
+				HTTPTimeoutIdle:    120 * time.Second,
+				WSEnabled:          true,
+				WSIp:               "127.0.0.1",
+				WSPort:             nodeconfig.DefaultWSPort,
+				DebugEnabled:       false,
+				EthRPCsEnabled:     true,
+				LegacyRPCsEnabled:  true,
+				RpcFilterFile:      "./.hmy/rpc_filter.txt",
+				RateLimiterEnabled: true,
+				RequestsPerSecond:  nodeconfig.DefaultRPCRateLimit,
+			},
+		},
+	}
+	for i, tt := range tests {
+		assertObject := assert.New(t)
+		name := fmt.Sprintf("TestToRPCServerConfig: #%d", i)
+		t.Run(name, func(t *testing.T) {
+			assertObject.Equal(
+				tt.input.ToRPCServerConfig(),
+				tt.output,
+				name,
+			)
+		})
+	}
+}
diff --git a/internal/configs/bootnode/config.go b/internal/configs/bootnode/config.go
new file mode 100644
index 0000000000..75f2f783dd
--- /dev/null
+++ b/internal/configs/bootnode/config.go
@@ -0,0 +1,191 @@
+// Package nodeconfig includes all the configuration variables for a node.
+// It is a global configuration for node and other services.
+// It will be included in node module, and other modules.
+package bootnode
+
+import (
+	"fmt"
+	"time"
+
+	shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding"
+	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/webhooks"
+	"github.com/libp2p/go-libp2p/core/peer"
+)
+
+// Role defines a role of a node.
+type Role byte
+
+// All constants for different node roles.
+const (
+	Unknown Role = iota
+	BootNode
+)
+
+func (role Role) String() string {
+	switch role {
+	case BootNode:
+		return "BootNode"
+	default:
+		return "Unknown"
+	}
+}
+
+// NetworkType describes the type of Harmony network
+type NetworkType string
+
+// Constants for NetworkType
+// TODO: replace this with iota. Leave the string parsing in command line
+const (
+	Mainnet   = "mainnet"
+	Testnet   = "testnet"
+	Pangaea   = "pangaea"
+	Partner   = "partner"
+	Stressnet = "stressnet"
+	Devnet    = "devnet"
+	Localnet  = "localnet"
+)
+
+// ChainConfig returns the chain configuration for the network type.
+func (t NetworkType) ChainConfig() params.ChainConfig {
+	switch t {
+	case Mainnet:
+		return *params.MainnetChainConfig
+	case Pangaea:
+		return *params.PangaeaChainConfig
+	case Partner:
+		return *params.PartnerChainConfig
+	case Stressnet:
+		return *params.StressnetChainConfig
+	case Localnet:
+		return *params.LocalnetChainConfig
+	default:
+		return *params.TestnetChainConfig
+	}
+}
+
+func (n NetworkType) String() string {
+	if n == "" {
+		return Testnet // default to testnet
+	}
+	return string(n)
+}
+
+var version string
+var peerID peer.ID // PeerID of the node
+
+// ConfigType is the structure of all node related configuration variables
+type ConfigType struct {
+	ShardID   uint32          // ShardID of this node; TODO ek – revisit when resharding
+	role      Role            // Role of the node
+	Port      string          // Port of the node.
+	IP        string          // IP of the node.
+	RPCServer RPCServerConfig // RPC server port and ip
+	DebugMode bool            // log every single process and error to help to debug the syncing issues
+	NtpServer string
+
+	shardingSchedule shardingconfig.Schedule
+	networkType      NetworkType
+	DNSZone          string
+	WebHooks         struct {
+		Hooks *webhooks.Hooks
+	}
+	TraceEnable bool
+}
+
+// RPCServerConfig is the config for rpc listen addresses
+type RPCServerConfig struct {
+	HTTPEnabled bool
+	HTTPIp      string
+	HTTPPort    int
+
+	HTTPTimeoutRead  time.Duration
+	HTTPTimeoutWrite time.Duration
+	HTTPTimeoutIdle  time.Duration
+
+	WSEnabled bool
+	WSIp      string
+	WSPort    int
+
+	DebugEnabled bool
+
+	NetworkRPCsEnabled bool
+
+	RpcFilterFile string
+
+	RateLimiterEnabled bool
+	RequestsPerSecond  int
+}
+
+// configs is a list of node configuration.
+// It has at least one configuration.
+// The first one is the default, global node configuration
+var shardConfigs []ConfigType
+var defaultConfig ConfigType
+
+// GetDefaultConfig returns default config.
+func GetDefaultConfig() *ConfigType {
+	return &defaultConfig
+}
+
+// SetDefaultRole ..
+func SetDefaultRole(r Role) {
+	defaultConfig.role = r
+}
+
+func (conf *ConfigType) String() string {
+	return fmt.Sprintf("%v", conf.ShardID)
+}
+
+// SetShardID set the ShardID
+func (conf *ConfigType) SetShardID(s uint32) {
+	conf.ShardID = s
+}
+
+// SetRole set the role
+func (conf *ConfigType) SetRole(r Role) {
+	conf.role = r
+}
+
+// GetShardID returns the shardID.
+func (conf *ConfigType) GetShardID() uint32 {
+	return conf.ShardID
+}
+
+// Role returns the role
+func (conf *ConfigType) Role() Role {
+	return conf.role
+}
+
+// SetNetworkType set the networkType
+func SetNetworkType(networkType NetworkType) {
+	defaultConfig.networkType = networkType
+	for i := range shardConfigs {
+		shardConfigs[i].networkType = networkType
+	}
+}
+
+// GetNetworkType gets the networkType
+func (conf *ConfigType) GetNetworkType() NetworkType {
+	return conf.networkType
+}
+
+// SetVersion set the version of the node binary
+func SetVersion(ver string) {
+	version = ver
+}
+
+// GetVersion return the version of the node binary
+func GetVersion() string {
+	return version
+}
+
+// SetPeerID set the peer ID of the node
+func SetPeerID(pid peer.ID) {
+	peerID = pid
+}
+
+// GetPeerID returns the peer ID of the node
+func GetPeerID() peer.ID {
+	return peerID
+}
diff --git a/internal/registry/registry.go b/internal/registry/registry.go
index 760bb8d381..ab8e532e3c 100644
--- a/internal/registry/registry.go
+++ b/internal/registry/registry.go
@@ -11,7 +11,7 @@ import (
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/shardchain"
 	"github.com/harmony-one/harmony/multibls"
-	"github.com/harmony-one/harmony/node/worker"
+	"github.com/harmony-one/harmony/node/harmony/worker"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/webhooks"
 )
diff --git a/node/boot/README.md b/node/boot/README.md
new file mode 100644
index 0000000000..b987b730f3
--- /dev/null
+++ b/node/boot/README.md
@@ -0,0 +1,5 @@
+BootNode struct is the core entity that represents a bootstrap node participating in the Harmony protocol.
+
+New nodes in a p2p network often make their initial connection to the p2p network through a set of nodes known as boot nodes. Information (e.g. addresses) about these boot nodes is e.g. embedded in an application binary or provided as a configuration option.
+
+The boot nodes serve as an entry point, providing a list of other nodes in the network to newcomers. After connecting to the boot nodes, the new node can connect to those other nodes in the network, thereby no longer relying on the boot nodes.
\ No newline at end of file
diff --git a/node/boot/api.go b/node/boot/api.go
new file mode 100644
index 0000000000..4f5887cb80
--- /dev/null
+++ b/node/boot/api.go
@@ -0,0 +1,138 @@
+package bootnode
+
+import (
+	"time"
+
+	"github.com/harmony-one/harmony/eth/rpc"
+	hmy_boot "github.com/harmony-one/harmony/hmy_boot"
+	bootnodeConfigs "github.com/harmony-one/harmony/internal/configs/bootnode"
+	nodeConfigs "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/internal/utils"
+	boot_rpc "github.com/harmony-one/harmony/rpc/boot"
+	rpc_common "github.com/harmony-one/harmony/rpc/boot/common"
+	"github.com/libp2p/go-libp2p/core/peer"
+)
+
+// PeerID returns self Peer ID
+func (bootnode *BootNode) PeerID() peer.ID {
+	return bootnode.host.GetID()
+}
+
+// PeerConnectivity ..
+func (bootnode *BootNode) PeerConnectivity() (int, int, int) {
+	return bootnode.host.PeerConnectivity()
+}
+
+// ListKnownPeers return known peers
+func (bootnode *BootNode) ListKnownPeers() peer.IDSlice {
+	bs := bootnode.host.GetP2PHost().Peerstore()
+	if bs == nil {
+		return peer.IDSlice{}
+	}
+	return bs.Peers()
+}
+
+// ListConnectedPeers return connected peers
+func (bootnode *BootNode) ListConnectedPeers() []peer.ID {
+	return bootnode.host.Network().Peers()
+}
+
+// ListPeer return list of peers for a certain topic
+func (bootnode *BootNode) ListPeer(topic string) []peer.ID {
+	return bootnode.host.ListPeer(topic)
+}
+
+// ListTopic return list of topics the node subscribed
+func (bootnode *BootNode) ListTopic() []string {
+	return bootnode.host.ListTopic()
+}
+
+// ListBlockedPeer return list of blocked peers
+func (bootnode *BootNode) ListBlockedPeer() []peer.ID {
+	return bootnode.host.ListBlockedPeer()
+}
+
+// GetNodeBootTime ..
+func (bootnode *BootNode) GetNodeBootTime() int64 {
+	return bootnode.unixTimeAtNodeStart
+}
+
+// StartRPC start RPC service
+func (bootnode *BootNode) StartRPC() error {
+	bootService := hmy_boot.New(bootnode)
+	// Gather all the possible APIs to surface
+	apis := bootnode.APIs(bootService)
+
+	err := boot_rpc.StartServers(bootService, apis, *bootnode.RPCConfig, bootnode.HarmonyConfig.RPCOpt)
+
+	return err
+}
+
+func (bootnode *BootNode) initRPCServerConfig() {
+	cfg := bootnode.HarmonyConfig
+
+	readTimeout, err := time.ParseDuration(cfg.HTTP.ReadTimeout)
+	if err != nil {
+		readTimeout, _ = time.ParseDuration(nodeConfigs.DefaultHTTPTimeoutRead)
+		utils.Logger().Warn().
+			Str("provided", cfg.HTTP.ReadTimeout).
+			Dur("updated", readTimeout).
+			Msg("Sanitizing invalid http read timeout")
+	}
+	writeTimeout, err := time.ParseDuration(cfg.HTTP.WriteTimeout)
+	if err != nil {
+		writeTimeout, _ = time.ParseDuration(nodeConfigs.DefaultHTTPTimeoutWrite)
+		utils.Logger().Warn().
+			Str("provided", cfg.HTTP.WriteTimeout).
+			Dur("updated", writeTimeout).
+			Msg("Sanitizing invalid http write timeout")
+	}
+	idleTimeout, err := time.ParseDuration(cfg.HTTP.IdleTimeout)
+	if err != nil {
+		idleTimeout, _ = time.ParseDuration(nodeConfigs.DefaultHTTPTimeoutIdle)
+		utils.Logger().Warn().
+			Str("provided", cfg.HTTP.IdleTimeout).
+			Dur("updated", idleTimeout).
+			Msg("Sanitizing invalid http idle timeout")
+	}
+	bootnode.RPCConfig = &bootnodeConfigs.RPCServerConfig{
+		HTTPEnabled:        cfg.HTTP.Enabled,
+		HTTPIp:             cfg.HTTP.IP,
+		HTTPPort:           cfg.HTTP.Port,
+		HTTPTimeoutRead:    readTimeout,
+		HTTPTimeoutWrite:   writeTimeout,
+		HTTPTimeoutIdle:    idleTimeout,
+		WSEnabled:          cfg.WS.Enabled,
+		WSIp:               cfg.WS.IP,
+		WSPort:             cfg.WS.Port,
+		DebugEnabled:       cfg.RPCOpt.DebugEnabled,
+		RpcFilterFile:      cfg.RPCOpt.RpcFilterFile,
+		RateLimiterEnabled: cfg.RPCOpt.RateLimterEnabled,
+		RequestsPerSecond:  cfg.RPCOpt.RequestsPerSecond,
+	}
+}
+
+func (bootnode *BootNode) GetRPCServerConfig() *bootnodeConfigs.RPCServerConfig {
+	return bootnode.RPCConfig
+}
+
+// StopRPC stop RPC service
+func (bootnode *BootNode) StopRPC() error {
+	return boot_rpc.StopServers()
+}
+
+// APIs return the collection of local RPC services.
+// NOTE, some of these services probably need to be moved to somewhere else.
+func (bootnode *BootNode) APIs(harmony *hmy_boot.BootService) []rpc.API {
+	// Append all the local APIs and return
+	return []rpc.API{}
+}
+
+func (bootnode *BootNode) GetConfig() rpc_common.Config {
+	return rpc_common.Config{
+		HarmonyConfig: *bootnode.HarmonyConfig,
+		NodeConfig:    *bootnode.NodeConfig,
+		ChainConfig:   params.ChainConfig{},
+	}
+}
diff --git a/node/boot/bootnode.go b/node/boot/bootnode.go
new file mode 100644
index 0000000000..ce3c97fbd0
--- /dev/null
+++ b/node/boot/bootnode.go
@@ -0,0 +1,94 @@
+package bootnode
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/harmony-one/harmony/api/service"
+	bootnodeConfigs "github.com/harmony-one/harmony/internal/configs/bootnode"
+	harmonyConfig "github.com/harmony-one/harmony/internal/configs/harmony"
+	nodeConfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/p2p"
+	"github.com/rcrowley/go-metrics"
+)
+
+const (
+	// NumTryBroadCast is the number of times trying to broadcast
+	NumTryBroadCast = 3
+	// MsgChanBuffer is the buffer of consensus message handlers.
+	MsgChanBuffer = 1024
+)
+
+// BootNode represents a protocol-participating node in the network
+type BootNode struct {
+	SelfPeer p2p.Peer
+	host     p2p.Host
+	// Service manager.
+	serviceManager *service.Manager
+	// harmony configurations
+	HarmonyConfig *harmonyConfig.HarmonyConfig
+	// node configuration, including group ID, shard ID, etc
+	NodeConfig *nodeConfig.ConfigType
+	// RPC configurations
+	RPCConfig *bootnodeConfigs.RPCServerConfig
+	// node start time
+	unixTimeAtNodeStart int64
+	// metrics
+	Metrics metrics.Registry
+}
+
+// New creates a new boot node.
+func New(
+	host p2p.Host,
+	hc *harmonyConfig.HarmonyConfig,
+) *BootNode {
+	node := BootNode{
+		unixTimeAtNodeStart: time.Now().Unix(),
+		HarmonyConfig:       hc,
+		NodeConfig:          &nodeConfig.ConfigType{},
+	}
+
+	if host != nil {
+		node.host = host
+		node.SelfPeer = host.GetSelfPeer()
+	}
+
+	// init metrics
+	initMetrics()
+	nodeStringCounterVec.WithLabelValues("version", nodeConfig.GetVersion()).Inc()
+
+	node.serviceManager = service.NewManager()
+
+	node.initRPCServerConfig()
+
+	return &node
+}
+
+// ServiceManager ...
+func (bootnode *BootNode) ServiceManager() *service.Manager {
+	return bootnode.serviceManager
+}
+
+// ShutDown gracefully shut down the node server and dump the in-memory blockchain state into DB.
+func (bootnode *BootNode) ShutDown() {
+	if err := bootnode.StopRPC(); err != nil {
+		utils.Logger().Error().Err(err).Msg("failed to stop boot RPC")
+	}
+
+	utils.Logger().Info().Msg("stopping boot services")
+	if err := bootnode.StopServices(); err != nil {
+		utils.Logger().Error().Err(err).Msg("failed to stop boot services")
+	}
+
+	utils.Logger().Info().Msg("stopping boot host")
+	if err := bootnode.host.Close(); err != nil {
+		utils.Logger().Error().Err(err).Msg("failed to stop boot p2p host")
+	}
+
+	const msg = "Successfully shut down boot!\n"
+	utils.Logger().Print(msg)
+	fmt.Print(msg)
+	os.Exit(0)
+}
diff --git a/node/boot/metrics.go b/node/boot/metrics.go
new file mode 100644
index 0000000000..80f94c921b
--- /dev/null
+++ b/node/boot/metrics.go
@@ -0,0 +1,32 @@
+package bootnode
+
+import (
+	"sync"
+
+	prom "github.com/harmony-one/harmony/api/service/prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	// nodeStringCounterVec is used to add version string or other static string
+	// info into the metrics api
+	nodeStringCounterVec = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "hmy",
+			Subsystem: "bootnode",
+			Name:      "metadata",
+			Help:      "a list of boot node metadata",
+		},
+		[]string{"key", "value"},
+	)
+
+	onceMetrics sync.Once
+)
+
+func initMetrics() {
+	onceMetrics.Do(func() {
+		prom.PromRegistry().MustRegister(
+			nodeStringCounterVec,
+		)
+	})
+}
diff --git a/node/boot/node_test.go b/node/boot/node_test.go
new file mode 100644
index 0000000000..4c57b1abea
--- /dev/null
+++ b/node/boot/node_test.go
@@ -0,0 +1,33 @@
+package bootnode
+
+import (
+	"testing"
+
+	harmonyConfigs "github.com/harmony-one/harmony/cmd/config"
+	"github.com/harmony-one/harmony/crypto/bls"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/p2p"
+)
+
+func TestNewBootNode(t *testing.T) {
+	blsKey := bls.RandPrivateKey()
+	pubKey := blsKey.GetPublicKey()
+	leader := p2p.Peer{IP: "127.0.0.1", Port: "8882", ConsensusPubKey: pubKey}
+	priKey, _, _ := utils.GenKeyP2P("127.0.0.1", "9902")
+	host, err := p2p.NewHost(p2p.HostConfig{
+		Self:      &leader,
+		BLSKey:    priKey,
+		BootNodes: nil,
+	})
+	if err != nil {
+		t.Fatalf("new boot node host failure: %v", err)
+	}
+
+	hc := harmonyConfigs.GetDefaultHmyConfigCopy(nodeconfig.NetworkType(nodeconfig.Devnet))
+	node := New(host, &hc)
+
+	if node == nil {
+		t.Error("boot node creation failed")
+	}
+}
diff --git a/node/boot/service_setup.go b/node/boot/service_setup.go
new file mode 100644
index 0000000000..a7eae3ffb6
--- /dev/null
+++ b/node/boot/service_setup.go
@@ -0,0 +1,20 @@
+package bootnode
+
+import (
+	"github.com/harmony-one/harmony/api/service"
+)
+
+// RegisterService register a service to the node service manager
+func (bootnode *BootNode) RegisterService(st service.Type, s service.Service) {
+	bootnode.serviceManager.Register(st, s)
+}
+
+// StartServices runs registered services.
+func (bootnode *BootNode) StartServices() error {
+	return bootnode.serviceManager.StartServices()
+}
+
+// StopServices runs registered services.
+func (bootnode *BootNode) StopServices() error {
+	return bootnode.serviceManager.StopServices()
+}
diff --git a/node/README.md b/node/harmony/README.md
similarity index 100%
rename from node/README.md
rename to node/harmony/README.md
diff --git a/node/addresses.go b/node/harmony/addresses.go
similarity index 100%
rename from node/addresses.go
rename to node/harmony/addresses.go
diff --git a/node/api.go b/node/harmony/api.go
similarity index 97%
rename from node/api.go
rename to node/harmony/api.go
index d49d40f076..56c4e0f4d2 100644
--- a/node/api.go
+++ b/node/harmony/api.go
@@ -9,9 +9,9 @@ import (
 	"github.com/harmony-one/harmony/hmy"
 	"github.com/harmony-one/harmony/internal/tikv"
 	"github.com/harmony-one/harmony/rosetta"
-	hmy_rpc "github.com/harmony-one/harmony/rpc"
-	rpc_common "github.com/harmony-one/harmony/rpc/common"
-	"github.com/harmony-one/harmony/rpc/filters"
+	hmy_rpc "github.com/harmony-one/harmony/rpc/harmony"
+	rpc_common "github.com/harmony-one/harmony/rpc/harmony/common"
+	"github.com/harmony-one/harmony/rpc/harmony/filters"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
 
diff --git a/node/double_signing.go b/node/harmony/double_signing.go
similarity index 100%
rename from node/double_signing.go
rename to node/harmony/double_signing.go
diff --git a/node/metrics.go b/node/harmony/metrics.go
similarity index 100%
rename from node/metrics.go
rename to node/harmony/metrics.go
diff --git a/node/node.go b/node/harmony/node.go
similarity index 99%
rename from node/node.go
rename to node/harmony/node.go
index 846eaecebb..19778506a6 100644
--- a/node/node.go
+++ b/node/harmony/node.go
@@ -33,7 +33,7 @@ import (
 	"github.com/harmony-one/harmony/internal/tikv/redis_helper"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/internal/utils/crosslinks"
-	"github.com/harmony-one/harmony/node/worker"
+	"github.com/harmony-one/harmony/node/harmony/worker"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/reward"
diff --git a/node/node.md b/node/harmony/node.md
similarity index 100%
rename from node/node.md
rename to node/harmony/node.md
diff --git a/node/node_cross_link.go b/node/harmony/node_cross_link.go
similarity index 100%
rename from node/node_cross_link.go
rename to node/harmony/node_cross_link.go
diff --git a/node/node_cross_shard.go b/node/harmony/node_cross_shard.go
similarity index 100%
rename from node/node_cross_shard.go
rename to node/harmony/node_cross_shard.go
diff --git a/node/node_explorer.go b/node/harmony/node_explorer.go
similarity index 100%
rename from node/node_explorer.go
rename to node/harmony/node_explorer.go
diff --git a/node/node_handler.go b/node/harmony/node_handler.go
similarity index 100%
rename from node/node_handler.go
rename to node/harmony/node_handler.go
diff --git a/node/node_handler_test.go b/node/harmony/node_handler_test.go
similarity index 100%
rename from node/node_handler_test.go
rename to node/harmony/node_handler_test.go
diff --git a/node/node_newblock.go b/node/harmony/node_newblock.go
similarity index 100%
rename from node/node_newblock.go
rename to node/harmony/node_newblock.go
diff --git a/node/node_newblock_test.go b/node/harmony/node_newblock_test.go
similarity index 100%
rename from node/node_newblock_test.go
rename to node/harmony/node_newblock_test.go
diff --git a/node/node_syncing.go b/node/harmony/node_syncing.go
similarity index 100%
rename from node/node_syncing.go
rename to node/harmony/node_syncing.go
diff --git a/node/node_test.go b/node/harmony/node_test.go
similarity index 100%
rename from node/node_test.go
rename to node/harmony/node_test.go
diff --git a/node/service_setup.go b/node/harmony/service_setup.go
similarity index 100%
rename from node/service_setup.go
rename to node/harmony/service_setup.go
diff --git a/node/worker/types.go b/node/harmony/worker/types.go
similarity index 100%
rename from node/worker/types.go
rename to node/harmony/worker/types.go
diff --git a/node/worker/worker.go b/node/harmony/worker/worker.go
similarity index 100%
rename from node/worker/worker.go
rename to node/harmony/worker/worker.go
diff --git a/node/worker/worker_test.go b/node/harmony/worker/worker_test.go
similarity index 100%
rename from node/worker/worker_test.go
rename to node/harmony/worker/worker_test.go
diff --git a/rosetta/common/operations.go b/rosetta/common/operations.go
index c8e6014ded..a8269447ca 100644
--- a/rosetta/common/operations.go
+++ b/rosetta/common/operations.go
@@ -8,7 +8,7 @@ import (
 
 	"github.com/coinbase/rosetta-sdk-go/types"
 
-	rpcV2 "github.com/harmony-one/harmony/rpc/v2"
+	rpcV2 "github.com/harmony-one/harmony/rpc/harmony/v2"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
diff --git a/rosetta/services/account.go b/rosetta/services/account.go
index da9a67c226..15b600e674 100644
--- a/rosetta/services/account.go
+++ b/rosetta/services/account.go
@@ -6,13 +6,13 @@ import (
 	"math/big"
 
 	"github.com/harmony-one/harmony/core/vm"
+	"github.com/harmony-one/harmony/eth/rpc"
 
 	"github.com/coinbase/rosetta-sdk-go/server"
 	"github.com/coinbase/rosetta-sdk-go/types"
 	ethCommon "github.com/ethereum/go-ethereum/common"
 
 	hmyTypes "github.com/harmony-one/harmony/core/types"
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/hmy"
 	internalCommon "github.com/harmony-one/harmony/internal/common"
 	"github.com/harmony-one/harmony/rosetta/common"
diff --git a/rosetta/services/block.go b/rosetta/services/block.go
index c488c5ff95..d187f2ea6d 100644
--- a/rosetta/services/block.go
+++ b/rosetta/services/block.go
@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/hmy/tracers"
 
 	"github.com/harmony-one/harmony/core"
@@ -21,7 +22,6 @@ import (
 	"github.com/harmony-one/harmony/core/rawdb"
 	hmytypes "github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/hmy"
 	"github.com/harmony-one/harmony/rosetta/common"
 	stakingTypes "github.com/harmony-one/harmony/staking/types"
diff --git a/rosetta/services/call_service.go b/rosetta/services/call_service.go
index 9d26bab282..fd128a845e 100644
--- a/rosetta/services/call_service.go
+++ b/rosetta/services/call_service.go
@@ -11,7 +11,7 @@ import (
 	"github.com/harmony-one/harmony/hmy"
 	internal_common "github.com/harmony-one/harmony/internal/common"
 	"github.com/harmony-one/harmony/rosetta/common"
-	rpc2 "github.com/harmony-one/harmony/rpc"
+	rpc2 "github.com/harmony-one/harmony/rpc/harmony"
 	"github.com/pkg/errors"
 )
 
diff --git a/rosetta/services/construction_check.go b/rosetta/services/construction_check.go
index c842770ab7..b88e3bf6c5 100644
--- a/rosetta/services/construction_check.go
+++ b/rosetta/services/construction_check.go
@@ -15,7 +15,7 @@ import (
 	ethRpc "github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/rosetta/common"
-	"github.com/harmony-one/harmony/rpc"
+	rpc "github.com/harmony-one/harmony/rpc/harmony"
 )
 
 // ConstructMetadataOptions is constructed by ConstructionPreprocess for ConstructionMetadata options
diff --git a/rosetta/services/network.go b/rosetta/services/network.go
index 75ac52614c..a2879d9a8d 100644
--- a/rosetta/services/network.go
+++ b/rosetta/services/network.go
@@ -16,7 +16,7 @@ import (
 	"github.com/harmony-one/harmony/hmy"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/rosetta/common"
-	commonRPC "github.com/harmony-one/harmony/rpc/common"
+	commonRPC "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/harmony-one/harmony/shard"
 )
 
diff --git a/rosetta/services/network_test.go b/rosetta/services/network_test.go
index 3145df42c3..abdfd6b8fb 100644
--- a/rosetta/services/network_test.go
+++ b/rosetta/services/network_test.go
@@ -8,7 +8,7 @@ import (
 
 	"github.com/coinbase/rosetta-sdk-go/types"
 	"github.com/harmony-one/harmony/rosetta/common"
-	commonRPC "github.com/harmony-one/harmony/rpc/common"
+	commonRPC "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
 
diff --git a/rosetta/services/tx_operation.go b/rosetta/services/tx_operation.go
index 67da5227ad..1166af4811 100644
--- a/rosetta/services/tx_operation.go
+++ b/rosetta/services/tx_operation.go
@@ -16,7 +16,7 @@ import (
 	"github.com/harmony-one/harmony/hmy"
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/rosetta/common"
-	rpcV2 "github.com/harmony-one/harmony/rpc/v2"
+	rpcV2 "github.com/harmony-one/harmony/rpc/harmony/v2"
 	"github.com/harmony-one/harmony/staking"
 	stakingTypes "github.com/harmony-one/harmony/staking/types"
 )
diff --git a/rpc/boot/boot.go b/rpc/boot/boot.go
new file mode 100644
index 0000000000..68756207b8
--- /dev/null
+++ b/rpc/boot/boot.go
@@ -0,0 +1,58 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/harmony-one/harmony/eth/rpc"
+	hmyboot "github.com/harmony-one/harmony/hmy_boot"
+)
+
+// PublicBootService provides an API to access Harmony related information.
+// It offers only methods that operate on public data that is freely available to anyone.
+type PublicBootService struct {
+	hmyboot *hmyboot.BootService
+	version Version
+}
+
+// NewPublicBootAPI creates a new API for the RPC interface
+func NewPublicBootAPI(hmyboot *hmyboot.BootService, version Version) rpc.API {
+	return rpc.API{
+		Namespace: version.Namespace(),
+		Version:   APIVersion,
+		Service:   &PublicBootService{hmyboot, version},
+		Public:    true,
+	}
+}
+
+// ProtocolVersion returns the current Harmony protocol version this node supports
+// Note that the return type is an interface to account for the different versions
+func (s *PublicBootService) ProtocolVersion(
+	ctx context.Context,
+) (interface{}, error) {
+	// Format response according to version
+	switch s.version {
+	case V1, Eth:
+		return hexutil.Uint(s.hmyboot.ProtocolVersion()), nil
+	case V2:
+		return s.hmyboot.ProtocolVersion(), nil
+	default:
+		return nil, ErrUnknownRPCVersion
+	}
+}
+
+// GetNodeMetadata produces a NodeMetadata record, data is from the answering RPC node
+func (s *PublicBootService) GetNodeMetadata(
+	ctx context.Context,
+) (StructuredResponse, error) {
+	// Response output is the same for all versions
+	return NewStructuredResponse(s.hmyboot.GetNodeMetadata())
+}
+
+// GetPeerInfo produces a NodePeerInfo record
+func (s *PublicBootService) GetPeerInfo(
+	ctx context.Context,
+) (StructuredResponse, error) {
+	// Response output is the same for all versions
+	return NewStructuredResponse(s.hmyboot.GetPeerInfo())
+}
diff --git a/rpc/boot/common/types.go b/rpc/boot/common/types.go
new file mode 100644
index 0000000000..e2c4c80005
--- /dev/null
+++ b/rpc/boot/common/types.go
@@ -0,0 +1,40 @@
+package rpc
+
+import (
+	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/internal/params"
+	"github.com/libp2p/go-libp2p/core/peer"
+)
+
+// StructuredResponse type of RPCs
+type StructuredResponse = map[string]interface{}
+
+type C struct {
+	TotalKnownPeers int `json:"total-known-peers"`
+	Connected       int `json:"connected"`
+	NotConnected    int `json:"not-connected"`
+}
+
+// BootNodeMetadata captures select metadata of the RPC answering boot node
+type BootNodeMetadata struct {
+	Version      string  `json:"version"`
+	Network      string  `json:"network"`
+	NodeBootTime int64   `json:"node-unix-start-time"`
+	PeerID       peer.ID `json:"peerid"`
+	C            C       `json:"p2p-connectivity"`
+}
+
+// BootNodePeerInfo captures the peer connectivity info of the boot node
+type BootNodePeerInfo struct {
+	PeerID         peer.ID   `json:"peerid"`
+	KnownPeers     []peer.ID `json:"known-peers"`
+	ConnectedPeers []peer.ID `json:"connected-peers"`
+	C              C         `json:"c"`
+}
+
+type Config struct {
+	HarmonyConfig harmonyconfig.HarmonyConfig
+	NodeConfig    nodeconfig.ConfigType
+	ChainConfig   params.ChainConfig
+}
diff --git a/rpc/boot/error.go b/rpc/boot/error.go
new file mode 100644
index 0000000000..8ec9d73d4b
--- /dev/null
+++ b/rpc/boot/error.go
@@ -0,0 +1,18 @@
+package rpc
+
+import (
+	"errors"
+)
+
+var (
+	// ErrInvalidLogLevel when invalid log level is provided
+	ErrInvalidLogLevel = errors.New("invalid log level")
+	// ErrIncorrectChainID when ChainID does not match running node
+	ErrIncorrectChainID = errors.New("incorrect chain id")
+	// ErrInvalidChainID when ChainID of signer does not match that of running node
+	ErrInvalidChainID = errors.New("invalid chain id for signer")
+	// ErrNotBeaconShard when rpc is called on not beacon chain node
+	ErrNotBeaconShard = errors.New("cannot call this rpc on non beaconchain node")
+	// ErrUnknownRPCVersion when rpc method has an unknown or unhandled version
+	ErrUnknownRPCVersion = errors.New("API service has an unknown version")
+)
diff --git a/rpc/boot/rpc.go b/rpc/boot/rpc.go
new file mode 100644
index 0000000000..a9da9ca119
--- /dev/null
+++ b/rpc/boot/rpc.go
@@ -0,0 +1,167 @@
+package rpc
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/harmony-one/harmony/eth/rpc"
+	hmyboot "github.com/harmony-one/harmony/hmy_boot"
+	bootnodeConfigs "github.com/harmony-one/harmony/internal/configs/bootnode"
+	"github.com/harmony-one/harmony/internal/configs/harmony"
+	"github.com/harmony-one/harmony/internal/utils"
+)
+
+// Version enum
+const (
+	V1 Version = iota
+	V2
+	Eth
+	Debug
+	Trace
+)
+
+const (
+	// APIVersion used for DApp's, bumped after RPC refactor (7/2020)
+	APIVersion = "1.0"
+	// LogTag is the tag found in the log for all RPC logs
+	LogTag = "[BOOT_RPC]"
+	// HTTPPortOffset ..
+	HTTPPortOffset = 500
+	// WSPortOffset ..
+	WSPortOffset = 800
+)
+
+var (
+	// HTTPModules ..
+	HTTPModules = []string{"hmyboot", "hmybootv2", "eth", "debug", "trace"}
+	// WSModules ..
+	WSModules = []string{"hmyboot", "hmybootv2", "eth", "debug", "trace"}
+
+	httpListener     net.Listener
+	httpHandler      *rpc.Server
+	wsListener       net.Listener
+	wsHandler        *rpc.Server
+	httpEndpoint     = ""
+	wsEndpoint       = ""
+	httpVirtualHosts = []string{"*"}
+	httpOrigins      = []string{"*"}
+	wsOrigins        = []string{"*"}
+)
+
+// Version of the RPC
+type Version int
+
+// Namespace of the RPC version
+func (n Version) Namespace() string {
+	return HTTPModules[n]
+}
+
+// StartServers starts the http & ws servers
+func StartServers(hmyboot *hmyboot.BootService, apis []rpc.API, config bootnodeConfigs.RPCServerConfig, rpcOpt harmony.RpcOptConfig) error {
+	apis = append(apis, getBootAPIs(hmyboot, config)...)
+	// load method filter from file (if exist)
+	var rmf rpc.RpcMethodFilter
+	rpcFilterFilePath := strings.TrimSpace(rpcOpt.RpcFilterFile)
+	if len(rpcFilterFilePath) > 0 {
+		if err := rmf.LoadRpcMethodFiltersFromFile(rpcFilterFilePath); err != nil {
+			return err
+		}
+	} else {
+		rmf.ExposeAll()
+	}
+
+	if config.HTTPEnabled {
+		timeouts := rpc.HTTPTimeouts{
+			ReadTimeout:  config.HTTPTimeoutRead,
+			WriteTimeout: config.HTTPTimeoutWrite,
+			IdleTimeout:  config.HTTPTimeoutIdle,
+		}
+		httpEndpoint = fmt.Sprintf("%v:%v", config.HTTPIp, config.HTTPPort)
+		if err := startBootServiceHTTP(apis, &rmf, timeouts); err != nil {
+			return err
+		}
+	}
+
+	if config.WSEnabled {
+		wsEndpoint = fmt.Sprintf("%v:%v", config.WSIp, config.WSPort)
+		if err := startBootServiceWS(apis, &rmf); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// StopServers stops the http & ws servers
+func StopServers() error {
+	if httpListener != nil {
+		if err := httpListener.Close(); err != nil {
+			return err
+		}
+		httpListener = nil
+		utils.Logger().Info().
+			Str("url", fmt.Sprintf("http://%s", httpEndpoint)).
+			Msg("HTTP endpoint closed")
+	}
+	if httpHandler != nil {
+		httpHandler.Stop()
+		httpHandler = nil
+	}
+	if wsListener != nil {
+		if err := wsListener.Close(); err != nil {
+			return err
+		}
+		wsListener = nil
+		utils.Logger().Info().
+			Str("url", fmt.Sprintf("http://%s", wsEndpoint)).
+			Msg("WS endpoint closed")
+	}
+	if wsHandler != nil {
+		wsHandler.Stop()
+		wsHandler = nil
+	}
+	return nil
+}
+
+// getBootAPIs returns all the API methods for the RPC interface
+func getBootAPIs(hmyboot *hmyboot.BootService, config bootnodeConfigs.RPCServerConfig) []rpc.API {
+	publicAPIs := []rpc.API{
+		// Public methods
+		NewPublicBootAPI(hmyboot, V1),
+		NewPublicBootAPI(hmyboot, V2),
+	}
+
+	return publicAPIs
+}
+
+func startBootServiceHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter, httpTimeouts rpc.HTTPTimeouts) (err error) {
+	httpListener, httpHandler, err = rpc.StartHTTPEndpoint(
+		httpEndpoint, apis, HTTPModules, rmf, httpOrigins, httpVirtualHosts, httpTimeouts,
+	)
+	if err != nil {
+		return err
+	}
+
+	utils.Logger().Info().
+		Str("url", fmt.Sprintf("http://%s", httpEndpoint)).
+		Str("cors", strings.Join(httpOrigins, ",")).
+		Str("vhosts", strings.Join(httpVirtualHosts, ",")).
+		Msg("HTTP endpoint opened")
+
+	fmt.Printf("Started Boot Node RPC server at: %v\n", httpEndpoint)
+	return nil
+}
+
+func startBootServiceWS(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) {
+	wsListener, wsHandler, err = rpc.StartWSEndpoint(wsEndpoint, apis, WSModules, rmf, wsOrigins, true)
+	if err != nil {
+		return err
+	}
+
+	utils.Logger().Info().
+		Str("url", fmt.Sprintf("ws://%s", wsEndpoint)).
+		Msg("WebSocket endpoint opened")
+	fmt.Printf("Started Boot Node WS server at: %v\n", wsEndpoint)
+	return nil
+}
diff --git a/rpc/boot/types.go b/rpc/boot/types.go
new file mode 100644
index 0000000000..014bda54e9
--- /dev/null
+++ b/rpc/boot/types.go
@@ -0,0 +1,27 @@
+package rpc
+
+import (
+	"bytes"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+// StructuredResponse type of RPCs
+type StructuredResponse = map[string]interface{}
+
+// NewStructuredResponse creates a structured response from the given input
+func NewStructuredResponse(input interface{}) (StructuredResponse, error) {
+	var objMap StructuredResponse
+	var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary
+	dat, err := jsonIter.Marshal(input)
+	if err != nil {
+		return nil, err
+	}
+	d := jsonIter.NewDecoder(bytes.NewReader(dat))
+	d.UseNumber()
+	err = d.Decode(&objMap)
+	if err != nil {
+		return nil, err
+	}
+	return objMap, nil
+}
diff --git a/rpc/blockchain.go b/rpc/harmony/blockchain.go
similarity index 99%
rename from rpc/blockchain.go
rename to rpc/harmony/blockchain.go
index c9a6a13138..64f84c709b 100644
--- a/rpc/blockchain.go
+++ b/rpc/harmony/blockchain.go
@@ -26,10 +26,10 @@ import (
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/numeric"
-	rpc_common "github.com/harmony-one/harmony/rpc/common"
-	eth "github.com/harmony-one/harmony/rpc/eth"
-	v1 "github.com/harmony-one/harmony/rpc/v1"
-	v2 "github.com/harmony-one/harmony/rpc/v2"
+	rpc_common "github.com/harmony-one/harmony/rpc/harmony/common"
+	eth "github.com/harmony-one/harmony/rpc/harmony/eth"
+	v1 "github.com/harmony-one/harmony/rpc/harmony/v1"
+	v2 "github.com/harmony-one/harmony/rpc/harmony/v2"
 	"github.com/harmony-one/harmony/shard"
 	stakingReward "github.com/harmony-one/harmony/staking/reward"
 )
diff --git a/rpc/common/block.go b/rpc/harmony/common/block.go
similarity index 100%
rename from rpc/common/block.go
rename to rpc/harmony/common/block.go
diff --git a/rpc/common/hacks.go b/rpc/harmony/common/hacks.go
similarity index 100%
rename from rpc/common/hacks.go
rename to rpc/harmony/common/hacks.go
diff --git a/rpc/common/types.go b/rpc/harmony/common/types.go
similarity index 100%
rename from rpc/common/types.go
rename to rpc/harmony/common/types.go
diff --git a/rpc/contract.go b/rpc/harmony/contract.go
similarity index 100%
rename from rpc/contract.go
rename to rpc/harmony/contract.go
diff --git a/rpc/error.go b/rpc/harmony/error.go
similarity index 100%
rename from rpc/error.go
rename to rpc/harmony/error.go
diff --git a/rpc/eth/block.go b/rpc/harmony/eth/block.go
similarity index 96%
rename from rpc/eth/block.go
rename to rpc/harmony/eth/block.go
index 783734b74e..900fa62c21 100644
--- a/rpc/eth/block.go
+++ b/rpc/harmony/eth/block.go
@@ -3,7 +3,7 @@ package eth
 import (
 	"github.com/harmony-one/harmony/core/types"
 	internal_common "github.com/harmony-one/harmony/internal/common"
-	rpc_common "github.com/harmony-one/harmony/rpc/common"
+	rpc_common "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/pkg/errors"
 )
 
diff --git a/rpc/eth/rpc.go b/rpc/harmony/eth/rpc.go
similarity index 100%
rename from rpc/eth/rpc.go
rename to rpc/harmony/eth/rpc.go
diff --git a/rpc/eth/types.go b/rpc/harmony/eth/types.go
similarity index 100%
rename from rpc/eth/types.go
rename to rpc/harmony/eth/types.go
diff --git a/rpc/filters/api.go b/rpc/harmony/filters/api.go
similarity index 99%
rename from rpc/filters/api.go
rename to rpc/harmony/filters/api.go
index 84c171cc10..e95b81a881 100644
--- a/rpc/filters/api.go
+++ b/rpc/harmony/filters/api.go
@@ -6,6 +6,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/utils"
 
 	"github.com/harmony-one/harmony/internal/tikv/redis_helper"
@@ -14,8 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/harmony-one/harmony/block"
 	"github.com/harmony-one/harmony/core/types"
-	"github.com/harmony-one/harmony/eth/rpc"
-	hmy_rpc "github.com/harmony-one/harmony/rpc"
+	hmy_rpc "github.com/harmony-one/harmony/rpc/harmony"
 )
 
 var (
diff --git a/rpc/filters/filter.go b/rpc/harmony/filters/filter.go
similarity index 100%
rename from rpc/filters/filter.go
rename to rpc/harmony/filters/filter.go
diff --git a/rpc/filters/filter_criteria.go b/rpc/harmony/filters/filter_criteria.go
similarity index 100%
rename from rpc/filters/filter_criteria.go
rename to rpc/harmony/filters/filter_criteria.go
diff --git a/rpc/filters/filter_system.go b/rpc/harmony/filters/filter_system.go
similarity index 100%
rename from rpc/filters/filter_system.go
rename to rpc/harmony/filters/filter_system.go
diff --git a/rpc/harmony.go b/rpc/harmony/harmony.go
similarity index 100%
rename from rpc/harmony.go
rename to rpc/harmony/harmony.go
diff --git a/rpc/metrics.go b/rpc/harmony/metrics.go
similarity index 100%
rename from rpc/metrics.go
rename to rpc/harmony/metrics.go
diff --git a/rpc/net.go b/rpc/harmony/net.go
similarity index 100%
rename from rpc/net.go
rename to rpc/harmony/net.go
diff --git a/rpc/pool.go b/rpc/harmony/pool.go
similarity index 98%
rename from rpc/pool.go
rename to rpc/harmony/pool.go
index ee4e34828f..91eea75b0a 100644
--- a/rpc/pool.go
+++ b/rpc/harmony/pool.go
@@ -20,9 +20,9 @@ import (
 	common2 "github.com/harmony-one/harmony/internal/common"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
-	eth "github.com/harmony-one/harmony/rpc/eth"
-	v1 "github.com/harmony-one/harmony/rpc/v1"
-	v2 "github.com/harmony-one/harmony/rpc/v2"
+	eth "github.com/harmony-one/harmony/rpc/harmony/eth"
+	v1 "github.com/harmony-one/harmony/rpc/harmony/v1"
+	v2 "github.com/harmony-one/harmony/rpc/harmony/v2"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
diff --git a/rpc/preimages.go b/rpc/harmony/preimages.go
similarity index 100%
rename from rpc/preimages.go
rename to rpc/harmony/preimages.go
diff --git a/rpc/private_debug.go b/rpc/harmony/private_debug.go
similarity index 100%
rename from rpc/private_debug.go
rename to rpc/harmony/private_debug.go
diff --git a/rpc/public_debug.go b/rpc/harmony/public_debug.go
similarity index 100%
rename from rpc/public_debug.go
rename to rpc/harmony/public_debug.go
diff --git a/rpc/rpc.go b/rpc/harmony/rpc.go
similarity index 98%
rename from rpc/rpc.go
rename to rpc/harmony/rpc.go
index 15d7d93baf..a216f9843f 100644
--- a/rpc/rpc.go
+++ b/rpc/harmony/rpc.go
@@ -10,9 +10,9 @@ import (
 	"github.com/harmony-one/harmony/internal/configs/harmony"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
-	eth "github.com/harmony-one/harmony/rpc/eth"
-	v1 "github.com/harmony-one/harmony/rpc/v1"
-	v2 "github.com/harmony-one/harmony/rpc/v2"
+	eth "github.com/harmony-one/harmony/rpc/harmony/eth"
+	v1 "github.com/harmony-one/harmony/rpc/harmony/v1"
+	v2 "github.com/harmony-one/harmony/rpc/harmony/v2"
 )
 
 // Version enum
diff --git a/rpc/staking.go b/rpc/harmony/staking.go
similarity index 100%
rename from rpc/staking.go
rename to rpc/harmony/staking.go
diff --git a/rpc/tracer.go b/rpc/harmony/tracer.go
similarity index 100%
rename from rpc/tracer.go
rename to rpc/harmony/tracer.go
diff --git a/rpc/tracerParity.go b/rpc/harmony/tracerParity.go
similarity index 100%
rename from rpc/tracerParity.go
rename to rpc/harmony/tracerParity.go
diff --git a/rpc/transaction.go b/rpc/harmony/transaction.go
similarity index 99%
rename from rpc/transaction.go
rename to rpc/harmony/transaction.go
index 4b45045851..5191566a36 100644
--- a/rpc/transaction.go
+++ b/rpc/harmony/transaction.go
@@ -20,9 +20,9 @@ import (
 	internal_common "github.com/harmony-one/harmony/internal/common"
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/internal/utils"
-	eth "github.com/harmony-one/harmony/rpc/eth"
-	v1 "github.com/harmony-one/harmony/rpc/v1"
-	v2 "github.com/harmony-one/harmony/rpc/v2"
+	eth "github.com/harmony-one/harmony/rpc/harmony/eth"
+	v1 "github.com/harmony-one/harmony/rpc/harmony/v1"
+	v2 "github.com/harmony-one/harmony/rpc/harmony/v2"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
diff --git a/rpc/types.go b/rpc/harmony/types.go
similarity index 100%
rename from rpc/types.go
rename to rpc/harmony/types.go
index 7bcd931831..e3a97afda4 100644
--- a/rpc/types.go
+++ b/rpc/harmony/types.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/harmony-one/harmony/eth/rpc"
 	internal_common "github.com/harmony-one/harmony/internal/common"
 	"github.com/pkg/errors"
 
@@ -19,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/harmony-one/harmony/block"
 	"github.com/harmony-one/harmony/core/types"
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
diff --git a/rpc/types_test.go b/rpc/harmony/types_test.go
similarity index 100%
rename from rpc/types_test.go
rename to rpc/harmony/types_test.go
diff --git a/rpc/utils/utils.go b/rpc/harmony/utils/utils.go
similarity index 100%
rename from rpc/utils/utils.go
rename to rpc/harmony/utils/utils.go
diff --git a/rpc/v1/block.go b/rpc/harmony/v1/block.go
similarity index 96%
rename from rpc/v1/block.go
rename to rpc/harmony/v1/block.go
index 714816f91c..5437a07812 100644
--- a/rpc/v1/block.go
+++ b/rpc/harmony/v1/block.go
@@ -2,7 +2,7 @@ package v1
 
 import (
 	"github.com/harmony-one/harmony/core/types"
-	rpc_common "github.com/harmony-one/harmony/rpc/common"
+	rpc_common "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/pkg/errors"
 )
 
diff --git a/rpc/v1/legacy.go b/rpc/harmony/v1/legacy.go
similarity index 100%
rename from rpc/v1/legacy.go
rename to rpc/harmony/v1/legacy.go
diff --git a/rpc/v1/types.go b/rpc/harmony/v1/types.go
similarity index 100%
rename from rpc/v1/types.go
rename to rpc/harmony/v1/types.go
diff --git a/rpc/v2/block.go b/rpc/harmony/v2/block.go
similarity index 96%
rename from rpc/v2/block.go
rename to rpc/harmony/v2/block.go
index 0a1fed60ed..5c1b975a29 100644
--- a/rpc/v2/block.go
+++ b/rpc/harmony/v2/block.go
@@ -2,7 +2,7 @@ package v2
 
 import (
 	"github.com/harmony-one/harmony/core/types"
-	rpc_common "github.com/harmony-one/harmony/rpc/common"
+	rpc_common "github.com/harmony-one/harmony/rpc/harmony/common"
 	"github.com/pkg/errors"
 )
 
diff --git a/rpc/v2/legacy.go b/rpc/harmony/v2/legacy.go
similarity index 100%
rename from rpc/v2/legacy.go
rename to rpc/harmony/v2/legacy.go
diff --git a/rpc/v2/types.go b/rpc/harmony/v2/types.go
similarity index 100%
rename from rpc/v2/types.go
rename to rpc/harmony/v2/types.go
diff --git a/rpc/web3.go b/rpc/harmony/web3.go
similarity index 100%
rename from rpc/web3.go
rename to rpc/harmony/web3.go
diff --git a/test/chain/main.go b/test/chain/main.go
index 4b935292f0..e523c0359a 100644
--- a/test/chain/main.go
+++ b/test/chain/main.go
@@ -19,7 +19,7 @@ import (
 	"github.com/harmony-one/harmony/core/vm"
 	"github.com/harmony-one/harmony/crypto/hash"
 	"github.com/harmony-one/harmony/internal/params"
-	pkgworker "github.com/harmony-one/harmony/node/worker"
+	pkgworker "github.com/harmony-one/harmony/node/harmony/worker"
 )
 
 const (
diff --git a/test/deploy.sh b/test/deploy.sh
index 470a4b4f13..0f3d3f56ad 100755
--- a/test/deploy.sh
+++ b/test/deploy.sh
@@ -54,7 +54,7 @@ function setup() {
 
 function launch_bootnode() {
   echo "launching boot node ..."
-  ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -max_conn_per_ip 100 -force_public true >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" &
+  ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -rpc_http_port 8888 -rpc_ws_port 8889 -network "localnet" -max_conn_per_ip 100 -force_public true >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" &
   sleep 1
   BN_MA=$(grep "BN_MA" "${log_folder}"/bootnode.log | awk -F\= ' { print $2 } ')
   echo "bootnode launched." + " $BN_MA"
diff --git a/test/helpers/transaction.go b/test/helpers/transaction.go
index 11efd386f0..9367a957da 100644
--- a/test/helpers/transaction.go
+++ b/test/helpers/transaction.go
@@ -9,7 +9,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 
 	hmytypes "github.com/harmony-one/harmony/core/types"
-	rpcV2 "github.com/harmony-one/harmony/rpc/v2"
+	rpcV2 "github.com/harmony-one/harmony/rpc/harmony/v2"
 	stakingTypes "github.com/harmony-one/harmony/staking/types"
 )
 

From 3e23739cf8d197077f413b4f27843fa9af5091f3 Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Wed, 11 Sep 2024 17:08:33 +0300
Subject: [PATCH 10/24] feat(Makefile) - cover verbose mode for the local debug
 (#4755)

---
 Makefile               | 4 ++++
 test/debug-external.sh | 7 ++++++-
 test/debug.sh          | 7 ++++++-
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index ce00f80d08..392b1126b7 100644
--- a/Makefile
+++ b/Makefile
@@ -73,6 +73,8 @@ debug:
 	# uncomment the following lines to enable debug logging for libp2p, it produces a lot of logs, so disabled by default
 	#export GOLOG_LOG_LEVEL=debug
 	#export GOLOG_OUTPUT=stdout
+	# uncomment the following line to enable max logging
+	# export VERBOSE=true
 	bash ./test/debug.sh
 
 debug-kill:
@@ -82,6 +84,8 @@ debug-ext:
 	# update localnet block per epoch to ensure a stable localnet
 	sed -i 's/localnetBlocksPerEpoch\s*=\s*[0-9]*/localnetBlocksPerEpoch = 64/' internal/configs/sharding/localnet.go
 	sed -i 's/localnetBlocksPerEpochV2\s*=\s*[0-9]*/localnetBlocksPerEpochV2 = 64/' internal/configs/sharding/localnet.go
+	# uncomment the following line to enable max logging
+	# export VERBOSE=true
 	bash ./test/debug-external.sh &
 	echo sleep 10s before creating the external validator
 	sleep 10
diff --git a/test/debug-external.sh b/test/debug-external.sh
index 27966e4784..301974f456 100755
--- a/test/debug-external.sh
+++ b/test/debug-external.sh
@@ -5,4 +5,9 @@ rm -rf tmp_log* 2> /dev/null
 rm *.rlp 2> /dev/null
 rm -rf .dht* 2> /dev/null
 scripts/go_executable_build.sh -S || exit 1  # dynamic builds are faster for debug iteration...
-./test/deploy.sh -B -D 600000 ./test/configs/local-resharding-with-external.txt
+if [ "${VERBOSE}" = "true" ]; then
+    echo "[WARN] - running with verbose logs"
+    ./test/deploy.sh -v -B -D 600000 ./test/configs/local-resharding-with-external.txt
+else
+    ./test/deploy.sh -B -D 600000 ./test/configs/local-resharding-with-external.txt
+fi
diff --git a/test/debug.sh b/test/debug.sh
index 1513a645bf..fd097f6392 100755
--- a/test/debug.sh
+++ b/test/debug.sh
@@ -5,4 +5,9 @@ rm -rf tmp_log* 2> /dev/null
 rm *.rlp 2> /dev/null
 rm -rf .dht* 2> /dev/null
 scripts/go_executable_build.sh -S || exit 1  # dynamic builds are faster for debug iteration...
-./test/deploy.sh -B -D 600000 ./test/configs/local-resharding.txt
+if [ "${VERBOSE}" = "true" ]; then
+    echo "[WARN] - running with verbose logs"
+    ./test/deploy.sh -v -B -D 600000 ./test/configs/local-resharding.txt
+else
+    ./test/deploy.sh -B -D 600000 ./test/configs/local-resharding.txt
+fi

From e08df0c96cabbcd70b9fa3501521b06ecb679896 Mon Sep 17 00:00:00 2001
From: Gheis Mohammadi <Gheis.Mohammadi@gmail.com>
Date: Fri, 20 Sep 2024 16:38:40 +0800
Subject: [PATCH 11/24] Improve Crosslinks Retrieving and Processing (#4754)

* add mutex to read cross links
* refactor process crosslinks to handle maximum pending crosslinks
* fix not found issue for pending crosslinks
---
 core/blockchain_impl.go         | 21 +++++++++++++++++----
 node/harmony/node_cross_link.go | 10 +++++-----
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go
index a065ecfd6d..dbc12e91a6 100644
--- a/core/blockchain_impl.go
+++ b/core/blockchain_impl.go
@@ -68,6 +68,7 @@ import (
 	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
 	lru "github.com/hashicorp/golang-lru"
+	goleveldb "github.com/syndtr/goleveldb/leveldb"
 )
 
 var (
@@ -2435,7 +2436,7 @@ func (bc *BlockChainImpl) ReadPendingSlashingCandidates() slash.Records {
 	return append(bc.pendingSlashes[0:0], bc.pendingSlashes...)
 }
 
-func (bc *BlockChainImpl) ReadPendingCrossLinks() ([]types.CrossLink, error) {
+func (bc *BlockChainImpl) readPendingCrossLinks() ([]types.CrossLink, error) {
 	cls := []types.CrossLink{}
 	bytes := []byte{}
 	if cached, ok := bc.pendingCrossLinksCache.Get(pendingCLCacheKey); ok {
@@ -2443,8 +2444,12 @@ func (bc *BlockChainImpl) ReadPendingCrossLinks() ([]types.CrossLink, error) {
 		return cls, nil
 	} else {
 		by, err := rawdb.ReadPendingCrossLinks(bc.db)
-		if err != nil || len(by) == 0 {
+		if err == goleveldb.ErrNotFound {
+			return []types.CrossLink{}, nil
+		} else if err != nil {
 			return nil, err
+		} else if len(by) == 0 {
+			return []types.CrossLink{}, nil
 		}
 		bytes = by
 	}
@@ -2525,11 +2530,19 @@ func (bc *BlockChainImpl) AddPendingSlashingCandidates(
 	return bc.writeSlashes(bc.pendingSlashes)
 }
 
+// ReadPendingCrossLinks returns pending crosslinks
+func (bc *BlockChainImpl) ReadPendingCrossLinks() ([]types.CrossLink, error) {
+	bc.pendingCrossLinksMutex.Lock()
+	defer bc.pendingCrossLinksMutex.Unlock()
+
+	return bc.readPendingCrossLinks()
+}
+
 func (bc *BlockChainImpl) AddPendingCrossLinks(pendingCLs []types.CrossLink) (int, error) {
 	bc.pendingCrossLinksMutex.Lock()
 	defer bc.pendingCrossLinksMutex.Unlock()
 
-	cls, err := bc.ReadPendingCrossLinks()
+	cls, err := bc.readPendingCrossLinks()
 	if err != nil || len(cls) == 0 {
 		err := bc.CachePendingCrossLinks(pendingCLs)
 		return len(pendingCLs), err
@@ -2543,7 +2556,7 @@ func (bc *BlockChainImpl) DeleteFromPendingCrossLinks(crossLinks []types.CrossLi
 	bc.pendingCrossLinksMutex.Lock()
 	defer bc.pendingCrossLinksMutex.Unlock()
 
-	cls, err := bc.ReadPendingCrossLinks()
+	cls, err := bc.readPendingCrossLinks()
 	if err != nil || len(cls) == 0 {
 		return 0, err
 	}
diff --git a/node/harmony/node_cross_link.go b/node/harmony/node_cross_link.go
index 61b57a4fff..26a3509e75 100644
--- a/node/harmony/node_cross_link.go
+++ b/node/harmony/node_cross_link.go
@@ -103,16 +103,16 @@ func (node *Node) processCrossLinkHeartbeatMessage(msgPayload []byte) error {
 func (node *Node) ProcessCrossLinkMessage(msgPayload []byte) {
 	if node.IsRunningBeaconChain() {
 		pendingCLs, err := node.Blockchain().ReadPendingCrossLinks()
-		if err != nil {
+		if err == nil && len(pendingCLs) >= maxPendingCrossLinkSize {
 			utils.Logger().Debug().
-				Err(err).
 				Msgf("[ProcessingCrossLink] Pending Crosslink reach maximum size: %d", len(pendingCLs))
 			return
 		}
-		if len(pendingCLs) >= maxPendingCrossLinkSize {
+		if err != nil {
 			utils.Logger().Debug().
-				Msgf("[ProcessingCrossLink] Pending Crosslink reach maximum size: %d", len(pendingCLs))
-			return
+				Err(err).
+				Int("num crosslinks", len(pendingCLs)).
+				Msg("[ProcessingCrossLink] Read Pending Crosslink failed")
 		}
 
 		existingCLs := map[common2.Hash]struct{}{}

From 84672e8bc974b14d20c290e45636a59f9f84edda Mon Sep 17 00:00:00 2001
From: Frozen <355847+Frozen@users.noreply.github.com>
Date: Fri, 20 Sep 2024 04:43:28 -0400
Subject: [PATCH 12/24] Fix for localnet syncing ports. (#4753)

* Fix for localnet syncing ports.
* Cleanup.
* align external localnet ports with internal

---------

Co-authored-by: Nita Neou (Soph) <soph@harmony.one>
---
 node/harmony/node_syncing.go                  | 11 ++--
 node/harmony/node_test.go                     | 10 ++--
 .../local-resharding-with-external.txt        | 50 +++++++++----------
 test/deploy.sh                                |  1 +
 4 files changed, 37 insertions(+), 35 deletions(-)

diff --git a/node/harmony/node_syncing.go b/node/harmony/node_syncing.go
index b1ee21ea7d..15f097e2ec 100644
--- a/node/harmony/node_syncing.go
+++ b/node/harmony/node_syncing.go
@@ -206,10 +206,13 @@ func (p *LocalSyncingPeerProvider) SyncingPeers(shardID uint32) (peers []p2p.Pee
 		return nil, errors.Errorf(
 			"shard ID %d out of range 0..%d", shardID, p.numShards-1)
 	}
-	firstPort := uint32(p.basePort) + shardID
-	endPort := uint32(p.basePort) + p.numShards*p.shardSize
-	for port := firstPort; port < endPort; port += p.numShards {
-		if port == uint32(p.selfPort) {
+	shards := [][]string{
+		{"6000", "6004", "6008", "6012", "6016", "6020", "6100", "6104", "6108", "6112", "6116", "6120"},
+		{"6002", "6006", "6010", "6014", "6018", "6022", "6102", "6106", "6110", "6114", "6118", "6122"},
+	}
+	selfport := fmt.Sprint(p.selfPort)
+	for _, port := range shards[shardID] {
+		if port == selfport {
 			continue // do not sync from self
 		}
 		peers = append(peers, p2p.Peer{IP: "127.0.0.1", Port: fmt.Sprint(port)})
diff --git a/node/harmony/node_test.go b/node/harmony/node_test.go
index d96a266241..f77353d49d 100644
--- a/node/harmony/node_test.go
+++ b/node/harmony/node_test.go
@@ -113,22 +113,22 @@ func TestLocalSyncingPeerProvider(t *testing.T) {
 		p := makeLocalSyncingPeerProvider()
 		expectedBeaconPeers := []p2p.Peer{
 			{IP: "127.0.0.1", Port: "6000"},
-			{IP: "127.0.0.1", Port: "6002"},
 			{IP: "127.0.0.1", Port: "6004"},
+			{IP: "127.0.0.1", Port: "6008"},
 		}
 		if actualPeers, err := p.SyncingPeers(0); assert.NoError(t, err) {
-			assert.ElementsMatch(t, actualPeers, expectedBeaconPeers)
+			assert.ElementsMatch(t, actualPeers[:3], expectedBeaconPeers)
 		}
 	})
 	t.Run("Shard1Chain", func(t *testing.T) {
 		p := makeLocalSyncingPeerProvider()
 		expectedShard1Peers := []p2p.Peer{
 			// port 6001 omitted because self
-			{IP: "127.0.0.1", Port: "6003"},
-			{IP: "127.0.0.1", Port: "6005"},
+			{IP: "127.0.0.1", Port: "6002"},
+			{IP: "127.0.0.1", Port: "6006"},
 		}
 		if actualPeers, err := p.SyncingPeers(1); assert.NoError(t, err) {
-			assert.ElementsMatch(t, actualPeers, expectedShard1Peers)
+			assert.ElementsMatch(t, actualPeers[:2], expectedShard1Peers)
 		}
 	})
 	t.Run("InvalidShard", func(t *testing.T) {
diff --git a/test/configs/local-resharding-with-external.txt b/test/configs/local-resharding-with-external.txt
index bb12da4c49..d146dd2678 100644
--- a/test/configs/local-resharding-with-external.txt
+++ b/test/configs/local-resharding-with-external.txt
@@ -1,37 +1,35 @@
 # shard 0
 # internal node
 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key
-127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key
-127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key
-127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key
-127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key
-127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key
-# external node
-127.0.0.1 9014 external  .hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key
-127.0.0.1 9016 external  .hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key
-127.0.0.1 9018 external  .hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key
+127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key
+127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key
+127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key
+127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key
+127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key
 # fn node
-127.0.0.1 9050 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key
-127.0.0.1 9052 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key
-# 127.0.0.1 9054 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key
+127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key
+127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key
+# external node
+127.0.0.1 9108 external  .hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key
+127.0.0.1 9112 external  .hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key
+127.0.0.1 9116 external  .hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key
 # explorer node
-127.0.0.1 9098 explorer null 0
+127.0.0.1 9120 explorer null 0
 
 # shard 1
 # internal node
-127.0.0.1 9100 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key
-127.0.0.1 9102 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key
-127.0.0.1 9104 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key
-127.0.0.1 9106 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key
-127.0.0.1 9108 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key
-127.0.0.1 9110 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key
+127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key
+127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key
+127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key
+127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key
+127.0.0.1 9118 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key
+127.0.0.1 9122 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key
+# fn node
+127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key
+127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key
 # external node
-127.0.0.1 9114 external  .hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key
-127.0.0.1 9116 external  .hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key
+127.0.0.1 9110 external  .hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key
+127.0.0.1 9114 external  .hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key
 127.0.0.1 9118 external  .hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key
-# fn node
-127.0.0.1 9150 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key
-127.0.0.1 9152 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key
-# 127.0.0.1 9154 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key
 # explorer node
-127.0.0.1 9096 explorer null 1
+127.0.0.1 9122 explorer null 1
diff --git a/test/deploy.sh b/test/deploy.sh
index 0f3d3f56ad..daf48df802 100755
--- a/test/deploy.sh
+++ b/test/deploy.sh
@@ -198,3 +198,4 @@ setup
 launch_localnet
 sleep "${DURATION}"
 cleanup || true
+

From 797511c89f6a211bdc5cb1d4c18e06615ce2a9f7 Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Tue, 24 Sep 2024 07:09:44 +0300
Subject: [PATCH 13/24] feat(localnet) - add localnet log aggregation tool
 (#4758)

* feat(localnet) - add localnet log aggregation with Promtail-Loki-Grafana
* feat(log_aggregation) - fix PR comments, add restart command, add anonymus access - it will directly open explore page now, add log commands to the help
---
 .gitignore                                   |  3 ++
 Makefile                                     | 11 +++++++
 scripts/docker/README.md                     |  4 +--
 test/logs_aggregator/docker-compose.yml      | 25 ++++++++++++++
 test/logs_aggregator/loki-config.yml         | 34 ++++++++++++++++++++
 test/logs_aggregator/loki-datasource.yaml    |  9 ++++++
 test/logs_aggregator/promtail-config.yml     | 18 +++++++++++
 test/logs_aggregator/start_log_aggregator.sh | 17 ++++++++++
 test/logs_aggregator/stop_log_aggregator.sh  | 11 +++++++
 9 files changed, 130 insertions(+), 2 deletions(-)
 create mode 100644 test/logs_aggregator/docker-compose.yml
 create mode 100644 test/logs_aggregator/loki-config.yml
 create mode 100644 test/logs_aggregator/loki-datasource.yaml
 create mode 100644 test/logs_aggregator/promtail-config.yml
 create mode 100644 test/logs_aggregator/start_log_aggregator.sh
 create mode 100644 test/logs_aggregator/stop_log_aggregator.sh

diff --git a/.gitignore b/.gitignore
index bbdb537722..5b3554c8d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -98,3 +98,6 @@ profiles/*.pb.gz
 # cache db
 cache/
 cache_*_db
+
+# log aggregation logs folder variable file, no sense to track it
+test/logs_aggregator/.env
diff --git a/Makefile b/Makefile
index 392b1126b7..67ffc4f32c 100644
--- a/Makefile
+++ b/Makefile
@@ -54,6 +54,9 @@ help:
 	@echo "travis_rosetta_checker - run the Travis Rosetta checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH"
 	@echo "debug_external - cleans up environment, rebuilds the binary, and deploys with external nodes"
 	@echo "build_localnet_validator - imports validator keys, funds validator accounts, waits for the epoch, and creates external validators on a local network"
+	@echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, needs docker compose and started localnet"
+	@echo "debug-stop-log - stops a docker compose Promtail->Loki->Grafana stack"
+	@echo "debug-restart-log - restart a docker compose Promtail->Loki->Grafana stack"
 
 libs:
 	make -C $(TOP)/mcl -j8
@@ -216,3 +219,11 @@ build_localnet_validator:
 
 protofiles:
 	bash ./scripts/gogenerate.sh
+
+debug-start-log:
+	bash ./test/logs_aggregator/start_log_aggregator.sh
+
+debug-stop-log:
+	bash ./test/logs_aggregator/stop_log_aggregator.sh
+
+debug-restart-log: debug-stop-log debug-start-log
diff --git a/scripts/docker/README.md b/scripts/docker/README.md
index 1aab228c3f..b232a19887 100644
--- a/scripts/docker/README.md
+++ b/scripts/docker/README.md
@@ -34,7 +34,7 @@ If you need to open another shell, just do:
 docker exec -it harmonyone/harmony /bin/bash
 ```
 
-We also provide a `docker-compose` file for local testing
+We also provide a `docker compose` file for local testing
 
 To use the container in kubernetes, you can use a configmap or secret to mount the `harmony.conf` into the container
 ```bash
@@ -67,4 +67,4 @@ metadata:
 data:
   harmony.conf: |
     ...
-```
\ No newline at end of file
+```
diff --git a/test/logs_aggregator/docker-compose.yml b/test/logs_aggregator/docker-compose.yml
new file mode 100644
index 0000000000..5ca06f64e4
--- /dev/null
+++ b/test/logs_aggregator/docker-compose.yml
@@ -0,0 +1,25 @@
+services:
+  loki:
+    container_name: loki
+    image: grafana/loki:latest
+    ports:
+      - 3100:3100
+    volumes:
+      - ./loki-config.yml:/etc/loki/loki-config.yaml
+  promtail:
+    container_name: promtail
+    image: grafana/promtail:latest
+    volumes:
+      - ./promtail-config.yml:/etc/promtail/promtail-config.yaml
+      - ${CURRENT_SESSION_LOGS}:/var/log/
+  grafana:
+    container_name: grafana
+    image: grafana/grafana
+    # DANGER: USE THIS TWO OPTIONS ONLY AGAINST LOCAL deploy
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    ports:
+      - 3000:3000
+    volumes:
+      - ./loki-datasource.yaml:/etc/grafana/provisioning/datasources/loki-datasource.yaml
diff --git a/test/logs_aggregator/loki-config.yml b/test/logs_aggregator/loki-config.yml
new file mode 100644
index 0000000000..a754536355
--- /dev/null
+++ b/test/logs_aggregator/loki-config.yml
@@ -0,0 +1,34 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 10096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
diff --git a/test/logs_aggregator/loki-datasource.yaml b/test/logs_aggregator/loki-datasource.yaml
new file mode 100644
index 0000000000..6f0ef788b3
--- /dev/null
+++ b/test/logs_aggregator/loki-datasource.yaml
@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    jsonData:
+      maxLines: 1000
diff --git a/test/logs_aggregator/promtail-config.yml b/test/logs_aggregator/promtail-config.yml
new file mode 100644
index 0000000000..5a1547d0d5
--- /dev/null
+++ b/test/logs_aggregator/promtail-config.yml
@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 10080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - "localhost"
+    labels:
+      job: varlogs
+      __path__: /var/log/*.log
diff --git a/test/logs_aggregator/start_log_aggregator.sh b/test/logs_aggregator/start_log_aggregator.sh
new file mode 100644
index 0000000000..8d0d405610
--- /dev/null
+++ b/test/logs_aggregator/start_log_aggregator.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+echo "working in ${DIR}"
+cd $DIR && pwd
+logs="$(ls -1 '../../tmp_log/')"
+path="$(readlink -f '../../tmp_log/')"
+echo "Current localnet logs are placed into '${path}/${logs}'"
+echo "CURRENT_SESSION_LOGS='${path}/${logs}'" > .env
+echo "CURRENT_SESSION_LOGS='${path}/${logs}'"
+echo "starting docker compose lor log aggregation"
+docker compose up --detach
+sleep 5
+echo "Opening Grafana"
+python3 -m webbrowser "http://localhost:3000/explore"
diff --git a/test/logs_aggregator/stop_log_aggregator.sh b/test/logs_aggregator/stop_log_aggregator.sh
new file mode 100644
index 0000000000..bc876fab1e
--- /dev/null
+++ b/test/logs_aggregator/stop_log_aggregator.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+echo "working in ${DIR}"
+cd $DIR && pwd
+echo "[INFO] - stopping log aggregation"
+docker compose rm --force --stop --volumes
+echo "[INFO] - cleanup .env"
+echo > .env

From 991d1a0f266673a0a1e8e9e157cfb28dc364c650 Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Wed, 25 Sep 2024 08:02:12 +0300
Subject: [PATCH 14/24] feat(Makefile) - add switch for the legacy sync, make
 stream sync (#4761)

default option
---
 Makefile               |  8 ++++----
 test/debug-external.sh | 11 +++++++++--
 test/debug.sh          | 11 +++++++++--
 test/deploy.sh         | 23 ++++++++++++++++-------
 4 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 67ffc4f32c..2f7fd619b0 100644
--- a/Makefile
+++ b/Makefile
@@ -76,8 +76,8 @@ debug:
 	# uncomment the following lines to enable debug logging for libp2p, it produces a lot of logs, so disabled by default
 	#export GOLOG_LOG_LEVEL=debug
 	#export GOLOG_OUTPUT=stdout
-	# uncomment the following line to enable max logging
-	# export VERBOSE=true
+	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
+	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
 	bash ./test/debug.sh
 
 debug-kill:
@@ -87,8 +87,8 @@ debug-ext:
 	# update localnet block per epoch to ensure a stable localnet
 	sed -i 's/localnetBlocksPerEpoch\s*=\s*[0-9]*/localnetBlocksPerEpoch = 64/' internal/configs/sharding/localnet.go
 	sed -i 's/localnetBlocksPerEpochV2\s*=\s*[0-9]*/localnetBlocksPerEpochV2 = 64/' internal/configs/sharding/localnet.go
-	# uncomment the following line to enable max logging
-	# export VERBOSE=true
+	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
+	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
 	bash ./test/debug-external.sh &
 	echo sleep 10s before creating the external validator
 	sleep 10
diff --git a/test/debug-external.sh b/test/debug-external.sh
index 301974f456..4a75003dec 100755
--- a/test/debug-external.sh
+++ b/test/debug-external.sh
@@ -5,9 +5,16 @@ rm -rf tmp_log* 2> /dev/null
 rm *.rlp 2> /dev/null
 rm -rf .dht* 2> /dev/null
 scripts/go_executable_build.sh -S || exit 1  # dynamic builds are faster for debug iteration...
+LEGACY_SYNC=${LEGACY_SYNC:-"false"}
+if [ "${LEGACY_SYNC}" == "true" ]; then
+    echo "[WARN] - using legacy sync"
+    SYNC_OPT=(-L "true")
+else
+    SYNC_OPT=(-L "false")
+fi
 if [ "${VERBOSE}" = "true" ]; then
     echo "[WARN] - running with verbose logs"
-    ./test/deploy.sh -v -B -D 600000 ./test/configs/local-resharding-with-external.txt
+    ./test/deploy.sh -v -B -D 600000 "${SYNC_OPT[@]}" ./test/configs/local-resharding-with-external.txt
 else
-    ./test/deploy.sh -B -D 600000 ./test/configs/local-resharding-with-external.txt
+    ./test/deploy.sh -B -D 600000 "${SYNC_OPT[@]}" ./test/configs/local-resharding-with-external.txt
 fi
diff --git a/test/debug.sh b/test/debug.sh
index fd097f6392..61a4bb7001 100755
--- a/test/debug.sh
+++ b/test/debug.sh
@@ -5,9 +5,16 @@ rm -rf tmp_log* 2> /dev/null
 rm *.rlp 2> /dev/null
 rm -rf .dht* 2> /dev/null
 scripts/go_executable_build.sh -S || exit 1  # dynamic builds are faster for debug iteration...
+LEGACY_SYNC=${LEGACY_SYNC:-"false"}
+if [ "${LEGACY_SYNC}" == "true" ]; then
+    echo "[WARN] - using legacy sync"
+    SYNC_OPT=(-L "true")
+else
+    SYNC_OPT=(-L "false")
+fi
 if [ "${VERBOSE}" = "true" ]; then
     echo "[WARN] - running with verbose logs"
-    ./test/deploy.sh -v -B -D 600000 ./test/configs/local-resharding.txt
+    ./test/deploy.sh -v -B -D 600000 "${SYNC_OPT[@]}" ./test/configs/local-resharding.txt
 else
-    ./test/deploy.sh -B -D 600000 ./test/configs/local-resharding.txt
+    ./test/deploy.sh -B -D 600000 "${SYNC_OPT[@]}" ./test/configs/local-resharding.txt
 fi
diff --git a/test/deploy.sh b/test/deploy.sh
index daf48df802..a4428f1c35 100755
--- a/test/deploy.sh
+++ b/test/deploy.sh
@@ -63,8 +63,8 @@ function launch_bootnode() {
 function launch_localnet() {
   launch_bootnode
 
-  unset -v base_args
-  declare -a base_args args
+  unset -v base_args sync_options
+  declare -a base_args args sync_options
 
   if ${VERBOSE}; then
     verbosity=5
@@ -72,14 +72,22 @@ function launch_localnet() {
     verbosity=3
   fi
 
-  base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100")
+  base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" \
+    "--network=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" \
+    "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100")
+  if [ "${LEGACY_SYNC}" == "true" ]; then
+    sync_options=("--dns=true" "--sync=false" "--dns.client=true" "--sync.downloader=false" "--sync.stagedsync=false")
+  else
+    sync_options=("--dns=false" "--sync=true" "--dns.client=false" "--sync.downloader=true" "--sync.stagedsync=true")
+  fi
+
+  base_args+=("${sync_options[@]}")
   sleep 2
 
   # Start nodes
   i=-1
   while IFS='' read -r line || [[ -n "$line" ]]; do
     i=$((i + 1))
-
     # Read config for i-th node form config file
     IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}"
     args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false")
@@ -90,7 +98,6 @@ function launch_localnet() {
     if [[ $EXPOSEAPIS == "true" ]]; then
       args=("${args[@]}" "--http.ip=0.0.0.0" "--ws.ip=0.0.0.0")
     fi
-
     # Setup BLS key for i-th localnet node
     if [[ ! -e "$bls_key" ]]; then
       args=("${args[@]}" --blskey_file "BLSKEY")
@@ -152,6 +159,7 @@ USAGE: $ME [OPTIONS] config_file_name [extra args to node]
    -B             don't build the binary
    -v             verbosity in log (default: $VERBOSE)
    -e             expose WS & HTTP ip (default: $EXPOSEAPIS)
+   -L             start localnet in Legace sync mode(default: $LEGACY_SYNC)
 
 This script will build all the binaries and start harmony and based on the configuration file.
 
@@ -170,8 +178,9 @@ NETWORK=localnet
 VERBOSE=false
 NOBUILD=false
 EXPOSEAPIS=false
+LEGACY_SYNC=false
 
-while getopts "hD:m:s:nBN:ve" option; do
+while getopts "hD:m:s:nBN:veL:" option; do
   case ${option} in
   h) usage ;;
   D) DURATION=$OPTARG ;;
@@ -182,6 +191,7 @@ while getopts "hD:m:s:nBN:ve" option; do
   N) NETWORK=$OPTARG ;;
   v) VERBOSE=true ;;
   e) EXPOSEAPIS=true ;;
+  L) LEGACY_SYNC=$OPTARG ;;
   *) usage ;;
   esac
 done
@@ -198,4 +208,3 @@ setup
 launch_localnet
 sleep "${DURATION}"
 cleanup || true
-

From 1d8dfe66351cae5536bb04d60ab19d64f34ea439 Mon Sep 17 00:00:00 2001
From: Frozen <355847+Frozen@users.noreply.github.com>
Date: Wed, 25 Sep 2024 05:18:43 -0400
Subject: [PATCH 15/24] Broadcast vote power. (#4748)

* Broadcast vote power.
* Moved `GetLastSigningPower` from private api to public.
* Fix for view change sign power.
* Rebased and fixed conflicts.
* Fix for broadcasting consensus vote power.
---
 api/proto/message/harmonymessage.pb.go   | 410 +++++++++++++++--------
 api/proto/message/harmonymessage.proto   |  10 +
 consensus/consensus.go                   |  16 +
 consensus/consensus_service.go           |  50 +++
 consensus/consensus_v2.go                |  11 +
 consensus/fbft_log.go                    |   1 +
 consensus/quorum/one-node-one-vote.go    |   8 +
 consensus/quorum/one-node-staked-vote.go |   8 +
 consensus/quorum/quorum.go               |   1 +
 consensus/quorum/thread_safe_decider.go  |   6 +
 consensus/view_change.go                 |   1 +
 hmy/hmy.go                               |   1 -
 node/harmony/api.go                      |  37 +-
 node/harmony/node.go                     |  24 +-
 rpc/harmony/private_debug.go             |  14 -
 rpc/harmony/public_debug.go              |   7 +
 16 files changed, 408 insertions(+), 197 deletions(-)

diff --git a/api/proto/message/harmonymessage.pb.go b/api/proto/message/harmonymessage.pb.go
index 314403ccbf..f23cec0de2 100644
--- a/api/proto/message/harmonymessage.pb.go
+++ b/api/proto/message/harmonymessage.pb.go
@@ -94,6 +94,7 @@ const (
 	MessageType_DRAND_COMMIT MessageType = 11
 	// Deprecated: Marked as deprecated in harmonymessage.proto.
 	MessageType_LOTTERY_REQUEST MessageType = 12 // it should be either ENTER or GETPLAYERS but it will be removed later.
+	MessageType_LAST_SIGN_POWER MessageType = 13
 )
 
 // Enum value maps for MessageType.
@@ -110,6 +111,7 @@ var (
 		10: "DRAND_INIT",
 		11: "DRAND_COMMIT",
 		12: "LOTTERY_REQUEST",
+		13: "LAST_SIGN_POWER",
 	}
 	MessageType_value = map[string]int32{
 		"NEWNODE_BEACON_STAKING": 0,
@@ -123,6 +125,7 @@ var (
 		"DRAND_INIT":             10,
 		"DRAND_COMMIT":           11,
 		"LOTTERY_REQUEST":        12,
+		"LAST_SIGN_POWER":        13,
 	}
 )
 
@@ -225,6 +228,7 @@ type Message struct {
 	//	*Message_Drand
 	//	*Message_Viewchange
 	//	*Message_LotteryRequest
+	//	*Message_LastSignPower
 	Request isMessage_Request `protobuf_oneof:"request"`
 }
 
@@ -326,6 +330,13 @@ func (x *Message) GetLotteryRequest() *LotteryRequest {
 	return nil
 }
 
+func (x *Message) GetLastSignPower() *LastSignPowerBroadcast {
+	if x, ok := x.GetRequest().(*Message_LastSignPower); ok {
+		return x.LastSignPower
+	}
+	return nil
+}
+
 type isMessage_Request interface {
 	isMessage_Request()
 }
@@ -355,6 +366,10 @@ type Message_LotteryRequest struct {
 	LotteryRequest *LotteryRequest `protobuf:"bytes,8,opt,name=lottery_request,json=lotteryRequest,proto3,oneof"`
 }
 
+type Message_LastSignPower struct {
+	LastSignPower *LastSignPowerBroadcast `protobuf:"bytes,9,opt,name=last_sign_power,json=lastSignPower,proto3,oneof"`
+}
+
 func (*Message_Staking) isMessage_Request() {}
 
 func (*Message_Consensus) isMessage_Request() {}
@@ -365,6 +380,8 @@ func (*Message_Viewchange) isMessage_Request() {}
 
 func (*Message_LotteryRequest) isMessage_Request() {}
 
+func (*Message_LastSignPower) isMessage_Request() {}
+
 type Response struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -965,12 +982,91 @@ func (x *ViewChangeRequest) GetPreparedBlock() []byte {
 	return nil
 }
 
+type LastSignPowerBroadcast struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Prepare      int64  `protobuf:"varint,1,opt,name=prepare,proto3" json:"prepare,omitempty"`
+	Commit       int64  `protobuf:"varint,2,opt,name=commit,proto3" json:"commit,omitempty"`
+	Change       int64  `protobuf:"varint,3,opt,name=change,proto3" json:"change,omitempty"`
+	SenderPubkey []byte `protobuf:"bytes,4,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"`
+	ShardId      uint32 `protobuf:"varint,5,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"`
+}
+
+func (x *LastSignPowerBroadcast) Reset() {
+	*x = LastSignPowerBroadcast{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_harmonymessage_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *LastSignPowerBroadcast) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LastSignPowerBroadcast) ProtoMessage() {}
+
+func (x *LastSignPowerBroadcast) ProtoReflect() protoreflect.Message {
+	mi := &file_harmonymessage_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LastSignPowerBroadcast.ProtoReflect.Descriptor instead.
+func (*LastSignPowerBroadcast) Descriptor() ([]byte, []int) {
+	return file_harmonymessage_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *LastSignPowerBroadcast) GetPrepare() int64 {
+	if x != nil {
+		return x.Prepare
+	}
+	return 0
+}
+
+func (x *LastSignPowerBroadcast) GetCommit() int64 {
+	if x != nil {
+		return x.Commit
+	}
+	return 0
+}
+
+func (x *LastSignPowerBroadcast) GetChange() int64 {
+	if x != nil {
+		return x.Change
+	}
+	return 0
+}
+
+func (x *LastSignPowerBroadcast) GetSenderPubkey() []byte {
+	if x != nil {
+		return x.SenderPubkey
+	}
+	return nil
+}
+
+func (x *LastSignPowerBroadcast) GetShardId() uint32 {
+	if x != nil {
+		return x.ShardId
+	}
+	return 0
+}
+
 var File_harmonymessage_proto protoreflect.FileDescriptor
 
 var file_harmonymessage_proto_rawDesc = []byte{
 	0x0a, 0x14, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
 	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d,
-	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xf3, 0x03, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xc5, 0x04, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61,
 	0x67, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x79,
 	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f,
 	0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
@@ -1001,123 +1097,140 @@ var file_harmonymessage_proto_rawDesc = []byte{
 	0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4c, 0x6f, 0x74,
 	0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, 0x02, 0x18, 0x01, 0x48,
 	0x00, 0x52, 0x0e, 0x6c, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xd9, 0x01, 0x0a,
-	0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x73, 0x65, 0x72,
-	0x76, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32,
-	0x1b, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
-	0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x73, 0x65,
-	0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x74, 0x79, 0x70,
-	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e,
-	0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
-	0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x50, 0x0a, 0x10, 0x6c, 0x6f,
-	0x74, 0x74, 0x65, 0x72, 0x79, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4c, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x02, 0x18, 0x01, 0x48, 0x00, 0x52, 0x0f, 0x6c, 0x6f, 0x74,
-	0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x0a, 0x0a, 0x08,
-	0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x4f, 0x0a, 0x0f, 0x4c, 0x6f, 0x74, 0x74,
-	0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x07, 0x70,
-	0x6c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01,
-	0x52, 0x07, 0x70, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x12, 0x1e, 0x0a, 0x08, 0x62, 0x61, 0x6c,
-	0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52,
-	0x08, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x22, 0xca, 0x01, 0x0a, 0x0e, 0x4c, 0x6f,
-	0x74, 0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x04,
-	0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x68, 0x61, 0x72,
-	0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4c, 0x6f, 0x74, 0x74,
-	0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x42,
-	0x02, 0x18, 0x01, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0b, 0x70, 0x72, 0x69,
-	0x76, 0x61, 0x74, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02,
-	0x18, 0x01, 0x52, 0x0a, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1a,
-	0x0a, 0x06, 0x61, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x42, 0x02,
-	0x18, 0x01, 0x52, 0x06, 0x61, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x3a, 0x0a, 0x04, 0x54, 0x79,
-	0x70, 0x65, 0x12, 0x0d, 0x0a, 0x05, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x00, 0x1a, 0x02, 0x08,
-	0x01, 0x12, 0x0e, 0x0a, 0x06, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x10, 0x01, 0x1a, 0x02, 0x08,
-	0x01, 0x12, 0x13, 0x0a, 0x0b, 0x50, 0x49, 0x43, 0x4b, 0x5f, 0x57, 0x49, 0x4e, 0x4e, 0x45, 0x52,
-	0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x22, 0x53, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x6b, 0x69, 0x6e,
-	0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0b, 0x74, 0x72, 0x61, 0x6e,
-	0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18,
-	0x01, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b,
-	0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42,
-	0x02, 0x18, 0x01, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x22, 0x89, 0x02, 0x0a, 0x10,
-	0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x12, 0x17, 0x0a, 0x07, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x04, 0x52, 0x06, 0x76, 0x69, 0x65, 0x77, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6c, 0x6f,
-	0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x62, 0x6c,
-	0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f,
-	0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49,
-	0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18,
-	0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68,
-	0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52,
-	0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72,
-	0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73,
-	0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70,
-	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61,
-	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x5f,
-	0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x18, 0x08, 0x20,
-	0x01, 0x28, 0x0c, 0x52, 0x12, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65,
-	0x79, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x22, 0x97, 0x01, 0x0a, 0x0c, 0x44, 0x72, 0x61, 0x6e,
-	0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72,
-	0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x42, 0x02, 0x18, 0x01, 0x52, 0x07,
-	0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0d, 0x73, 0x65, 0x6e, 0x64, 0x65,
-	0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02,
-	0x18, 0x01, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79,
-	0x12, 0x21, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48,
-	0x61, 0x73, 0x68, 0x12, 0x1c, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x04,
-	0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
-	0x64, 0x22, 0xad, 0x03, 0x0a, 0x11, 0x56, 0x69, 0x65, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x76, 0x69, 0x65, 0x77, 0x5f,
-	0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x76, 0x69, 0x65, 0x77, 0x49, 0x64,
-	0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x04, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, 0x12, 0x19, 0x0a,
-	0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52,
-	0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x6e, 0x64,
+	0x74, 0x12, 0x50, 0x0a, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x5f, 0x70,
+	0x6f, 0x77, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x68, 0x61, 0x72,
+	0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4c, 0x61, 0x73, 0x74,
+	0x53, 0x69, 0x67, 0x6e, 0x50, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x72, 0x6f, 0x61, 0x64, 0x63, 0x61,
+	0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x50, 0x6f,
+	0x77, 0x65, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xd9,
+	0x01, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x73,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0e, 0x32, 0x1b, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61,
+	0x67, 0x65, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b,
+	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x74,
+	0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x68, 0x61, 0x72, 0x6d,
+	0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61,
+	0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x50, 0x0a, 0x10,
+	0x6c, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x79, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79,
+	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4c, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x79, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x02, 0x18, 0x01, 0x48, 0x00, 0x52, 0x0f, 0x6c,
+	0x6f, 0x74, 0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x0a,
+	0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x4f, 0x0a, 0x0f, 0x4c, 0x6f,
+	0x74, 0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a,
+	0x07, 0x70, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x42, 0x02,
+	0x18, 0x01, 0x52, 0x07, 0x70, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x12, 0x1e, 0x0a, 0x08, 0x62,
+	0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x42, 0x02, 0x18,
+	0x01, 0x52, 0x08, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x22, 0xca, 0x01, 0x0a, 0x0e,
+	0x4c, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b,
+	0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x68,
+	0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4c, 0x6f,
+	0x74, 0x74, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, 0x79, 0x70,
+	0x65, 0x42, 0x02, 0x18, 0x01, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0b, 0x70,
+	0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x42, 0x02, 0x18, 0x01, 0x52, 0x0a, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79,
+	0x12, 0x1a, 0x0a, 0x06, 0x61, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03,
+	0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x61, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x3a, 0x0a, 0x04,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x05, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x00, 0x1a,
+	0x02, 0x08, 0x01, 0x12, 0x0e, 0x0a, 0x06, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x10, 0x01, 0x1a,
+	0x02, 0x08, 0x01, 0x12, 0x13, 0x0a, 0x0b, 0x50, 0x49, 0x43, 0x4b, 0x5f, 0x57, 0x49, 0x4e, 0x4e,
+	0x45, 0x52, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x22, 0x53, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x6b,
+	0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0b, 0x74, 0x72,
+	0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x42,
+	0x02, 0x18, 0x01, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x12, 0x1b, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x22, 0x89, 0x02,
+	0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x04, 0x52, 0x06, 0x76, 0x69, 0x65, 0x77, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62,
+	0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08,
+	0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72,
+	0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72,
+	0x64, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x61, 0x73,
+	0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61,
+	0x73, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28,
+	0x0c, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x6e, 0x64,
+	0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52,
+	0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a,
+	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07,
+	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x65, 0x6e, 0x64, 0x65,
+	0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x18,
+	0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62,
+	0x6b, 0x65, 0x79, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x22, 0x97, 0x01, 0x0a, 0x0c, 0x44, 0x72,
+	0x61, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x08, 0x73, 0x68,
+	0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x42, 0x02, 0x18, 0x01,
+	0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0d, 0x73, 0x65, 0x6e,
+	0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c,
+	0x42, 0x02, 0x18, 0x01, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b,
+	0x65, 0x79, 0x12, 0x21, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63,
+	0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x1c, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x22, 0xad, 0x03, 0x0a, 0x11, 0x56, 0x69, 0x65, 0x77, 0x43, 0x68, 0x61, 0x6e,
+	0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x76, 0x69, 0x65,
+	0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x76, 0x69, 0x65, 0x77,
+	0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, 0x12,
+	0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65,
+	0x6e, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x0c, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12,
+	0x23, 0x0a, 0x0d, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79,
+	0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x75,
+	0x62, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
+	0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x25,
+	0x0a, 0x0e, 0x76, 0x69, 0x65, 0x77, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x67,
+	0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x76, 0x69, 0x65, 0x77, 0x63, 0x68, 0x61, 0x6e,
+	0x67, 0x65, 0x53, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x69, 0x65, 0x77, 0x69, 0x64, 0x5f,
+	0x73, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x76, 0x69, 0x65, 0x77, 0x69,
+	0x64, 0x53, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x32, 0x5f, 0x61, 0x67, 0x67, 0x73, 0x69,
+	0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x32, 0x41, 0x67, 0x67, 0x73,
+	0x69, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x32, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70,
+	0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x32, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70,
+	0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x33, 0x5f, 0x61, 0x67, 0x67, 0x73, 0x69, 0x67, 0x73, 0x18, 0x0b,
+	0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x33, 0x41, 0x67, 0x67, 0x73, 0x69, 0x67, 0x73, 0x12,
+	0x1b, 0x0a, 0x09, 0x6d, 0x33, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x18, 0x0c, 0x20, 0x01,
+	0x28, 0x0c, 0x52, 0x08, 0x6d, 0x33, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x12, 0x25, 0x0a, 0x0e,
+	0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x0d,
+	0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x42, 0x6c,
+	0x6f, 0x63, 0x6b, 0x22, 0xa2, 0x01, 0x0a, 0x16, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x69, 0x67, 0x6e,
+	0x50, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x72, 0x6f, 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x12, 0x18,
+	0x0a, 0x07, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x07, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d,
+	0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
+	0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03,
+	0x52, 0x06, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x6e, 0x64,
 	0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52,
-	0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a,
-	0x0d, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x05,
-	0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b,
-	0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x06, 0x20,
-	0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x25, 0x0a, 0x0e,
-	0x76, 0x69, 0x65, 0x77, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x67, 0x18, 0x07,
-	0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x76, 0x69, 0x65, 0x77, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65,
-	0x53, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x69, 0x65, 0x77, 0x69, 0x64, 0x5f, 0x73, 0x69,
-	0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x76, 0x69, 0x65, 0x77, 0x69, 0x64, 0x53,
-	0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x32, 0x5f, 0x61, 0x67, 0x67, 0x73, 0x69, 0x67, 0x73,
-	0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x32, 0x41, 0x67, 0x67, 0x73, 0x69, 0x67,
-	0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x32, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x18, 0x0a,
-	0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x32, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x12, 0x1d,
-	0x0a, 0x0a, 0x6d, 0x33, 0x5f, 0x61, 0x67, 0x67, 0x73, 0x69, 0x67, 0x73, 0x18, 0x0b, 0x20, 0x01,
-	0x28, 0x0c, 0x52, 0x09, 0x6d, 0x33, 0x41, 0x67, 0x67, 0x73, 0x69, 0x67, 0x73, 0x12, 0x1b, 0x0a,
-	0x09, 0x6d, 0x33, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0c,
-	0x52, 0x08, 0x6d, 0x33, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72,
-	0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x0d, 0x20, 0x01,
-	0x28, 0x0c, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x42, 0x6c, 0x6f, 0x63,
-	0x6b, 0x2a, 0x50, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65,
-	0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x53, 0x45, 0x4e, 0x53, 0x55, 0x53, 0x10, 0x00, 0x12,
-	0x0f, 0x0a, 0x07, 0x53, 0x54, 0x41, 0x4b, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x1a, 0x02, 0x08, 0x01,
-	0x12, 0x0d, 0x0a, 0x05, 0x44, 0x52, 0x41, 0x4e, 0x44, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12,
-	0x12, 0x0a, 0x0e, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52,
-	0x54, 0x10, 0x03, 0x2a, 0xd1, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54,
-	0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x16, 0x4e, 0x45, 0x57, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x42,
-	0x45, 0x41, 0x43, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x4b, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x1a,
-	0x02, 0x08, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x4e, 0x4e, 0x4f, 0x55, 0x4e, 0x43, 0x45, 0x10,
-	0x01, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x10, 0x02, 0x12, 0x0c,
-	0x0a, 0x08, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06,
-	0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x4d,
-	0x49, 0x54, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0e, 0x0a, 0x0a, 0x56, 0x49, 0x45, 0x57, 0x43,
-	0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x45, 0x57, 0x56, 0x49,
-	0x45, 0x57, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0a, 0x44, 0x52, 0x41, 0x4e, 0x44, 0x5f, 0x49, 0x4e,
-	0x49, 0x54, 0x10, 0x0a, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x14, 0x0a, 0x0c, 0x44, 0x52, 0x41, 0x4e,
-	0x44, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x0b, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x17,
-	0x0a, 0x0f, 0x4c, 0x4f, 0x54, 0x54, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53,
-	0x54, 0x10, 0x0c, 0x1a, 0x02, 0x08, 0x01, 0x32, 0x4f, 0x0a, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e,
-	0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63,
-	0x65, 0x73, 0x73, 0x12, 0x17, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73,
-	0x73, 0x61, 0x67, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x18, 0x2e, 0x68,
-	0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x13, 0x5a, 0x11, 0x2e, 0x2f, 0x3b, 0x68,
-	0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x19, 0x0a,
+	0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x2a, 0x50, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x53, 0x45,
+	0x4e, 0x53, 0x55, 0x53, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x07, 0x53, 0x54, 0x41, 0x4b, 0x49, 0x4e,
+	0x47, 0x10, 0x01, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x0d, 0x0a, 0x05, 0x44, 0x52, 0x41, 0x4e, 0x44,
+	0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54,
+	0x5f, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x10, 0x03, 0x2a, 0xe6, 0x01, 0x0a, 0x0b, 0x4d,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x16, 0x4e, 0x45,
+	0x57, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x42, 0x45, 0x41, 0x43, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41,
+	0x4b, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x4e,
+	0x4e, 0x4f, 0x55, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x50,
+	0x41, 0x52, 0x45, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45,
+	0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x04, 0x12,
+	0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0e,
+	0x0a, 0x0a, 0x56, 0x49, 0x45, 0x57, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x06, 0x12, 0x0b,
+	0x0a, 0x07, 0x4e, 0x45, 0x57, 0x56, 0x49, 0x45, 0x57, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0a, 0x44,
+	0x52, 0x41, 0x4e, 0x44, 0x5f, 0x49, 0x4e, 0x49, 0x54, 0x10, 0x0a, 0x1a, 0x02, 0x08, 0x01, 0x12,
+	0x14, 0x0a, 0x0c, 0x44, 0x52, 0x41, 0x4e, 0x44, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10,
+	0x0b, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x17, 0x0a, 0x0f, 0x4c, 0x4f, 0x54, 0x54, 0x45, 0x52, 0x59,
+	0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x0c, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x13,
+	0x0a, 0x0f, 0x4c, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x49, 0x47, 0x4e, 0x5f, 0x50, 0x4f, 0x57, 0x45,
+	0x52, 0x10, 0x0d, 0x32, 0x4f, 0x0a, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12,
+	0x17, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f, 0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x18, 0x2e, 0x68, 0x61, 0x72, 0x6d, 0x6f,
+	0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x42, 0x13, 0x5a, 0x11, 0x2e, 0x2f, 0x3b, 0x68, 0x61, 0x72, 0x6d, 0x6f,
+	0x6e, 0x79, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
 }
 
 var (
@@ -1133,19 +1246,20 @@ func file_harmonymessage_proto_rawDescGZIP() []byte {
 }
 
 var file_harmonymessage_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
-var file_harmonymessage_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
+var file_harmonymessage_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
 var file_harmonymessage_proto_goTypes = []interface{}{
-	(ServiceType)(0),          // 0: harmonymessage.ServiceType
-	(MessageType)(0),          // 1: harmonymessage.MessageType
-	(LotteryRequest_Type)(0),  // 2: harmonymessage.LotteryRequest.Type
-	(*Message)(nil),           // 3: harmonymessage.Message
-	(*Response)(nil),          // 4: harmonymessage.Response
-	(*LotteryResponse)(nil),   // 5: harmonymessage.LotteryResponse
-	(*LotteryRequest)(nil),    // 6: harmonymessage.LotteryRequest
-	(*StakingRequest)(nil),    // 7: harmonymessage.StakingRequest
-	(*ConsensusRequest)(nil),  // 8: harmonymessage.ConsensusRequest
-	(*DrandRequest)(nil),      // 9: harmonymessage.DrandRequest
-	(*ViewChangeRequest)(nil), // 10: harmonymessage.ViewChangeRequest
+	(ServiceType)(0),               // 0: harmonymessage.ServiceType
+	(MessageType)(0),               // 1: harmonymessage.MessageType
+	(LotteryRequest_Type)(0),       // 2: harmonymessage.LotteryRequest.Type
+	(*Message)(nil),                // 3: harmonymessage.Message
+	(*Response)(nil),               // 4: harmonymessage.Response
+	(*LotteryResponse)(nil),        // 5: harmonymessage.LotteryResponse
+	(*LotteryRequest)(nil),         // 6: harmonymessage.LotteryRequest
+	(*StakingRequest)(nil),         // 7: harmonymessage.StakingRequest
+	(*ConsensusRequest)(nil),       // 8: harmonymessage.ConsensusRequest
+	(*DrandRequest)(nil),           // 9: harmonymessage.DrandRequest
+	(*ViewChangeRequest)(nil),      // 10: harmonymessage.ViewChangeRequest
+	(*LastSignPowerBroadcast)(nil), // 11: harmonymessage.LastSignPowerBroadcast
 }
 var file_harmonymessage_proto_depIdxs = []int32{
 	0,  // 0: harmonymessage.Message.service_type:type_name -> harmonymessage.ServiceType
@@ -1155,17 +1269,18 @@ var file_harmonymessage_proto_depIdxs = []int32{
 	9,  // 4: harmonymessage.Message.drand:type_name -> harmonymessage.DrandRequest
 	10, // 5: harmonymessage.Message.viewchange:type_name -> harmonymessage.ViewChangeRequest
 	6,  // 6: harmonymessage.Message.lottery_request:type_name -> harmonymessage.LotteryRequest
-	0,  // 7: harmonymessage.Response.service_type:type_name -> harmonymessage.ServiceType
-	1,  // 8: harmonymessage.Response.type:type_name -> harmonymessage.MessageType
-	5,  // 9: harmonymessage.Response.lottery_response:type_name -> harmonymessage.LotteryResponse
-	2,  // 10: harmonymessage.LotteryRequest.type:type_name -> harmonymessage.LotteryRequest.Type
-	3,  // 11: harmonymessage.ClientService.Process:input_type -> harmonymessage.Message
-	4,  // 12: harmonymessage.ClientService.Process:output_type -> harmonymessage.Response
-	12, // [12:13] is the sub-list for method output_type
-	11, // [11:12] is the sub-list for method input_type
-	11, // [11:11] is the sub-list for extension type_name
-	11, // [11:11] is the sub-list for extension extendee
-	0,  // [0:11] is the sub-list for field type_name
+	11, // 7: harmonymessage.Message.last_sign_power:type_name -> harmonymessage.LastSignPowerBroadcast
+	0,  // 8: harmonymessage.Response.service_type:type_name -> harmonymessage.ServiceType
+	1,  // 9: harmonymessage.Response.type:type_name -> harmonymessage.MessageType
+	5,  // 10: harmonymessage.Response.lottery_response:type_name -> harmonymessage.LotteryResponse
+	2,  // 11: harmonymessage.LotteryRequest.type:type_name -> harmonymessage.LotteryRequest.Type
+	3,  // 12: harmonymessage.ClientService.Process:input_type -> harmonymessage.Message
+	4,  // 13: harmonymessage.ClientService.Process:output_type -> harmonymessage.Response
+	13, // [13:14] is the sub-list for method output_type
+	12, // [12:13] is the sub-list for method input_type
+	12, // [12:12] is the sub-list for extension type_name
+	12, // [12:12] is the sub-list for extension extendee
+	0,  // [0:12] is the sub-list for field type_name
 }
 
 func init() { file_harmonymessage_proto_init() }
@@ -1270,6 +1385,18 @@ func file_harmonymessage_proto_init() {
 				return nil
 			}
 		}
+		file_harmonymessage_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LastSignPowerBroadcast); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	file_harmonymessage_proto_msgTypes[0].OneofWrappers = []interface{}{
 		(*Message_Staking)(nil),
@@ -1277,6 +1404,7 @@ func file_harmonymessage_proto_init() {
 		(*Message_Drand)(nil),
 		(*Message_Viewchange)(nil),
 		(*Message_LotteryRequest)(nil),
+		(*Message_LastSignPower)(nil),
 	}
 	file_harmonymessage_proto_msgTypes[1].OneofWrappers = []interface{}{
 		(*Response_LotteryResponse)(nil),
@@ -1287,7 +1415,7 @@ func file_harmonymessage_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_harmonymessage_proto_rawDesc,
 			NumEnums:      3,
-			NumMessages:   8,
+			NumMessages:   9,
 			NumExtensions: 0,
 			NumServices:   1,
 		},
diff --git a/api/proto/message/harmonymessage.proto b/api/proto/message/harmonymessage.proto
index f26cbf6085..8491300256 100644
--- a/api/proto/message/harmonymessage.proto
+++ b/api/proto/message/harmonymessage.proto
@@ -29,6 +29,7 @@ enum MessageType {
   DRAND_INIT = 10 [deprecated=true];
   DRAND_COMMIT = 11 [deprecated=true];
   LOTTERY_REQUEST = 12 [deprecated=true]; // it should be either ENTER or GETPLAYERS but it will be removed later.
+  LAST_SIGN_POWER = 13;
 }
 
 // This is universal message for all communication protocols.
@@ -47,6 +48,7 @@ message Message {
     ViewChangeRequest viewchange = 7;
     // Refactor this later after demo.
     LotteryRequest lottery_request = 8 [deprecated=true];
+    LastSignPowerBroadcast last_sign_power = 9;
   }
 }
 
@@ -116,3 +118,11 @@ message ViewChangeRequest {
   bytes m3_bitmap= 12;
   bytes prepared_block = 13;
 }
+
+message LastSignPowerBroadcast {
+  int64 prepare = 1;
+  int64 commit = 2;
+  int64 change = 3;
+  bytes sender_pubkey = 4;
+  uint32 shard_id = 5;
+}
diff --git a/consensus/consensus.go b/consensus/consensus.go
index 1a649e5ee9..e88cdff980 100644
--- a/consensus/consensus.go
+++ b/consensus/consensus.go
@@ -138,6 +138,10 @@ type Consensus struct {
 	// Both flags only for initialization state.
 	start           bool
 	isInitialLeader bool
+
+	// value receives from
+	lastKnownSignPower int64
+	lastKnowViewChange int64
 }
 
 // Blockchain returns the blockchain.
@@ -395,6 +399,18 @@ func (consensus *Consensus) InitConsensusWithValidators() (err error) {
 	return nil
 }
 
+func (consensus *Consensus) SetLastKnownSignPower(signPower, viewChange int64) {
+	atomic.StoreInt64(&consensus.lastKnownSignPower, signPower)
+	atomic.StoreInt64(&consensus.lastKnowViewChange, viewChange)
+}
+
+func (consensus *Consensus) GetLastKnownSignPower() int64 {
+	if consensus.IsViewChangingMode() {
+		return atomic.LoadInt64(&consensus.lastKnowViewChange)
+	}
+	return atomic.LoadInt64(&consensus.lastKnownSignPower)
+}
+
 type downloadAsync struct {
 }
 
diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go
index e64f3007ab..83dd839698 100644
--- a/consensus/consensus_service.go
+++ b/consensus/consensus_service.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	bls_core "github.com/harmony-one/bls/ffi/go/bls"
+	"github.com/harmony-one/harmony/api/proto"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	consensus_engine "github.com/harmony-one/harmony/consensus/engine"
 	"github.com/harmony-one/harmony/consensus/quorum"
@@ -17,8 +18,10 @@ import (
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/crypto/hash"
 	"github.com/harmony-one/harmony/internal/chain"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/multibls"
+	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
 	"github.com/harmony-one/harmony/webhooks"
@@ -153,6 +156,53 @@ func (consensus *Consensus) updateBitmaps() {
 
 }
 
+func (consensus *Consensus) sendLastSignPower() {
+	if consensus.isLeader() {
+		k, err := consensus.getLeaderPrivateKey(consensus.getLeaderPubKey().Object)
+		if err != nil {
+			consensus.getLogger().Err(err).Msg("Leader not found in the committee")
+			return
+		}
+		comm := getOrZero(consensus.decider.CurrentTotalPower(quorum.Commit))
+		prep := getOrZero(consensus.decider.CurrentTotalPower(quorum.Prepare))
+		view := consensus.decider.ComputeTotalPowerByMask(consensus.vc.GetViewIDBitmap(consensus.current.viewChangingID)).Int64()
+		for view > 100 {
+			view /= 10
+		}
+		msg := &msg_pb.Message{
+			ServiceType: msg_pb.ServiceType_CONSENSUS,
+			Type:        msg_pb.MessageType_LAST_SIGN_POWER,
+			Request: &msg_pb.Message_LastSignPower{
+				LastSignPower: &msg_pb.LastSignPowerBroadcast{
+					Prepare:      prep,
+					Commit:       comm,
+					Change:       view,
+					SenderPubkey: k.Pub.Bytes[:],
+					ShardId:      consensus.ShardID,
+				},
+			},
+		}
+		marshaledMessage, err := consensus.signAndMarshalConsensusMessage(msg, k.Pri)
+		if err != nil {
+			consensus.getLogger().Err(err).
+				Msg("[constructNewViewMessage] failed to sign and marshal the new view message")
+			return
+		}
+		if comm == 0 && prep == 0 && view == 0 {
+			return
+		}
+		msgToSend := proto.ConstructConsensusMessage(marshaledMessage)
+		if err := consensus.msgSender.SendWithoutRetry(
+			[]nodeconfig.GroupID{
+				nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
+			p2p.ConstructMessage(msgToSend),
+		); err != nil {
+			consensus.getLogger().Err(err).
+				Msg("[LastSignPower] could not send out the ViewChange message")
+		}
+	}
+}
+
 // ResetState resets the state of the consensus
 func (consensus *Consensus) resetState() {
 	consensus.switchPhase("ResetState", FBFTAnnounce)
diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go
index dbc8e8bf3d..80f6d110ae 100644
--- a/consensus/consensus_v2.go
+++ b/consensus/consensus_v2.go
@@ -20,6 +20,7 @@ import (
 	vrf_bls "github.com/harmony-one/harmony/crypto/vrf/bls"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/vdf/src/vdf_go"
@@ -89,6 +90,8 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, peer libp2p
 	case t == msg_pb.MessageType_NEWVIEW:
 		members := consensus.decider.Participants()
 		fbftMsg, err = ParseNewViewMessage(msg, members)
+	case t == msg_pb.MessageType_LAST_SIGN_POWER:
+		return nil
 	default:
 		fbftMsg, err = consensus.parseFBFTMessage(msg)
 	}
@@ -820,6 +823,7 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg
 	}
 	consensus.fBFTLog.PruneCacheBeforeBlock(blk.NumberU64())
 	consensus.resetState()
+	consensus.sendLastSignPower()
 }
 
 func (consensus *Consensus) postCatchup(initBN uint64) {
@@ -968,3 +972,10 @@ func (consensus *Consensus) DeleteMessagesLessThan(number uint64) {
 	defer consensus.mutex.Unlock()
 	consensus.fBFTLog.deleteMessagesLessThan(number)
 }
+
+func getOrZero(n *numeric.Dec, err error) int64 {
+	if n == nil {
+		return 0
+	}
+	return (*n).Mul(numeric.NewDec(100)).TruncateInt64()
+}
diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go
index cec74e314b..60c4d248e1 100644
--- a/consensus/fbft_log.go
+++ b/consensus/fbft_log.go
@@ -36,6 +36,7 @@ type FBFTMessage struct {
 	M3AggSig           *bls_core.Sign
 	M3Bitmap           *bls_cosi.Mask
 	Verified           bool
+	LastVotePower      int64
 }
 
 func (m *FBFTMessage) Hash() []byte {
diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go
index e9e229fdf3..4962dfc86e 100644
--- a/consensus/quorum/one-node-one-vote.go
+++ b/consensus/quorum/one-node-one-vote.go
@@ -71,6 +71,14 @@ func (v *uniformVoteWeight) IsQuorumAchievedByMask(mask *bls_cosi.Mask) bool {
 	return true
 }
 
+func (v *uniformVoteWeight) ComputeTotalPowerByMask(mask *bls_cosi.Mask) numeric.Dec {
+	if mask == nil {
+		return numeric.ZeroDec()
+	}
+	currentTotalPower := utils.CountOneBits(mask.Bitmap)
+	return numeric.NewDec(currentTotalPower)
+}
+
 // QuorumThreshold ..
 func (v *uniformVoteWeight) QuorumThreshold() numeric.Dec {
 	return numeric.NewDec(v.TwoThirdsSignersCount())
diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go
index 2532f9691e..45f4289eab 100644
--- a/consensus/quorum/one-node-staked-vote.go
+++ b/consensus/quorum/one-node-staked-vote.go
@@ -163,6 +163,14 @@ func (v *stakedVoteWeight) IsQuorumAchievedByMask(mask *bls_cosi.Mask) bool {
 	return (*currentTotalPower).GT(threshold)
 }
 
+func (v *stakedVoteWeight) ComputeTotalPowerByMask(mask *bls_cosi.Mask) numeric.Dec {
+	if mask == nil {
+		return numeric.ZeroDec()
+	}
+	currentTotalPower := v.computeTotalPowerByMask(mask)
+	return *currentTotalPower
+}
+
 // ComputeTotalPowerByMask computes the total power indicated by bitmap mask
 func (v *stakedVoteWeight) computeTotalPowerByMask(mask *bls_cosi.Mask) *numeric.Dec {
 	currentTotal := numeric.ZeroDec()
diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go
index 070a85e029..f2cfe82bee 100644
--- a/consensus/quorum/quorum.go
+++ b/consensus/quorum/quorum.go
@@ -118,6 +118,7 @@ type Decider interface {
 	) (*votepower.Ballot, error)
 	IsQuorumAchieved(Phase) bool
 	IsQuorumAchievedByMask(mask *bls_cosi.Mask) bool
+	ComputeTotalPowerByMask(mask *bls_cosi.Mask) numeric.Dec
 	QuorumThreshold() numeric.Dec
 	IsAllSigsCollected() bool
 	ResetPrepareAndCommitVotes()
diff --git a/consensus/quorum/thread_safe_decider.go b/consensus/quorum/thread_safe_decider.go
index 6b6279e53e..ac5ae75dec 100644
--- a/consensus/quorum/thread_safe_decider.go
+++ b/consensus/quorum/thread_safe_decider.go
@@ -169,3 +169,9 @@ func (a threadSafeDeciderImpl) IsQuorumAchieved(p Phase) bool {
 	defer a.mu.Unlock()
 	return a.decider.IsQuorumAchieved(p)
 }
+
+func (a threadSafeDeciderImpl) ComputeTotalPowerByMask(mask *bls.Mask) numeric.Dec {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.decider.ComputeTotalPowerByMask(mask)
+}
diff --git a/consensus/view_change.go b/consensus/view_change.go
index 2752270cba..43e9921950 100644
--- a/consensus/view_change.go
+++ b/consensus/view_change.go
@@ -424,6 +424,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) {
 			Msg("[onViewChange] process View Change message error")
 		return
 	}
+	consensus.sendLastSignPower()
 
 	// received enough view change messages, change state to normal consensus
 	if consensus.decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.isViewChangingMode() {
diff --git a/hmy/hmy.go b/hmy/hmy.go
index 12c3378844..c82ea7e78e 100644
--- a/hmy/hmy.go
+++ b/hmy/hmy.go
@@ -120,7 +120,6 @@ type NodeAPI interface {
 	GetConfig() commonRPC.Config
 	ShutDown()
 	GetLastSigningPower() (float64, error)
-	GetLastSigningPower2() (float64, error)
 }
 
 // New creates a new Harmony object (including the
diff --git a/node/harmony/api.go b/node/harmony/api.go
index 56c4e0f4d2..7bb8138428 100644
--- a/node/harmony/api.go
+++ b/node/harmony/api.go
@@ -1,10 +1,7 @@
 package node
 
 import (
-	"github.com/harmony-one/harmony/consensus/quorum"
-	"github.com/harmony-one/harmony/consensus/votepower"
 	"github.com/harmony-one/harmony/core/types"
-	"github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/hmy"
 	"github.com/harmony-one/harmony/internal/tikv"
@@ -171,37 +168,7 @@ func (node *Node) GetConfig() rpc_common.Config {
 
 // GetLastSigningPower get last signed power
 func (node *Node) GetLastSigningPower() (float64, error) {
-	power, err := node.Consensus.Decider().CurrentTotalPower(quorum.Commit)
-	if err != nil {
-		return 0, err
-	}
-
-	round := float64(power.MulInt64(10000).RoundInt64()) / 10000
-	return round, nil
-}
-
-func (node *Node) GetLastSigningPower2() (float64, error) {
-	bc := node.Consensus.Blockchain()
-	cur := bc.CurrentBlock()
-	ss, err := bc.ReadShardState(cur.Epoch())
-	if err != nil {
-		return 0, err
-	}
-	roster, err := votepower.Compute(&ss.Shards[bc.ShardID()], cur.Epoch())
-	if err != nil {
-		return 0, err
-	}
-	blsPubKeys, err := ss.Shards[bc.ShardID()].BLSPublicKeys()
-	if err != nil {
-		return 0, err
-	}
-
-	mask := bls.NewMask(blsPubKeys)
-	err = mask.SetMask(cur.Header().LastCommitBitmap())
-	if err != nil {
-		return 0, err
-	}
-	power := roster.VotePowerByMask(mask)
-	round := float64(power.MulInt64(10000).RoundInt64()) / 10000
+	p := node.Consensus.GetLastKnownSignPower()
+	round := float64(p) / 100
 	return round, nil
 }
diff --git a/node/harmony/node.go b/node/harmony/node.go
index 19778506a6..4cb34eccec 100644
--- a/node/harmony/node.go
+++ b/node/harmony/node.go
@@ -528,11 +528,16 @@ func validateShardBoundMessage(consensus *consensus.Consensus, peer libp2p_peer.
 		}
 	}
 
-	maybeCon, maybeVC := m.GetConsensus(), m.GetViewchange()
-	senderKey := []byte{}
-	senderBitmap := []byte{}
+	var (
+		maybeCon     = m.GetConsensus()
+		maybeVC      = m.GetViewchange()
+		maybeSP      = m.GetLastSignPower()
+		senderKey    []byte
+		senderBitmap []byte
+	)
 
-	if maybeCon != nil {
+	switch {
+	case maybeCon != nil:
 		if maybeCon.ShardId != consensus.ShardID {
 			nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_shard"}).Inc()
 			return nil, nil, true, errors.WithStack(errWrongShardID)
@@ -546,7 +551,7 @@ func validateShardBoundMessage(consensus *consensus.Consensus, peer libp2p_peer.
 		if maybeCon.ViewId+5 < consensus.GetCurBlockViewID() {
 			return nil, nil, true, errors.WithStack(errViewIDTooOld)
 		}
-	} else if maybeVC != nil {
+	case maybeVC != nil:
 		if maybeVC.ShardId != consensus.ShardID {
 			nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_shard"}).Inc()
 			return nil, nil, true, errors.WithStack(errWrongShardID)
@@ -556,7 +561,14 @@ func validateShardBoundMessage(consensus *consensus.Consensus, peer libp2p_peer.
 		if maybeVC.ViewId+5 < consensus.GetViewChangingID() {
 			return nil, nil, true, errors.WithStack(errViewIDTooOld)
 		}
-	} else {
+	case maybeSP != nil:
+		if maybeSP.ShardId != consensus.ShardID {
+			nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_shard"}).Inc()
+			return nil, nil, true, errors.WithStack(errWrongShardID)
+		}
+		senderKey = maybeSP.SenderPubkey
+		consensus.SetLastKnownSignPower(maybeSP.Commit, maybeSP.Change)
+	default:
 		nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid"}).Inc()
 		return nil, nil, true, errors.WithStack(errNoSenderPubKey)
 	}
diff --git a/rpc/harmony/private_debug.go b/rpc/harmony/private_debug.go
index 97ade82dd2..2da571b5f2 100644
--- a/rpc/harmony/private_debug.go
+++ b/rpc/harmony/private_debug.go
@@ -58,17 +58,3 @@ func (s *PrivateDebugService) GetConfig(
 ) (StructuredResponse, error) {
 	return NewStructuredResponse(s.hmy.NodeAPI.GetConfig())
 }
-
-// GetLastSigningPower get last signed power
-func (s *PrivateDebugService) GetLastSigningPower(
-	ctx context.Context,
-) (float64, error) {
-	return s.hmy.NodeAPI.GetLastSigningPower()
-}
-
-// GetLastSigningPower2 get last signed power
-func (s *PrivateDebugService) GetLastSigningPower2(
-	ctx context.Context,
-) (float64, error) {
-	return s.hmy.NodeAPI.GetLastSigningPower2()
-}
diff --git a/rpc/harmony/public_debug.go b/rpc/harmony/public_debug.go
index 08283b8342..e4f2c12d90 100644
--- a/rpc/harmony/public_debug.go
+++ b/rpc/harmony/public_debug.go
@@ -36,3 +36,10 @@ func (s *PublicDebugService) SetLogVerbosity(ctx context.Context, level int) (ma
 	utils.SetLogVerbosity(verbosity)
 	return map[string]interface{}{"verbosity": verbosity.String()}, nil
 }
+
+// GetLastSigningPower get last signed power
+func (s *PublicDebugService) GetLastSigningPower(
+	ctx context.Context,
+) (float64, error) {
+	return s.hmy.NodeAPI.GetLastSigningPower()
+}

From 0aaa180beceb83473103ffd6050a10aec1aa0699 Mon Sep 17 00:00:00 2001
From: Frozen <355847+Frozen@users.noreply.github.com>
Date: Fri, 27 Sep 2024 03:59:04 -0400
Subject: [PATCH 16/24] Broadcast epoch block (#4756)

* Broadcast epoch block
* Broadcast epoch block through node messages
* Cleanup
* Fix for detecting leader
---
 api/proto/node/node.go                  | 10 +++++++++
 api/service/legacysync/epoch_syncing.go |  2 +-
 api/service/legacysync/syncing.go       | 28 +------------------------
 consensus/consensus_v2.go               | 28 ++++++++++++++++++++++---
 consensus/leader.go                     |  6 +++---
 core/types.go                           | 27 ++++++++++++++++++++++++
 node/harmony/node.go                    |  4 ++++
 node/harmony/node_cross_link.go         | 21 +++++++++++++++++++
 node/harmony/node_handler.go            |  5 ++++-
 node/harmony/node_syncing.go            |  8 +++----
 10 files changed, 100 insertions(+), 39 deletions(-)

diff --git a/api/proto/node/node.go b/api/proto/node/node.go
index aff7ff382c..6aa4bc7271 100644
--- a/api/proto/node/node.go
+++ b/api/proto/node/node.go
@@ -47,6 +47,7 @@ const (
 	Receipt                             // cross-shard transaction receipts
 	SlashCandidate                      // A report of a double-signing event
 	CrosslinkHeartbeat                  // Heart beat signal for crosslinks. Needed for epoch chain.
+	Epoch
 )
 
 var (
@@ -61,6 +62,7 @@ var (
 	crossLinkB          = byte(CrossLink)
 	crossLinkHeardBeatB = byte(CrosslinkHeartbeat)
 	receiptB            = byte(Receipt)
+	epochB              = byte(Epoch)
 	// H suffix means header
 	slashH              = []byte{nodeB, blockB, slashB}
 	transactionListH    = []byte{nodeB, txnB, sendB}
@@ -69,6 +71,7 @@ var (
 	crossLinkH          = []byte{nodeB, blockB, crossLinkB}
 	cxReceiptH          = []byte{nodeB, blockB, receiptB}
 	crossLinkHeartBeatH = []byte{nodeB, blockB, crossLinkHeardBeatB}
+	epochBlockH         = []byte{nodeB, blockB, epochB}
 )
 
 // ConstructTransactionListMessageAccount constructs serialized transactions in account model
@@ -139,6 +142,13 @@ func ConstructCrossLinkMessage(bc engine.ChainReader, headers []*block.Header) [
 	return byteBuffer.Bytes()
 }
 
+// ConstructEpochBlockMessage creates epoch block message
+func ConstructEpochBlockMessage(blockBytes []byte) []byte {
+	byteBuffer := bytes.NewBuffer(epochBlockH)
+	byteBuffer.Write(blockBytes)
+	return byteBuffer.Bytes()
+}
+
 // ConstructCXReceiptsProof constructs cross shard receipts and related proof including
 // merkle proof, blockHeader and  commitSignatures
 func ConstructCXReceiptsProof(cxReceiptsProof *types.CXReceiptsProof) []byte {
diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go
index 09c4a62da8..e2a03ded7f 100644
--- a/api/service/legacysync/epoch_syncing.go
+++ b/api/service/legacysync/epoch_syncing.go
@@ -202,7 +202,7 @@ func ProcessStateSync(syncConfig *SyncConfig, heights []uint64, bc core.BlockCha
 func processWithPayload(payload [][]byte, bc core.BlockChain) error {
 	decoded := make([]*types.Block, 0, len(payload))
 	for idx, blockBytes := range payload {
-		block, err := RlpDecodeBlockOrBlockWithSig(blockBytes)
+		block, err := core.RlpDecodeBlockOrBlockWithSig(blockBytes)
 		if err != nil {
 			return errors.Wrap(err, "failed decode")
 		}
diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go
index 8746139607..a1dc0e0b5f 100644
--- a/api/service/legacysync/syncing.go
+++ b/api/service/legacysync/syncing.go
@@ -14,7 +14,6 @@ import (
 
 	"github.com/Workiva/go-datastructures/queue"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/harmony-one/harmony/api/service/legacysync/downloader"
 	pb "github.com/harmony-one/harmony/api/service/legacysync/downloader/proto"
 	"github.com/harmony-one/harmony/consensus"
@@ -356,13 +355,6 @@ func CompareSyncPeerConfigByblockHashes(a *SyncPeerConfig, b *SyncPeerConfig) in
 	return 0
 }
 
-// BlockWithSig the serialization structure for request DownloaderRequest_BLOCKWITHSIG
-// The block is encoded as block + commit signature
-type BlockWithSig struct {
-	Block              *types.Block
-	CommitSigAndBitmap []byte
-}
-
 // GetBlocks gets blocks by calling grpc request to the corresponding peer.
 func (peerConfig *SyncPeerConfig) GetBlocks(hashes [][]byte) ([][]byte, error) {
 	response := peerConfig.client.GetBlocksAndSigs(hashes)
@@ -663,7 +655,7 @@ func (ss *StateSync) handleBlockSyncResult(payload [][]byte, tasks syncBlockTask
 
 	for i, blockBytes := range payload {
 		// For forward compatibility at server side, it can be types.block or BlockWithSig
-		blockObj, err := RlpDecodeBlockOrBlockWithSig(blockBytes)
+		blockObj, err := core.RlpDecodeBlockOrBlockWithSig(blockBytes)
 		if err != nil {
 			utils.Logger().Warn().
 				Err(err).
@@ -689,24 +681,6 @@ func (ss *StateSync) handleBlockSyncResult(payload [][]byte, tasks syncBlockTask
 	return failedTasks
 }
 
-// RlpDecodeBlockOrBlockWithSig decode payload to types.Block or BlockWithSig.
-// Return the block with commitSig if set.
-func RlpDecodeBlockOrBlockWithSig(payload []byte) (*types.Block, error) {
-	var block *types.Block
-	if err := rlp.DecodeBytes(payload, &block); err == nil {
-		// received payload as *types.Block
-		return block, nil
-	}
-
-	var bws BlockWithSig
-	if err := rlp.DecodeBytes(payload, &bws); err == nil {
-		block := bws.Block
-		block.SetCurrentCommitSig(bws.CommitSigAndBitmap)
-		return block, nil
-	}
-	return nil, errors.New("failed to decode to either types.Block or BlockWithSig")
-}
-
 // downloadTaskQueue is wrapper around Queue with item to be SyncBlockTask
 type downloadTaskQueue struct {
 	q *queue.Queue
diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go
index 80f6d110ae..e4f2a6b59c 100644
--- a/consensus/consensus_v2.go
+++ b/consensus/consensus_v2.go
@@ -9,8 +9,10 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
 	bls2 "github.com/harmony-one/bls/ffi/go/bls"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
+	proto_node "github.com/harmony-one/harmony/api/proto/node"
 	"github.com/harmony-one/harmony/block"
 	"github.com/harmony-one/harmony/consensus/quorum"
 	"github.com/harmony-one/harmony/consensus/signature"
@@ -136,7 +138,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, peer libp2p
 	return nil
 }
 
-func (consensus *Consensus) finalCommit() {
+func (consensus *Consensus) finalCommit(isLeader bool) {
 	numCommits := consensus.decider.SignersCount(quorum.Commit)
 
 	consensus.getLogger().Info().
@@ -187,7 +189,7 @@ func (consensus *Consensus) finalCommit() {
 	// Note: leader already sent 67% commit in preCommit. The 100% commit won't be sent immediately
 	// to save network traffic. It will only be sent in retry if consensus doesn't move forward.
 	// Or if the leader is changed for next block, the 100% committed sig will be sent to the next leader immediately.
-	if !consensus.isLeader() || block.IsLastBlockInEpoch() {
+	if !isLeader || block.IsLastBlockInEpoch() {
 		// send immediately
 		if err := consensus.msgSender.SendWithRetry(
 			block.NumberU64(),
@@ -228,6 +230,26 @@ func (consensus *Consensus) finalCommit() {
 			Msg("[finalCommit] Leader failed to commit the confirmed block")
 	}
 
+	if consensus.ShardID == 0 && isLeader && block.IsLastBlockInEpoch() {
+		blockWithSig := core.BlockWithSig{
+			Block:              block,
+			CommitSigAndBitmap: commitSigAndBitmap,
+		}
+		encodedBlock, err := rlp.EncodeToBytes(blockWithSig)
+		if err != nil {
+			consensus.getLogger().Debug().Msg("[Announce] Failed encoding block")
+			return
+		}
+		err = consensus.host.SendMessageToGroups(
+			[]nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(1)},
+			p2p.ConstructMessage(
+				proto_node.ConstructEpochBlockMessage(encodedBlock)),
+		)
+		if err != nil {
+			consensus.getLogger().Warn().Err(err).Msg("[finalCommit] failed to send epoch block")
+		}
+	}
+
 	// Dump new block into level db
 	// In current code, we add signatures in block in tryCatchup, the block dump to explorer does not contains signatures
 	// but since explorer doesn't need signatures, it should be fine
@@ -252,7 +274,7 @@ func (consensus *Consensus) finalCommit() {
 
 	// If still the leader, send commit sig/bitmap to finish the new block proposal,
 	// else, the block proposal will timeout by itself.
-	if consensus.isLeader() {
+	if isLeader {
 		if block.IsLastBlockInEpoch() {
 			// No pipelining
 			go func() {
diff --git a/consensus/leader.go b/consensus/leader.go
index d851668aa8..8f3fd3d59a 100644
--- a/consensus/leader.go
+++ b/consensus/leader.go
@@ -299,7 +299,7 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) {
 			consensus.preCommitAndPropose(blockObj)
 		}
 
-		go func(viewID uint64) {
+		go func(viewID uint64, isLeader bool) {
 			waitTime := 1000 * time.Millisecond
 			maxWaitTime := time.Until(consensus.NextBlockDue) - 200*time.Millisecond
 			if maxWaitTime > waitTime {
@@ -313,9 +313,9 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) {
 			consensus.mutex.Lock()
 			defer consensus.mutex.Unlock()
 			if viewID == consensus.getCurBlockViewID() {
-				consensus.finalCommit()
+				consensus.finalCommit(isLeader)
 			}
-		}(viewID)
+		}(viewID, consensus.isLeader())
 
 		consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED)
 	}
diff --git a/core/types.go b/core/types.go
index 4dfd24164f..5ec8a010af 100644
--- a/core/types.go
+++ b/core/types.go
@@ -17,11 +17,13 @@
 package core
 
 import (
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/harmony-one/harmony/consensus/reward"
 	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
 	stakingTypes "github.com/harmony-one/harmony/staking/types"
+	"github.com/pkg/errors"
 )
 
 // Validator is an interface which defines the standard for block validation. It
@@ -59,3 +61,28 @@ type Processor interface {
 	)
 	CacheProcessorResult(cacheKey interface{}, result *ProcessorResult)
 }
+
+// RlpDecodeBlockOrBlockWithSig decode payload to types.Block or BlockWithSig.
+// Return the block with commitSig if set.
+func RlpDecodeBlockOrBlockWithSig(payload []byte) (*types.Block, error) {
+	var block *types.Block
+	if err := rlp.DecodeBytes(payload, &block); err == nil {
+		// received payload as *types.Block
+		return block, nil
+	}
+
+	var bws BlockWithSig
+	if err := rlp.DecodeBytes(payload, &bws); err == nil {
+		block := bws.Block
+		block.SetCurrentCommitSig(bws.CommitSigAndBitmap)
+		return block, nil
+	}
+	return nil, errors.New("failed to decode to either types.Block or BlockWithSig")
+}
+
+// BlockWithSig the serialization structure for request DownloaderRequest_BLOCKWITHSIG
+// The block is encoded as block + commit signature
+type BlockWithSig struct {
+	Block              *types.Block
+	CommitSigAndBitmap []byte
+}
diff --git a/node/harmony/node.go b/node/harmony/node.go
index 4cb34eccec..edea6067ee 100644
--- a/node/harmony/node.go
+++ b/node/harmony/node.go
@@ -459,6 +459,10 @@ func (node *Node) validateNodeMessage(ctx context.Context, payload []byte) (
 			if node.IsRunningBeaconChain() {
 				return nil, 0, errInvalidShard
 			}
+		case proto_node.Epoch:
+			if node.IsRunningBeaconChain() {
+				return nil, 0, errInvalidShard
+			}
 		default:
 			nodeNodeMessageCounterVec.With(prometheus.Labels{"type": "invalid_block_type"}).Inc()
 			return nil, 0, errInvalidNodeMsg
diff --git a/node/harmony/node_cross_link.go b/node/harmony/node_cross_link.go
index 26a3509e75..e8a162be9c 100644
--- a/node/harmony/node_cross_link.go
+++ b/node/harmony/node_cross_link.go
@@ -28,6 +28,27 @@ func (node *Node) ProcessCrossLinkHeartbeatMessage(msgPayload []byte) {
 	}
 }
 
+func (node *Node) processEpochBlockMessage(msgPayload []byte) error {
+	if node.IsRunningBeaconChain() {
+		return errors.New("received beacon block for beacon chain")
+	}
+	block, err := core.RlpDecodeBlockOrBlockWithSig(msgPayload)
+	if err != nil {
+		return errors.WithMessage(err, "failed to decode block")
+	}
+	if _, err := node.EpochChain().InsertChain(types.Blocks{block}, true); err != nil {
+		return errors.WithMessage(err, "failed insert epoch block")
+	}
+	return nil
+}
+
+func (node *Node) ProcessEpochBlockMessage(msgPayload []byte) {
+	if err := node.processEpochBlockMessage(msgPayload); err != nil {
+		utils.Logger().Err(err).
+			Msg("[ProcessEpochBlock] failed process epoch block")
+	}
+}
+
 func (node *Node) processCrossLinkHeartbeatMessage(msgPayload []byte) error {
 	hb := types.CrosslinkHeartbeat{}
 	err := rlp.DecodeBytes(msgPayload, &hb)
diff --git a/node/harmony/node_handler.go b/node/harmony/node_handler.go
index 02c6f30884..926e9e3047 100644
--- a/node/harmony/node_handler.go
+++ b/node/harmony/node_handler.go
@@ -42,6 +42,8 @@ func (node *Node) processSkippedMsgTypeByteValue(
 		node.ProcessCrossLinkMessage(content)
 	case proto_node.CrosslinkHeartbeat:
 		node.ProcessCrossLinkHeartbeatMessage(content)
+	case proto_node.Epoch:
+		node.ProcessEpochBlockMessage(content)
 	default:
 		utils.Logger().Error().
 			Int("message-iota-value", int(cat)).
@@ -86,7 +88,8 @@ func (node *Node) HandleNodeMessage(
 			proto_node.SlashCandidate,
 			proto_node.Receipt,
 			proto_node.CrossLink,
-			proto_node.CrosslinkHeartbeat:
+			proto_node.CrosslinkHeartbeat,
+			proto_node.Epoch:
 			// skip first byte which is blockMsgType
 			node.processSkippedMsgTypeByteValue(blockMsgType, msgPayload[1:])
 		}
diff --git a/node/harmony/node_syncing.go b/node/harmony/node_syncing.go
index 15f097e2ec..34eaaec0c0 100644
--- a/node/harmony/node_syncing.go
+++ b/node/harmony/node_syncing.go
@@ -554,7 +554,7 @@ func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, in
 			response.Type = downloader_pb.DownloaderResponse_INSYNC
 			return response, nil
 		}
-		block, err := legacysync.RlpDecodeBlockOrBlockWithSig(request.BlockHash)
+		block, err := core.RlpDecodeBlockOrBlockWithSig(request.BlockHash)
 		if err != nil {
 			utils.Logger().Warn().Err(err).Msg("[SYNC] unable to decode received new block")
 			return response, err
@@ -721,7 +721,7 @@ func (node *Node) getEncodedBlockWithSigByHash(hash common.Hash) ([]byte, error)
 	if err != nil {
 		return nil, err
 	}
-	bwh := legacysync.BlockWithSig{
+	bwh := core.BlockWithSig{
 		Block:              blk,
 		CommitSigAndBitmap: sab,
 	}
@@ -742,7 +742,7 @@ func (node *Node) getEncodedBlockWithSigByHeight(height uint64) ([]byte, error)
 	if err != nil {
 		return nil, err
 	}
-	bwh := legacysync.BlockWithSig{
+	bwh := core.BlockWithSig{
 		Block:              blk,
 		CommitSigAndBitmap: sab,
 	}
@@ -754,7 +754,7 @@ func (node *Node) getEncodedBlockWithSigByHeight(height uint64) ([]byte, error)
 }
 
 func getEncodedBlockWithSigFromBlock(block *types.Block) ([]byte, error) {
-	bwh := legacysync.BlockWithSig{
+	bwh := core.BlockWithSig{
 		Block:              block,
 		CommitSigAndBitmap: block.GetCurrentCommitSig(),
 	}

From feb518058b63c4ab9c4cc83ac9bf142de471c2c9 Mon Sep 17 00:00:00 2001
From: Soph <35721420+sophoah@users.noreply.github.com>
Date: Fri, 27 Sep 2024 14:59:39 +0700
Subject: [PATCH 17/24] [localnet] fix external config (#4763)

---
 test/configs/local-resharding-with-external.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/configs/local-resharding-with-external.txt b/test/configs/local-resharding-with-external.txt
index d146dd2678..ae35b24dae 100644
--- a/test/configs/local-resharding-with-external.txt
+++ b/test/configs/local-resharding-with-external.txt
@@ -22,8 +22,8 @@
 127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key
 127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key
 127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key
-127.0.0.1 9118 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key
-127.0.0.1 9122 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key
+127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key
+127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key
 # fn node
 127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key
 127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key

From aba5af0dc1723831df7d31020dfe5c949b065475 Mon Sep 17 00:00:00 2001
From: Gheis Mohammadi <Gheis.Mohammadi@gmail.com>
Date: Thu, 3 Oct 2024 11:55:13 +0800
Subject: [PATCH 18/24] switch localnet to yamux muxer (#4764)

---
 test/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/deploy.sh b/test/deploy.sh
index a4428f1c35..10797b8a02 100755
--- a/test/deploy.sh
+++ b/test/deploy.sh
@@ -72,7 +72,7 @@ function launch_localnet() {
     verbosity=3
   fi
 
-  base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" \
+  base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" --p2p.muxer "yamux" \
     "--network=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" \
     "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100")
   if [ "${LEGACY_SYNC}" == "true" ]; then

From 2d8fb0edc7c6e12fd8e6f2c4e71a325bebdc725d Mon Sep 17 00:00:00 2001
From: Uladzislau Muraveika <uladzislau.muraveika@protofire.io>
Date: Fri, 4 Oct 2024 06:44:01 +0300
Subject: [PATCH 19/24] feat(RPC) - GetLastSigningPower on public harmony API
 (#4766)

---
 rpc/harmony/harmony.go      | 7 +++++++
 rpc/harmony/public_debug.go | 7 -------
 test/deploy.sh              | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/rpc/harmony/harmony.go b/rpc/harmony/harmony.go
index c8bfe80628..a93c757d9e 100644
--- a/rpc/harmony/harmony.go
+++ b/rpc/harmony/harmony.go
@@ -107,3 +107,10 @@ func (s *PublicHarmonyService) GetNumPendingCrossLinks() (int, error) {
 
 	return len(links), nil
 }
+
+// GetLastSigningPower get last signed power
+func (s *PublicHarmonyService) GetLastSigningPower(
+	ctx context.Context,
+) (float64, error) {
+	return s.hmy.NodeAPI.GetLastSigningPower()
+}
diff --git a/rpc/harmony/public_debug.go b/rpc/harmony/public_debug.go
index e4f2c12d90..08283b8342 100644
--- a/rpc/harmony/public_debug.go
+++ b/rpc/harmony/public_debug.go
@@ -36,10 +36,3 @@ func (s *PublicDebugService) SetLogVerbosity(ctx context.Context, level int) (ma
 	utils.SetLogVerbosity(verbosity)
 	return map[string]interface{}{"verbosity": verbosity.String()}, nil
 }
-
-// GetLastSigningPower get last signed power
-func (s *PublicDebugService) GetLastSigningPower(
-	ctx context.Context,
-) (float64, error) {
-	return s.hmy.NodeAPI.GetLastSigningPower()
-}
diff --git a/test/deploy.sh b/test/deploy.sh
index 10797b8a02..7db7aa28fc 100755
--- a/test/deploy.sh
+++ b/test/deploy.sh
@@ -133,7 +133,7 @@ function launch_localnet() {
       args=("${args[@]}" --run.legacy)
       ;;
     validator)
-      args=("${args[@]}" --run.legacy "--rpc.debug=true")
+      args=("${args[@]}" --run.legacy)
       ;;
     esac
 

From c823abd857d25c14c90e9930ff693a5be00f532e Mon Sep 17 00:00:00 2001
From: Gheis Mohammadi <Gheis.Mohammadi@gmail.com>
Date: Fri, 4 Oct 2024 13:21:28 +0800
Subject: [PATCH 20/24] Improve Crosslink Retrieve from RawDB (#4765)

* fix spell sing/sign
* add logs to the error for offchain crosslink write
* add check crosslink epoch before trying to read crosslinks from db
---
 consensus/consensus_block_proposing.go | 15 ++++++++-----
 core/blockchain_impl.go                | 31 ++++++++++++++++++++++----
 core/offchain.go                       |  5 +++++
 hmy/blockchain.go                      |  2 ++
 node/harmony/node_handler.go           | 13 ++++++-----
 5 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/consensus/consensus_block_proposing.go b/consensus/consensus_block_proposing.go
index 927b707e6d..ab0f25ae0f 100644
--- a/consensus/consensus_block_proposing.go
+++ b/consensus/consensus_block_proposing.go
@@ -154,6 +154,12 @@ func (consensus *Consensus) ProposeNewBlock(commitSigs chan []byte) (*types.Bloc
 		invalidToDelete := []types.CrossLink{}
 		if err == nil {
 			for _, pending := range allPending {
+				if !consensus.Blockchain().Config().IsCrossLink(pending.Epoch()) {
+					invalidToDelete = append(invalidToDelete, pending)
+					utils.Logger().Debug().
+						AnErr("[ProposeNewBlock] pending crosslink that's before crosslink epoch", err)
+					continue
+				}
 				// ReadCrossLink beacon chain usage.
 				exist, err := consensus.Blockchain().ReadCrossLink(pending.ShardID(), pending.BlockNum())
 				if err == nil || exist != nil {
@@ -170,14 +176,11 @@ func (consensus *Consensus) ProposeNewBlock(commitSigs chan []byte) (*types.Bloc
 				}
 				// if pending crosslink is older than the last crosslink, delete it and continue
 				if err == nil && exist == nil && last != nil && last.BlockNum() >= pending.BlockNum() {
+					// Crosslink is already verified before it's accepted to pending,
+					// no need to verify again in proposal.
 					invalidToDelete = append(invalidToDelete, pending)
-				}
-
-				// Crosslink is already verified before it's accepted to pending,
-				// no need to verify again in proposal.
-				if !consensus.Blockchain().Config().IsCrossLink(pending.Epoch()) {
 					utils.Logger().Debug().
-						AnErr("[ProposeNewBlock] pending crosslink that's before crosslink epoch", err)
+						AnErr("[ProposeNewBlock] pending crosslink is older than last shard crosslink", err)
 					continue
 				}
 
diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go
index dbc12e91a6..a8408528e1 100644
--- a/core/blockchain_impl.go
+++ b/core/blockchain_impl.go
@@ -90,7 +90,10 @@ var (
 	blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
 	blockExecutionTimer  = metrics.NewRegisteredTimer("chain/execution", nil)
 	blockWriteTimer      = metrics.NewRegisteredTimer("chain/write", nil)
-
+	// ErrCrosslinkNotFound is the error when no crosslink found
+	ErrCrosslinkNotFound = errors.New("crosslink not found")
+	// ErrZeroBytes is the error when it reads empty crosslink
+	ErrZeroBytes = errors.New("crosslink read failed, zero bytes")
 	// ErrNoGenesis is the error when there is no genesis.
 	ErrNoGenesis           = errors.New("Genesis not found in chain")
 	ErrEmptyChain          = errors.New("empty chain")
@@ -2367,8 +2370,15 @@ func (bc *BlockChainImpl) DeleteCrossLinks(cls []types.CrossLink) error {
 
 func (bc *BlockChainImpl) ReadCrossLink(shardID uint32, blockNum uint64) (*types.CrossLink, error) {
 	bytes, err := rawdb.ReadCrossLinkShardBlock(bc.db, shardID, blockNum)
-	if err != nil {
+	if err == goleveldb.ErrNotFound {
+		utils.Logger().Debug().Msgf("ReadCrossLink: not found")
+		return nil, ErrCrosslinkNotFound
+	} else if err != nil {
+		utils.Logger().Debug().Err(err).Msgf("ReadCrossLink failed")
 		return nil, err
+	} else if len(bytes) == 0 {
+		utils.Logger().Debug().Err(ErrZeroBytes).Msgf("ReadCrossLink failed")
+		return nil, ErrZeroBytes
 	}
 	crossLink, err := types.DeserializeCrossLink(bytes)
 
@@ -2377,8 +2387,11 @@ func (bc *BlockChainImpl) ReadCrossLink(shardID uint32, blockNum uint64) (*types
 
 func (bc *BlockChainImpl) LastContinuousCrossLink(batch rawdb.DatabaseWriter, shardID uint32) error {
 	oldLink, err := bc.ReadShardLastCrossLink(shardID)
-	if oldLink == nil || err != nil {
+	if err != nil {
+		utils.Logger().Debug().Err(err).Msgf("LastContinuousCrossLink failed")
 		return err
+	} else if oldLink == nil {
+		return nil
 	}
 	newLink := oldLink
 	// Starting from last checkpoint, keeping reading immediate next crosslink until there is a gap
@@ -2399,9 +2412,19 @@ func (bc *BlockChainImpl) LastContinuousCrossLink(batch rawdb.DatabaseWriter, sh
 }
 
 func (bc *BlockChainImpl) ReadShardLastCrossLink(shardID uint32) (*types.CrossLink, error) {
+	if !bc.chainConfig.IsCrossLink(bc.CurrentBlock().Epoch()) {
+		return nil, nil
+	}
 	bytes, err := rawdb.ReadShardLastCrossLink(bc.db, shardID)
-	if err != nil {
+	if err == goleveldb.ErrNotFound {
+		utils.Logger().Debug().Msgf("ReadShardLastCrossLink: not found")
+		return nil, ErrCrosslinkNotFound
+	} else if err != nil {
+		utils.Logger().Debug().Err(err).Msgf("ReadShardLastCrossLink failed")
 		return nil, err
+	} else if len(bytes) == 0 {
+		utils.Logger().Debug().Err(ErrZeroBytes).Msgf("ReadShardLastCrossLink failed")
+		return nil, ErrZeroBytes
 	}
 	return types.DeserializeCrossLink(bytes)
 }
diff --git a/core/offchain.go b/core/offchain.go
index 6ce7794e4f..9f3e80a60d 100644
--- a/core/offchain.go
+++ b/core/offchain.go
@@ -163,6 +163,11 @@ func (bc *BlockChainImpl) CommitOffChainData(
 					Uint64("blockNum", crossLink.BlockNum()).
 					Uint32("shardID", crossLink.ShardID()).
 					Msg("[insertChain/crosslinks] Cross Link Added to Beaconchain")
+			} else {
+				utils.Logger().Debug().Err(err).
+					Uint64("blockNum", crossLink.BlockNum()).
+					Uint32("shardID", crossLink.ShardID()).
+					Msg("[insertChain/crosslinks] Cross Link failed to Add to Beaconchain")
 			}
 
 			cl0, _ := bc.ReadShardLastCrossLink(crossLink.ShardID())
diff --git a/hmy/blockchain.go b/hmy/blockchain.go
index 558d2a821d..029d0b20a8 100644
--- a/hmy/blockchain.go
+++ b/hmy/blockchain.go
@@ -196,6 +196,8 @@ func (hmy *Harmony) GetLastCrossLinks() ([]*types.CrossLink, error) {
 		link, err := hmy.BlockChain.ReadShardLastCrossLink(i)
 		if err != nil {
 			return nil, err
+		} else if link == nil { // if it's before epoch crosslink HF, it returns nil
+			continue
 		}
 		crossLinks = append(crossLinks, link)
 	}
diff --git a/node/harmony/node_handler.go b/node/harmony/node_handler.go
index 926e9e3047..014c739e81 100644
--- a/node/harmony/node_handler.go
+++ b/node/harmony/node_handler.go
@@ -236,15 +236,15 @@ func (node *Node) BroadcastCrosslinkHeartbeatSignalFromBeaconToShards() { // lea
 		return
 	}
 
-	var privToSing *bls.PrivateKeyWrapper
+	var privToSign *bls.PrivateKeyWrapper
 	for _, priv := range node.Consensus.GetPrivateKeys() {
 		if node.Consensus.IsValidatorInCommittee(priv.Pub.Bytes) {
-			privToSing = &priv
+			privToSign = &priv
 			break
 		}
 	}
 
-	if privToSing == nil {
+	if privToSign == nil {
 		return
 	}
 	instance := shard.Schedule.InstanceForEpoch(curBlock.Epoch())
@@ -253,13 +253,14 @@ func (node *Node) BroadcastCrosslinkHeartbeatSignalFromBeaconToShards() { // lea
 		if err != nil {
 			utils.Logger().Error().Err(err).Msg("[BroadcastCrossLinkSignal] failed to get crosslinks")
 			continue
+		} else if lastLink == nil {
+			continue
 		}
-
 		hb := types.CrosslinkHeartbeat{
 			ShardID:                  lastLink.ShardID(),
 			LatestContinuousBlockNum: lastLink.BlockNum(),
 			Epoch:                    lastLink.Epoch().Uint64(),
-			PublicKey:                privToSing.Pub.Bytes[:],
+			PublicKey:                privToSign.Pub.Bytes[:],
 			Signature:                nil,
 		}
 
@@ -268,7 +269,7 @@ func (node *Node) BroadcastCrosslinkHeartbeatSignalFromBeaconToShards() { // lea
 			utils.Logger().Error().Err(err).Msg("[BroadcastCrossLinkSignal] failed to encode signal")
 			continue
 		}
-		hb.Signature = privToSing.Pri.SignHash(rs).Serialize()
+		hb.Signature = privToSign.Pri.SignHash(rs).Serialize()
 		bts := proto_node.ConstructCrossLinkHeartBeatMessage(hb)
 		node.host.SendMessageToGroups(
 			[]nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(shardID))},

From 544f2ac5a27de0c8612ea7da64af7a47d014fb98 Mon Sep 17 00:00:00 2001
From: Gheis Mohammadi <Gheis.Mohammadi@gmail.com>
Date: Fri, 4 Oct 2024 13:22:07 +0800
Subject: [PATCH 21/24] Not Checking Pending Crosslinks for First ten Epoch
 (#4757)

* not checking pending crosslinks for first ten epochs
* check crosslink epoch to delete old pending crosslinks
---
 node/harmony/node.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/node/harmony/node.go b/node/harmony/node.go
index edea6067ee..c33cd5dd55 100644
--- a/node/harmony/node.go
+++ b/node/harmony/node.go
@@ -1123,9 +1123,10 @@ func New(
 	node.serviceManager = service.NewManager()
 
 	// delete old pending crosslinks
-	if node.Blockchain().ShardID() == shard.BeaconChainShardID {
-		ten := big.NewInt(10)
-		crossLinkEpochThreshold := new(big.Int).Sub(node.Blockchain().CurrentHeader().Epoch(), ten)
+	ten := big.NewInt(10)
+	epoch := node.Blockchain().CurrentHeader().Epoch()
+	if node.Blockchain().ShardID() == shard.BeaconChainShardID && node.Blockchain().Config().IsCrossLink(epoch) {
+		crossLinkEpochThreshold := new(big.Int).Sub(epoch, ten)
 
 		invalidToDelete := make([]types.CrossLink, 0, 1000)
 		allPending, err := node.Blockchain().ReadPendingCrossLinks()

From 1588c05bfceb5b99c6b0b9490e5a88c58de3cb50 Mon Sep 17 00:00:00 2001
From: Frozen <355847+Frozen@users.noreply.github.com>
Date: Fri, 4 Oct 2024 05:21:14 -0400
Subject: [PATCH 22/24] Empty signing power workaround (#4663)

* Activation flag HIP32
* Fixed merge conflicts
* Function doc updated
---
 consensus/post_processing.go         |  2 +-
 hmy/staking.go                       |  6 +--
 internal/params/config.go            | 10 +++++
 shard/committee/assignment.go        | 13 +++---
 staking/availability/interface.go    |  2 +
 staking/availability/measure.go      |  9 ++--
 staking/availability/measure_test.go | 67 +++++++++++++++++++++++++---
 7 files changed, 88 insertions(+), 21 deletions(-)

diff --git a/consensus/post_processing.go b/consensus/post_processing.go
index 384546f713..849ab703bf 100644
--- a/consensus/post_processing.go
+++ b/consensus/post_processing.go
@@ -85,7 +85,7 @@ func (consensus *Consensus) PostConsensusProcessing(newBlock *types.Block) error
 					return nil
 				}
 				computed := availability.ComputeCurrentSigning(
-					snapshot.Validator, wrapper,
+					snapshot.Validator, wrapper, consensus.Blockchain().Config().IsHIP32(newBlock.Epoch()),
 				)
 				lastBlockOfEpoch := shard.Schedule.EpochLastBlock(consensus.Beaconchain().CurrentBlock().Header().Epoch().Uint64())
 
diff --git a/hmy/staking.go b/hmy/staking.go
index f80f355886..bab9c0adff 100644
--- a/hmy/staking.go
+++ b/hmy/staking.go
@@ -353,7 +353,7 @@ func (hmy *Harmony) GetValidatorInformation(
 	}
 
 	computed := availability.ComputeCurrentSigning(
-		snapshot.Validator, wrapper,
+		snapshot.Validator, wrapper, bc.Config().IsHIP32(now),
 	)
 
 	lastBlockOfEpoch := shard.Schedule.EpochLastBlock(hmy.BeaconChain.CurrentBlock().Header().Epoch().Uint64())
@@ -479,7 +479,7 @@ func (hmy *Harmony) GetMedianRawStakeSnapshot() (
 			// Compute for next epoch
 			epoch := big.NewInt(0).Add(hmy.CurrentBlock().Epoch(), big.NewInt(1))
 			instance := shard.Schedule.InstanceForEpoch(epoch)
-			return committee.NewEPoSRound(epoch, hmy.BlockChain, hmy.BlockChain.Config().IsEPoSBound35(epoch), instance.SlotsLimit(), int(instance.NumShards()))
+			return committee.NewEPoSRound(epoch, hmy.BlockChain, instance.SlotsLimit(), int(instance.NumShards()))
 		},
 	)
 	if err != nil {
@@ -634,7 +634,7 @@ func (hmy *Harmony) GetTotalStakingSnapshot() *big.Int {
 		snapshot, _ := hmy.BlockChain.ReadValidatorSnapshot(candidates[i])
 		validator, _ := hmy.BlockChain.ReadValidatorInformation(candidates[i])
 		if !committee.IsEligibleForEPoSAuction(
-			snapshot, validator,
+			snapshot, validator, hmy.BlockChain.Config(),
 		) {
 			continue
 		}
diff --git a/internal/params/config.go b/internal/params/config.go
index 61b1ae23fc..a258f131f7 100644
--- a/internal/params/config.go
+++ b/internal/params/config.go
@@ -365,6 +365,7 @@ var (
 		big.NewInt(0),                      // MaxRateEpoch
 		big.NewInt(0),
 		big.NewInt(0),
+		big.NewInt(0),
 	}
 
 	// TestChainConfig ...
@@ -414,6 +415,7 @@ var (
 		big.NewInt(0),        // MaxRateEpoch
 		big.NewInt(0),        // MaxRateEpoch
 		big.NewInt(0),
+		big.NewInt(0),
 	}
 
 	// TestRules ...
@@ -589,6 +591,10 @@ type ChainConfig struct {
 
 	// TopMaxRateEpoch will make sure the validator max-rate is less to 100% for the cases where the minRate + the validator max-rate-increase > 100%
 	TopMaxRateEpoch *big.Int `json:"top-max-rate-epoch,omitempty"`
+
+	// vote power feature  https://github.com/harmony-one/harmony/pull/4683
+	// if crosslink are not sent for an entire epoch signed and toSign will be 0 and 0. when that happen, next epoch there will no shard 1 validator elected in the committee.
+	HIP32Epoch *big.Int `json:"hip32-epoch,omitempty"`
 }
 
 // String implements the fmt.Stringer interface.
@@ -844,6 +850,10 @@ func (c *ChainConfig) IsValidatorCodeFix(epoch *big.Int) bool {
 	return isForked(c.ValidatorCodeFixEpoch, epoch)
 }
 
+func (c *ChainConfig) IsHIP32(epoch *big.Int) bool {
+	return isForked(c.HIP32Epoch, epoch)
+}
+
 func (c *ChainConfig) IsHIP30(epoch *big.Int) bool {
 	return isForked(c.HIP30Epoch, epoch)
 }
diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go
index 4978b61559..5fba3e6200 100644
--- a/shard/committee/assignment.go
+++ b/shard/committee/assignment.go
@@ -35,6 +35,7 @@ type StakingCandidatesReader interface {
 	) (*staking.ValidatorWrapper, error)
 	ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorSnapshot, error)
 	ValidatorCandidates() []common.Address
+	Config() *params.ChainConfig
 }
 
 // CandidatesForEPoS ..
@@ -73,7 +74,7 @@ func (p CandidateOrder) MarshalJSON() ([]byte, error) {
 
 // NewEPoSRound runs a fresh computation of EPoS using
 // latest data always
-func NewEPoSRound(epoch *big.Int, stakedReader StakingCandidatesReader, isExtendedBound bool, slotsLimit, shardCount int) (
+func NewEPoSRound(epoch *big.Int, stakedReader StakingCandidatesReader, slotsLimit, shardCount int) (
 	*CompletedEPoSRound, error,
 ) {
 	eligibleCandidate, err := prepareOrders(stakedReader, slotsLimit, shardCount)
@@ -84,7 +85,7 @@ func NewEPoSRound(epoch *big.Int, stakedReader StakingCandidatesReader, isExtend
 		epoch,
 	)
 	median, winners := effective.Apply(
-		eligibleCandidate, maxExternalSlots, isExtendedBound,
+		eligibleCandidate, maxExternalSlots, stakedReader.Config().IsEPoSBound35(epoch),
 	)
 	auctionCandidates := make([]*CandidateOrder, len(eligibleCandidate))
 
@@ -159,7 +160,7 @@ func prepareOrders(
 		if err != nil {
 			return nil, err
 		}
-		if !IsEligibleForEPoSAuction(snapshot, validator) {
+		if !IsEligibleForEPoSAuction(snapshot, validator, stakedReader.Config()) {
 			continue
 		}
 
@@ -208,7 +209,7 @@ func prepareOrders(
 }
 
 // IsEligibleForEPoSAuction ..
-func IsEligibleForEPoSAuction(snapshot *staking.ValidatorSnapshot, validator *staking.ValidatorWrapper) bool {
+func IsEligibleForEPoSAuction(snapshot *staking.ValidatorSnapshot, validator *staking.ValidatorWrapper, config *params.ChainConfig) bool {
 	// This original condition to check whether a validator is in last committee is not stable
 	// because cross-links may arrive after the epoch ends and it still got counted into the
 	// NumBlocksToSign, making this condition to be true when the validator is actually not in committee
@@ -219,7 +220,7 @@ func IsEligibleForEPoSAuction(snapshot *staking.ValidatorSnapshot, validator *st
 		// validator was in last epoch's committee
 		// validator with below-threshold signing activity won't be considered for next epoch
 		// and their status will be turned to inactive in FinalizeNewBlock
-		computed := availability.ComputeCurrentSigning(snapshot.Validator, validator)
+		computed := availability.ComputeCurrentSigning(snapshot.Validator, validator, config.IsHIP32(snapshot.Epoch))
 		if computed.IsBelowThreshold {
 			return false
 		}
@@ -349,7 +350,7 @@ func eposStakedCommittee(
 	}
 
 	// TODO(audit): make sure external validator BLS key are also not duplicate to Harmony's keys
-	completedEPoSRound, err := NewEPoSRound(epoch, stakerReader, stakerReader.Config().IsEPoSBound35(epoch), s.SlotsLimit(), shardCount)
+	completedEPoSRound, err := NewEPoSRound(epoch, stakerReader, s.SlotsLimit(), shardCount)
 
 	if err != nil {
 		return nil, err
diff --git a/staking/availability/interface.go b/staking/availability/interface.go
index 5b5c781943..f7df8482ef 100644
--- a/staking/availability/interface.go
+++ b/staking/availability/interface.go
@@ -4,6 +4,7 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/harmony-one/harmony/internal/params"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
@@ -12,6 +13,7 @@ type Reader interface {
 	ReadValidatorSnapshot(
 		addr common.Address,
 	) (*staking.ValidatorSnapshot, error)
+	Config() *params.ChainConfig
 }
 
 // RoundHeader is the interface of block.Header for calculating the BallotResult.
diff --git a/staking/availability/measure.go b/staking/availability/measure.go
index a1c48845dd..07f8c4e60e 100644
--- a/staking/availability/measure.go
+++ b/staking/availability/measure.go
@@ -138,9 +138,7 @@ func IncrementValidatorSigningCounts(
 }
 
 // ComputeCurrentSigning returns (signed, toSign, quotient, error)
-func ComputeCurrentSigning(
-	snapshot, wrapper *staking.ValidatorWrapper,
-) *staking.Computed {
+func ComputeCurrentSigning(snapshot, wrapper *staking.ValidatorWrapper, isHip32 bool) *staking.Computed {
 	statsNow, snapSigned, snapToSign :=
 		wrapper.Counters,
 		snapshot.Counters.NumBlocksSigned,
@@ -155,6 +153,9 @@ func ComputeCurrentSigning(
 	)
 
 	if toSign.Cmp(common.Big0) == 0 {
+		if isHip32 {
+			computed.IsBelowThreshold = false
+		}
 		return computed
 	}
 
@@ -205,7 +206,7 @@ func ComputeAndMutateEPOSStatus(
 		return err
 	}
 
-	computed := ComputeCurrentSigning(snapshot.Validator, wrapper)
+	computed := ComputeCurrentSigning(snapshot.Validator, wrapper, bc.Config().IsHIP32(snapshot.Epoch))
 
 	utils.Logger().
 		Info().Msg("check if signing percent is meeting required threshold")
diff --git a/staking/availability/measure_test.go b/staking/availability/measure_test.go
index 786d59ec04..8d635f45b9 100644
--- a/staking/availability/measure_test.go
+++ b/staking/availability/measure_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/harmony-one/harmony/crypto/bls"
+	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/effective"
@@ -188,7 +189,7 @@ func TestComputeCurrentSigning(t *testing.T) {
 		snapWrapper := makeTestWrapper(common.Address{}, test.snapSigned, test.snapToSign)
 		curWrapper := makeTestWrapper(common.Address{}, test.curSigned, test.curToSign)
 
-		computed := ComputeCurrentSigning(&snapWrapper, &curWrapper)
+		computed := ComputeCurrentSigning(&snapWrapper, &curWrapper, false)
 
 		if computed.Signed.Cmp(new(big.Int).SetInt64(test.diffSigned)) != 0 {
 			t.Errorf("test %v: computed signed not expected: %v / %v",
@@ -204,7 +205,48 @@ func TestComputeCurrentSigning(t *testing.T) {
 				i, computed.Percentage, expPct)
 		}
 		if computed.IsBelowThreshold != test.isBelowThreshold {
-			t.Errorf("test %v: computed is below threshold not expected: %v / %v",
+			t.Errorf("test %v: computed is below threshold `%v` expected: `%v`",
+				i, computed.IsBelowThreshold, test.isBelowThreshold)
+		}
+	}
+}
+
+func TestComputeCurrentSigningHIP32(t *testing.T) {
+	tests := []struct {
+		snapSigned, curSigned, diffSigned int64
+		snapToSign, curToSign, diffToSign int64
+		pctNum, pctDiv                    int64
+		isBelowThreshold                  bool
+	}{
+		{0, 0, 0, 0, 0, 0, 0, 1, false},
+		{0, 1, 1, 0, 1, 1, 1, 1, false},
+		{0, 2, 2, 0, 3, 3, 2, 3, true},
+		{0, 1, 1, 0, 3, 3, 1, 3, true},
+		{100, 225, 125, 200, 350, 150, 5, 6, false},
+		{100, 200, 100, 200, 350, 150, 2, 3, true},
+		{100, 200, 100, 200, 400, 200, 1, 2, true},
+	}
+	for i, test := range tests {
+		snapWrapper := makeTestWrapper(common.Address{}, test.snapSigned, test.snapToSign)
+		curWrapper := makeTestWrapper(common.Address{}, test.curSigned, test.curToSign)
+
+		computed := ComputeCurrentSigning(&snapWrapper, &curWrapper, true)
+
+		if computed.Signed.Cmp(new(big.Int).SetInt64(test.diffSigned)) != 0 {
+			t.Errorf("test %v: computed signed not expected: %v / %v",
+				i, computed.Signed, test.diffSigned)
+		}
+		if computed.ToSign.Cmp(new(big.Int).SetInt64(test.diffToSign)) != 0 {
+			t.Errorf("test %v: computed to sign not expected: %v / %v",
+				i, computed.ToSign, test.diffToSign)
+		}
+		expPct := numeric.NewDec(test.pctNum).Quo(numeric.NewDec(test.pctDiv))
+		if !computed.Percentage.Equal(expPct) {
+			t.Errorf("test %v: computed percentage not expected: %v / %v",
+				i, computed.Percentage, expPct)
+		}
+		if computed.IsBelowThreshold != test.isBelowThreshold {
+			t.Errorf("test %v: computed is below threshold `%v`, expected: `%v`",
 				i, computed.IsBelowThreshold, test.isBelowThreshold)
 		}
 	}
@@ -628,16 +670,22 @@ func (state testStateDB) GetCode(addr common.Address, isValidatorCode bool) []by
 }
 
 // testReader is the fake Reader for testing
-type testReader map[common.Address]staking.ValidatorWrapper
+type testReader struct {
+	m      map[common.Address]staking.ValidatorWrapper
+	config *params.ChainConfig
+}
 
 // newTestReader creates an empty test reader
 func newTestReader() testReader {
-	reader := make(testReader)
-	return reader
+	m := make(map[common.Address]staking.ValidatorWrapper)
+	return testReader{
+		m:      m,
+		config: &params.ChainConfig{},
+	}
 }
 
 func (reader testReader) ReadValidatorSnapshot(addr common.Address) (*staking.ValidatorSnapshot, error) {
-	wrapper, ok := reader[addr]
+	wrapper, ok := reader.m[addr]
 	if !ok {
 		return nil, errors.New("not a valid validator address")
 	}
@@ -646,8 +694,12 @@ func (reader testReader) ReadValidatorSnapshot(addr common.Address) (*staking.Va
 	}, nil
 }
 
+func (reader testReader) Config() *params.ChainConfig {
+	return reader.config
+}
+
 func (reader testReader) updateValidatorWrapper(addr common.Address, val *staking.ValidatorWrapper) {
-	reader[addr] = *val
+	reader.m[addr] = *val
 }
 
 func makeTestShardState(numShards, numSlots int) *shard.State {
@@ -724,6 +776,7 @@ func indexesToBitMap(idxs []int, n int) ([]byte, error) {
 	return res, nil
 }
 
+// makeTestWrapper makes test wrapper
 func makeTestWrapper(addr common.Address, numSigned, numToSign int64) staking.ValidatorWrapper {
 	var val staking.ValidatorWrapper
 	val.Address = addr

From 672cd76f879bfd118066b7c7d37d6069d5a25d07 Mon Sep 17 00:00:00 2001
From: Frozen <355847+Frozen@users.noreply.github.com>
Date: Tue, 8 Oct 2024 02:54:36 -0400
Subject: [PATCH 23/24] HIP32 epoch activation and new sharding config (#4769)

* 2151 epoch as 2024-10-30 19:50 UTC.
* Mainnet v5 with new sharding config
---
 internal/configs/sharding/mainnet.go | 11 +++++++++++
 internal/params/config.go            |  5 +++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/internal/configs/sharding/mainnet.go b/internal/configs/sharding/mainnet.go
index e237b85e97..90f4871569 100644
--- a/internal/configs/sharding/mainnet.go
+++ b/internal/configs/sharding/mainnet.go
@@ -73,6 +73,8 @@ type mainnetSchedule struct{}
 
 func (ms mainnetSchedule) InstanceForEpoch(epoch *big.Int) Instance {
 	switch {
+	case params.MainnetChainConfig.IsHIP32(epoch):
+		return mainnetV5
 	case params.MainnetChainConfig.IsHIP30(epoch):
 		return mainnetV4
 	case params.MainnetChainConfig.IsFeeCollectEpoch(epoch):
@@ -376,4 +378,13 @@ var (
 		hip30CollectionAddress, mainnetReshardingEpoch,
 		MainnetSchedule.BlocksPerEpoch(),
 	)
+	mainnetV5 = MustNewInstance(
+		2, 200, 2, 0.06,
+		numeric.MustNewDecFromStr("0.01"),
+		genesis.HarmonyAccountsPostHIP30,
+		genesis.FoundationalNodeAccountsV1_5, emptyAllowlist,
+		feeCollectorsMainnet, numeric.MustNewDecFromStr("0.25"),
+		hip30CollectionAddress, mainnetReshardingEpoch,
+		MainnetSchedule.BlocksPerEpoch(),
+	)
 )
diff --git a/internal/params/config.go b/internal/params/config.go
index a258f131f7..d2b9e52239 100644
--- a/internal/params/config.go
+++ b/internal/params/config.go
@@ -69,8 +69,8 @@ var (
 		SlotsLimitedEpoch:                     big.NewInt(999),  // Around Fri, 27 May 2022 09:41:02 UTC with 2s block time
 		CrossShardXferPrecompileEpoch:         big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC
 		AllowlistEpoch:                        EpochTBD,
-		LeaderRotationInternalValidatorsEpoch: EpochTBD,
-		LeaderRotationExternalValidatorsEpoch: EpochTBD,
+		LeaderRotationInternalValidatorsEpoch: big.NewInt(2151), // 2024-10-30 19:50 UTC
+		LeaderRotationExternalValidatorsEpoch: big.NewInt(2151), // 2024-10-30 19:50 UTC
 		FeeCollectEpoch:                       big.NewInt(1535), // 2023-07-20 05:51:07+00:00
 		ValidatorCodeFixEpoch:                 big.NewInt(1535), // 2023-07-20 05:51:07+00:00
 		HIP30Epoch:                            big.NewInt(1673), // 2023-11-02 17:30:00+00:00
@@ -79,6 +79,7 @@ var (
 		MaxRateEpoch:                          big.NewInt(1733), // 2023-12-17 12:20:15+00:00
 		DevnetExternalEpoch:                   EpochTBD,
 		TestnetExternalEpoch:                  EpochTBD,
+		HIP32Epoch:                            big.NewInt(2151), // 2024-10-30 19:50 UTC
 	}
 
 	// TestnetChainConfig contains the chain parameters to run a node on the harmony test network.

From 2e824e1266c3933e348ff480b33f061835c0f4aa Mon Sep 17 00:00:00 2001
From: Soph <35721420+sophoah@users.noreply.github.com>
Date: Tue, 8 Oct 2024 16:53:33 +0700
Subject: [PATCH 24/24] HIP32 set to epoch 2152 (#4770)

---
 internal/params/config.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/params/config.go b/internal/params/config.go
index d2b9e52239..77a4c37f34 100644
--- a/internal/params/config.go
+++ b/internal/params/config.go
@@ -69,8 +69,8 @@ var (
 		SlotsLimitedEpoch:                     big.NewInt(999),  // Around Fri, 27 May 2022 09:41:02 UTC with 2s block time
 		CrossShardXferPrecompileEpoch:         big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC
 		AllowlistEpoch:                        EpochTBD,
-		LeaderRotationInternalValidatorsEpoch: big.NewInt(2151), // 2024-10-30 19:50 UTC
-		LeaderRotationExternalValidatorsEpoch: big.NewInt(2151), // 2024-10-30 19:50 UTC
+		LeaderRotationInternalValidatorsEpoch: big.NewInt(2152), // 2024-10-31 13:02 UTC
+		LeaderRotationExternalValidatorsEpoch: big.NewInt(2152), // 2024-10-31 13:02 UTC
 		FeeCollectEpoch:                       big.NewInt(1535), // 2023-07-20 05:51:07+00:00
 		ValidatorCodeFixEpoch:                 big.NewInt(1535), // 2023-07-20 05:51:07+00:00
 		HIP30Epoch:                            big.NewInt(1673), // 2023-11-02 17:30:00+00:00
@@ -79,7 +79,7 @@ var (
 		MaxRateEpoch:                          big.NewInt(1733), // 2023-12-17 12:20:15+00:00
 		DevnetExternalEpoch:                   EpochTBD,
 		TestnetExternalEpoch:                  EpochTBD,
-		HIP32Epoch:                            big.NewInt(2151), // 2024-10-30 19:50 UTC
+		HIP32Epoch:                            big.NewInt(2152), // 2024-10-31 13:02 UTC
 	}
 
 	// TestnetChainConfig contains the chain parameters to run a node on the harmony test network.