From 6a7d933a68c941c3ef2fea2e88dc4d38370145dc Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 2 Feb 2022 23:05:05 -0400 Subject: [PATCH 01/50] chore: adding read/write mutexes to peerstate --- dot/peerset/peerset.go | 34 +------ dot/peerset/peerstate.go | 203 +++++++++++++++++++++++++------------- dot/peerset/reputation.go | 26 +++++ 3 files changed, 165 insertions(+), 98 deletions(-) create mode 100644 dot/peerset/reputation.go diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index accdcd4a7e..2bad5db818 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -6,7 +6,6 @@ package peerset import ( "context" "fmt" - "math" "strings" "time" @@ -122,33 +121,6 @@ type Message struct { PeerID peer.ID } -// Reputation represents reputation value of the node -type Reputation int32 - -// add handles overflow and underflow condition while adding two Reputation values. -func (r Reputation) add(num Reputation) Reputation { - if num > 0 { - if r > math.MaxInt32-num { - return math.MaxInt32 - } - } else if r < math.MinInt32-num { - return math.MinInt32 - } - return r + num -} - -// sub handles underflow condition while subtracting two Reputation values. -func (r Reputation) sub(num Reputation) Reputation { - if num < 0 { - if r > math.MaxInt32+num { - return math.MaxInt32 - } - } else if r < math.MinInt32+num { - return math.MinInt32 - } - return r - num -} - // ReputationChange is description of a reputation adjustment for a node type ReputationChange struct { // PeerReputation value @@ -294,7 +266,11 @@ func (ps *PeerSet) updateTime() error { continue } - lastDiscoveredTime := ps.peerState.lastConnectedAndDiscovered(set, peerID) + lastDiscoveredTime, err := ps.peerState.lastConnectedAndDiscovered(set, peerID) + if err != nil { + return err + } + if lastDiscoveredTime.Add(forgetAfterTime).Second() >= currTime.Second() { continue } diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index 90b5f59040..e062183fd0 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -110,10 +110,13 @@ type PeersState struct { // since, single Info can also manage the flow. sets []Info - mu sync.Mutex + peerStateRWMutex sync.RWMutex } func (ps *PeersState) getNode(p peer.ID) (*node, error) { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + if n, ok := ps.nodes[p]; ok { return n, nil } @@ -126,6 +129,7 @@ func NewPeerState(cfgs []*config) (*PeersState, error) { if len(cfgs) == 0 { return nil, ErrConfigSetIsEmpty } + infoSet := make([]Info, 0, len(cfgs)) for _, cfg := range cfgs { info := Info{ @@ -154,12 +158,15 @@ func (ps *PeersState) getSetLength() int { // peerStatus returns the status of peer based on its connection state // i.e. connectedPeer, notConnectedPeer or unknownPeer. func (ps *PeersState) peerStatus(set int, peerID peer.ID) string { - n, err := ps.getNode(peerID) - if err != nil { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + + node, has := ps.nodes[peerID] + if !has { return unknownPeer } - switch n.state[set] { + switch node.state[set] { case ingoing, outgoing: return connectedPeer case notConnected: @@ -171,6 +178,9 @@ func (ps *PeersState) peerStatus(set int, peerID peer.ID) string { // peers return the list of all the peers we know of. func (ps *PeersState) peers() []peer.ID { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + peerIDs := make([]peer.ID, 0, len(ps.nodes)) for k := range ps.nodes { peerIDs = append(peerIDs, k) @@ -178,86 +188,108 @@ func (ps *PeersState) peers() []peer.ID { return peerIDs } +type connectedPeerReputation struct { + peerID peer.ID + reputation Reputation +} + // sortedPeers returns the list of peers we are connected to of a specific set. func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { - if len(ps.sets) < idx { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + + if len(ps.sets) == 0 || len(ps.sets) < idx { logger.Debug("peer state doesn't have info for the provided index") return nil } - type kv struct { - peerID peer.ID - Node *node - } + connectedPeersReps := make([]connectedPeerReputation, 0) + + for peerID, node := range ps.nodes { + state := node.state[idx] - var ss []kv - for k, v := range ps.nodes { - state := v.state[idx] if isPeerConnected(state) { - ss = append(ss, kv{k, v}) + connectedPeersReps = append(connectedPeersReps, connectedPeerReputation{ + peerID: peerID, + reputation: node.rep, + }) } } - sort.Slice(ss, func(i, j int) bool { - return ss[i].Node.rep > ss[j].Node.rep + sort.Slice(connectedPeersReps, func(i, j int) bool { + return connectedPeersReps[i].reputation > connectedPeersReps[j].reputation }) - peerIDs := make(peer.IDSlice, len(ss)) - for i, kv := range ss { + peerIDs := make(peer.IDSlice, len(connectedPeersReps)) + for i, kv := range connectedPeersReps { peerIDs[i] = kv.peerID } return peerIDs } -func (ps *PeersState) addReputation(pid peer.ID, change ReputationChange) ( +func (ps *PeersState) addReputation(peerID peer.ID, change ReputationChange) ( newReputation Reputation, err error) { - ps.mu.Lock() - defer ps.mu.Unlock() - n, err := ps.getNode(pid) - if err != nil { - return 0, err + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + + node, has := ps.nodes[peerID] + if !has { + return 0, ErrPeerDoesNotExist } - newReputation = n.addReputation(change.Value) - ps.nodes[pid] = n + newReputation = node.addReputation(change.Value) + ps.nodes[peerID] = node return newReputation, nil } // highestNotConnectedPeer returns the peer with the highest Reputation and that we are not connected to. func (ps *PeersState) highestNotConnectedPeer(set int) peer.ID { - var maxRep = math.MinInt32 - var peerID peer.ID - for id, n := range ps.nodes { - if n.state[set] != notConnected { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + + maxRep := math.MinInt32 + var higestPeerID peer.ID + + for peerID, node := range ps.nodes { + if node.state[set] != notConnected { continue } - val := int(n.rep) + val := int(node.rep) if val >= maxRep { maxRep = val - peerID = id + higestPeerID = peerID } } - return peerID + return higestPeerID } func (ps *PeersState) hasFreeOutgoingSlot(set int) bool { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + return ps.sets[set].numOut < ps.sets[set].maxOut } // Note: that it is possible for numIn to be strictly superior to the max, in case we were // connected to reserved node then marked them as not reserved. func (ps *PeersState) hasFreeIncomingSlot(set int) bool { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + return ps.sets[set].numIn >= ps.sets[set].maxIn } // addNoSlotNode adds a node to the list of nodes that don't occupy slots. // has no effect if the node was already in the group. func (ps *PeersState) addNoSlotNode(idx int, peerID peer.ID) error { + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + if _, ok := ps.sets[idx].noSlotNodes[peerID]; ok { logger.Debugf("peer %s already exists in no slot node", peerID) return nil @@ -265,54 +297,63 @@ func (ps *PeersState) addNoSlotNode(idx int, peerID peer.ID) error { // Insert peerStatus ps.sets[idx].noSlotNodes[peerID] = struct{}{} - n, err := ps.getNode(peerID) - if err != nil { - return fmt.Errorf("could not get node for peer id %s: %w", peerID, err) + + node, has := ps.nodes[peerID] + if !has { + return fmt.Errorf("could not get node for peer id %s: %w", peerID, ErrPeerDoesNotExist) } - switch n.state[idx] { + switch node.state[idx] { case ingoing: ps.sets[idx].numIn-- case outgoing: ps.sets[idx].numOut-- } - ps.nodes[peerID] = n return nil } func (ps *PeersState) removeNoSlotNode(idx int, peerID peer.ID) error { + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + if _, ok := ps.sets[idx].noSlotNodes[peerID]; !ok { logger.Debugf("peer %s is not in no-slot node map", peerID) return nil } delete(ps.sets[idx].noSlotNodes, peerID) - n, err := ps.getNode(peerID) - if err != nil { - return fmt.Errorf("could not get node for peer id %s: %w", peerID, err) + + node, has := ps.nodes[peerID] + if !has { + return fmt.Errorf("could not get node for peer id %s: %w", peerID, ErrPeerDoesNotExist) } - switch n.state[idx] { + switch node.state[idx] { case ingoing: ps.sets[idx].numIn++ case outgoing: ps.sets[idx].numOut++ } + return nil } // disconnect updates the node status to the notConnected state. // It should be called only when the node is in connected state. func (ps *PeersState) disconnect(idx int, peerID peer.ID) error { + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + info := ps.sets[idx] - n, err := ps.getNode(peerID) - if err != nil { - return err + node, has := ps.nodes[peerID] + if !has { + return ErrPeerDoesNotExist } - if _, ok := info.noSlotNodes[peerID]; !ok { - switch n.state[idx] { + _, has = info.noSlotNodes[peerID] + if !has { + switch node.state[idx] { case ingoing: info.numIn-- case outgoing: @@ -323,48 +364,66 @@ func (ps *PeersState) disconnect(idx int, peerID peer.ID) error { } // set node state to notConnected. - n.state[idx] = notConnected - n.lastConnected[idx] = time.Now() + node.state[idx] = notConnected + node.lastConnected[idx] = time.Now() ps.sets[idx] = info + return nil } // discover takes input for set id and create a node and insert in the list. // the initial Reputation of the peer will be 0 and ingoing notMember state. func (ps *PeersState) discover(set int, peerID peer.ID) { + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + numSet := len(ps.sets) - if _, err := ps.getNode(peerID); err != nil { + + _, has := ps.nodes[peerID] + if !has { n := newNode(numSet) n.state[set] = notConnected ps.nodes[peerID] = n } } -func (ps *PeersState) lastConnectedAndDiscovered(set int, peerID peer.ID) time.Time { - node, err := ps.getNode(peerID) - if err != nil && node.state[set] == notConnected { - return node.lastConnected[set] +func (ps *PeersState) lastConnectedAndDiscovered(set int, peerID peer.ID) (time.Time, error) { + ps.peerStateRWMutex.RLock() + defer ps.peerStateRWMutex.RUnlock() + + node, has := ps.nodes[peerID] + if !has { + return time.Time{}, ErrPeerDoesNotExist } - return time.Now() + + if node.state[set] == notConnected { + return node.lastConnected[set], nil + } + + return time.Now(), nil } // forgetPeer removes the peer with reputation 0 from the peerSet. func (ps *PeersState) forgetPeer(set int, peerID peer.ID) error { - n, err := ps.getNode(peerID) - if err != nil { - return err + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + + node, has := ps.nodes[peerID] + if !has { + return ErrPeerDoesNotExist } - if n.state[set] != notMember { - n.state[set] = notMember + if node.state[set] != notMember { + node.state[set] = notMember } - if n.getReputation() != 0 { + if node.getReputation() != 0 { return nil } + // remove the peer from peerSet nodes entirely if it isn't a member of any set. remove := true - for _, state := range n.state { + for _, state := range node.state { if state != notMember { remove = false break @@ -383,18 +442,21 @@ func (ps *PeersState) forgetPeer(set int, peerID peer.ID) error { // If the slots are full, the node stays "not connected" and we return the error ErrOutgoingSlotsUnavailable. // non slot occupying nodes don't count towards the number of slots. func (ps *PeersState) tryOutgoing(setID int, peerID peer.ID) error { + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + _, isNoSlotNode := ps.sets[setID].noSlotNodes[peerID] if !ps.hasFreeOutgoingSlot(setID) && !isNoSlotNode { return ErrOutgoingSlotsUnavailable } - n, err := ps.getNode(peerID) - if err != nil { - return err + node, has := ps.nodes[peerID] + if !has { + return ErrPeerDoesNotExist } - n.state[setID] = outgoing + node.state[setID] = outgoing if !isNoSlotNode { ps.sets[setID].numOut++ } @@ -407,6 +469,9 @@ func (ps *PeersState) tryOutgoing(setID int, peerID peer.ID) error { // If the slots are full, the node stays "not connected" and we return Err. // non slot occupying nodes don't count towards the number of slots. func (ps *PeersState) tryAcceptIncoming(setID int, peerID peer.ID) error { + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + var isNoSlotOccupied bool if _, ok := ps.sets[setID].noSlotNodes[peerID]; ok { isNoSlotOccupied = true @@ -417,13 +482,13 @@ func (ps *PeersState) tryAcceptIncoming(setID int, peerID peer.ID) error { return ErrIncomingSlotsUnavailable } - n, err := ps.getNode(peerID) - if err != nil { + node, has := ps.nodes[peerID] + if !has { // state inconsistency tryOutgoing on an unknown node - return err + return ErrPeerDoesNotExist } - n.state[setID] = ingoing + node.state[setID] = ingoing if !isNoSlotOccupied { // this need to be added as incoming connection allocate slot. ps.sets[setID].numIn++ diff --git a/dot/peerset/reputation.go b/dot/peerset/reputation.go new file mode 100644 index 0000000000..56851e8858 --- /dev/null +++ b/dot/peerset/reputation.go @@ -0,0 +1,26 @@ +package peerset + +import "math" + +// Reputation represents reputation value of the node +type Reputation int32 + +// add handles overflow and underflow condition while adding two Reputation values. +func (r Reputation) add(num Reputation) Reputation { + if num > 0 && r > math.MaxInt32-num { + return math.MaxInt32 + } else if r < math.MinInt32-num { + return math.MinInt32 + } + return r + num +} + +// sub handles underflow condition while subtracting two Reputation values. +func (r Reputation) sub(num Reputation) Reputation { + if num < 0 && r > math.MaxInt32+num { + return math.MaxInt32 + } else if r < math.MinInt32+num { + return math.MinInt32 + } + return r - num +} From b0eb77fc19ae96c7885f3c1b9c7bd3caebd2a17a Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 3 Feb 2022 17:12:35 -0400 Subject: [PATCH 02/50] chore: remove channels and use the executor as parameter --- dot/network/service.go | 21 +--- dot/network/state.go | 4 +- dot/peerset/handler.go | 98 +++++++---------- dot/peerset/peerset.go | 161 ++++++++++++--------------- dot/peerset/peerset_test.go | 199 +++++++++++++++++++++++----------- dot/peerset/peerstate.go | 34 +++--- dot/peerset/peerstate_test.go | 15 +-- dot/peerset/reputation.go | 12 +- dot/peerset/test_helpers.go | 5 +- 9 files changed, 282 insertions(+), 267 deletions(-) diff --git a/dot/network/service.go b/dot/network/service.go index 363d8ba6ef..4f9b4b51a2 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -174,6 +174,7 @@ func NewService(cfg *Config) (*Service, error) { if cfg.batchSize == 0 { cfg.batchSize = defaultTxnBatchSize } + // create a new host instance host, err := newHost(ctx, cfg) if err != nil { @@ -695,13 +696,11 @@ func (s *Service) ReportPeer(change peerset.ReputationChange, p peer.ID) { } func (s *Service) startPeerSetHandler() { - s.host.cm.peerSetHandler.Start(s.ctx) + s.host.cm.peerSetHandler.Start(s.ctx, s.processMessage) // wait for peerSetHandler to start. if !s.noBootstrap { s.host.bootstrap() } - - go s.startProcessingMsg() } func (s *Service) processMessage(msg peerset.Message) { @@ -737,19 +736,3 @@ func (s *Service) processMessage(msg peerset.Message) { logger.Debugf("connection dropped successfully for peer %s", peerID) } } - -func (s *Service) startProcessingMsg() { - msgCh := s.host.cm.peerSetHandler.Messages() - for { - select { - case <-s.ctx.Done(): - return - case msg, ok := <-msgCh: - if !ok { - return - } - - s.processMessage(msg) - } - } -} diff --git a/dot/network/state.go b/dot/network/state.go index 7d4499f26d..907ce32c19 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -47,7 +47,7 @@ type TransactionHandler interface { // PeerSetHandler is the interface used by the connection manager to handle peerset. type PeerSetHandler interface { - Start(context.Context) + Start(context.Context, func(peerset.Message)) Stop() ReportPeer(peerset.ReputationChange, ...peer.ID) PeerAdd @@ -73,6 +73,4 @@ type PeerRemove interface { // Peer is the interface used by the PeerSetHandler to get the peer data from peerSet. type Peer interface { PeerReputation(peer.ID) (peerset.Reputation, error) - SortedPeers(idx int) chan peer.IDSlice - Messages() chan peerset.Message } diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index ab69592daa..5866107e5b 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -30,80 +30,74 @@ func NewPeerSetHandler(cfg *ConfigSet) (*Handler, error) { }, nil } +// AddReservedPeer adds reserved peer into peerSet. +func (h *Handler) SetReservedOnlyPeer(setID int, peers ...peer.ID) { + // TODO: not yet implemented (#1888) + logger.Errorf("failed to do action %s on peerSet: not implemented yet", setReservedOnly) +} + // AddReservedPeer adds reserved peer into peerSet. func (h *Handler) AddReservedPeer(setID int, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: addReservedPeer, - setID: setID, - peers: peers, + err := h.peerSet.addReservedPeers(setID, peers...) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", addReservedPeer, err) } } // RemoveReservedPeer remove reserved peer from peerSet. func (h *Handler) RemoveReservedPeer(setID int, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: removeReservedPeer, - setID: setID, - peers: peers, + err := h.peerSet.removeReservedPeers(setID, peers...) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", removeReservedPeer, err) } } // SetReservedPeer set the reserve peer into peerSet func (h *Handler) SetReservedPeer(setID int, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: setReservedPeers, - setID: setID, - peers: peers, + // TODO: this is not used yet, might required to implement RPC Call for this. + err := h.peerSet.setReservedPeer(setID, peers...) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", setReservedPeers, err) } } // AddPeer adds peer to peerSet. func (h *Handler) AddPeer(setID int, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: addToPeerSet, - setID: setID, - peers: peers, + err := h.peerSet.addPeer(setID, peers) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", addToPeerSet, err) } } // RemovePeer removes peer from peerSet. func (h *Handler) RemovePeer(setID int, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: removeFromPeerSet, - setID: setID, - peers: peers, + err := h.peerSet.removePeer(setID, peers...) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", removeFromPeerSet, err) } } // ReportPeer reports ReputationChange according to the peer behaviour. func (h *Handler) ReportPeer(rep ReputationChange, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: reportPeer, - reputation: rep, - peers: peers, + err := h.peerSet.reportPeer(rep, peers...) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", reportPeer, err) } } // Incoming calls when we have an incoming connection from peer. func (h *Handler) Incoming(setID int, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: incoming, - peers: peers, - setID: setID, + err := h.peerSet.incoming(setID, peers...) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", incoming, err) } } -// Messages return result message chan. -func (h *Handler) Messages() chan Message { - return h.peerSet.resultMsgCh -} - // DisconnectPeer calls for disconnecting a connection from peer. func (h *Handler) DisconnectPeer(setID int, peers ...peer.ID) { - h.actionQueue <- action{ - actionCall: disconnect, - setID: setID, - peers: peers, + err := h.peerSet.disconnect(setID, UnknownDrop, peers...) + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", disconnect, err) } } @@ -113,40 +107,22 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { if err != nil { return 0, err } - return n.getReputation(), nil + return n.rep, nil } // Start starts peerSet processing -func (h *Handler) Start(ctx context.Context) { +func (h *Handler) Start(ctx context.Context, processMessageFn func(Message)) { ctx, cancel := context.WithCancel(ctx) h.cancelCtx = cancel - - actionCh := make(chan action, msgChanSize) - h.closeCh = make(chan struct{}) - h.actionQueue = actionCh - - h.peerSet.start(ctx, actionCh) + h.peerSet.start(ctx, processMessageFn) } // SortedPeers return chan for sorted connected peer in the peerSet. -func (h *Handler) SortedPeers(setIdx int) chan peer.IDSlice { - resultPeersCh := make(chan peer.IDSlice) - h.actionQueue <- action{ - actionCall: sortedPeers, - resultPeersCh: resultPeersCh, - setID: setIdx, - } - - return resultPeersCh +func (h *Handler) SortedPeers(setIdx int) peer.IDSlice { + return h.peerSet.peerState.sortedPeers(setIdx) } // Stop closes the actionQueue and result message chan. func (h *Handler) Stop() { - select { - case <-h.closeCh: - default: - h.cancelCtx() - close(h.closeCh) - close(h.actionQueue) - } + h.cancelCtx() } diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 2bad5db818..c5dc31ab85 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/ChainSafe/gossamer/internal/log" @@ -94,7 +95,7 @@ func (a action) String() string { for i := range a.peers { peersStrings[i] = a.peers[i].String() } - return fmt.Sprintf("{call=%s, set-id=%d, reputation change %v, peers=[%s]", + return fmt.Sprintf("call=%s, set-id=%d, reputation change %v, peers=[%s]", a.actionCall.String(), a.setID, a.reputation, strings.Join(peersStrings, ", ")) } @@ -135,13 +136,13 @@ func newReputationChange(value Reputation, reason string) ReputationChange { // PeerSet is a container for all the components of a peerSet. type PeerSet struct { - peerState *PeersState + peerSetMutex sync.Mutex + peerState *PeersState reservedNode map[peer.ID]struct{} // TODO: this will be useful for reserved only mode // this is for future purpose if reserved-only flag is enabled (#1888). isReservedOnly bool - resultMsgCh chan Message // time when the PeerSet was created. created time.Time // last time when we updated the reputations of connected nodes. @@ -149,8 +150,8 @@ type PeerSet struct { // next time to do a periodic call to allocSlots with all Set. This is done once two // second, to match the period of the Reputation updates. nextPeriodicAllocSlots time.Duration - // chan for receiving action request. - actionQueue <-chan action + + processMessageFn func(Message) } // config is configuration of a single set. @@ -233,6 +234,9 @@ func reputationTick(reput Reputation) Reputation { // updateTime updates the value of latestTimeUpdate and performs all the updates that // happen over time, such as Reputation increases for staying connected. func (ps *PeerSet) updateTime() error { + ps.peerSetMutex.Lock() + defer ps.peerSetMutex.Unlock() + currTime := time.Now() // identify the time difference between current time and last update time for peer reputation in seconds. // update the latestTimeUpdate to currTime. @@ -245,16 +249,11 @@ func (ps *PeerSet) updateTime() error { // For each elapsed second, move the node reputation towards zero. for i := int64(0); i < secDiff; i++ { for _, peerID := range ps.peerState.peers() { - n, err := ps.peerState.getNode(peerID) + after, err := ps.peerState.updateReputationByTick(peerID) if err != nil { return err } - before := n.getReputation() - after := reputationTick(before) - n.setReputation(after) - ps.peerState.nodes[peerID] = n - if after != 0 { continue } @@ -308,21 +307,24 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { setLen := ps.peerState.getSetLength() for i := 0; i < setLen; i++ { - if ps.peerState.peerStatus(i, pid) == connectedPeer { - // disconnect peer - err = ps.peerState.disconnect(i, pid) - if err != nil { - return err - } + if ps.peerState.peerStatus(i, pid) != connectedPeer { + continue + } - ps.resultMsgCh <- Message{ - Status: Drop, - setID: uint64(i), - PeerID: pid, - } - if err = ps.allocSlots(i); err != nil { - return err - } + // disconnect peer + err = ps.peerState.disconnect(i, pid) + if err != nil { + return err + } + + ps.processMessageFn(Message{ + Status: Drop, + setID: uint64(i), + PeerID: pid, + }) + + if err = ps.allocSlots(i); err != nil { + return err } } } @@ -346,15 +348,15 @@ func (ps *PeerSet) allocSlots(setIdx int) error { peerState.discover(setIdx, reservePeer) } - var n *node - n, err = ps.peerState.getNode(reservePeer) + var node *node + node, err = ps.peerState.getNode(reservePeer) if err != nil { return err } - if n.getReputation() < BannedThresholdValue { + if node.rep < BannedThresholdValue { logger.Warnf("reputation is lower than banned threshold value, reputation: %d, banned threshold value: %d", - n.getReputation(), BannedThresholdValue) + node.rep, BannedThresholdValue) break } @@ -362,11 +364,12 @@ func (ps *PeerSet) allocSlots(setIdx int) error { return err } - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Connect, setID: uint64(setIdx), PeerID: reservePeer, - } + }) + } // nothing more to do if we're in reserved mode. @@ -381,7 +384,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { } n := peerState.nodes[peerID] - if n.getReputation() < BannedThresholdValue { + if n.rep < BannedThresholdValue { logger.Critical("highest rated peer is below bannedThresholdValue") break } @@ -391,11 +394,11 @@ func (ps *PeerSet) allocSlots(setIdx int) error { break } - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Connect, setID: uint64(setIdx), PeerID: peerID, - } + }) logger.Debugf("Sent connect message to peer %s", peerID) } @@ -448,11 +451,11 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { return err } - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Drop, setID: uint64(setID), PeerID: peerID, - } + }) } } @@ -506,11 +509,11 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { } if status := ps.peerState.peerStatus(setID, pid); status == connectedPeer { - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Drop, setID: uint64(setID), PeerID: pid, - } + }) // disconnect and forget err := ps.peerState.disconnect(setID, pid) @@ -542,11 +545,11 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { for _, pid := range peers { if ps.isReservedOnly { if _, ok := ps.reservedNode[pid]; !ok { - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Reject, setID: uint64(setID), PeerID: pid, - } + }) continue } } @@ -562,27 +565,39 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { } state := ps.peerState - p := state.nodes[pid] + + var nodeReputation Reputation + + state.peerStateRWMutex.RLock() + node, has := state.nodes[pid] + + if has { + nodeReputation = node.rep + } + + state.peerStateRWMutex.RUnlock() + switch { - case p.getReputation() < BannedThresholdValue: - ps.resultMsgCh <- Message{ + case nodeReputation < BannedThresholdValue: + ps.processMessageFn(Message{ Status: Reject, setID: uint64(setID), PeerID: pid, - } + }) + case state.tryAcceptIncoming(setID, pid) != nil: - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Reject, setID: uint64(setID), PeerID: pid, - } + }) default: logger.Debugf("incoming connection accepted from peer %s", pid) - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Accept, setID: uint64(setID), PeerID: pid, - } + }) } } @@ -623,11 +638,11 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e if err = state.disconnect(setIdx, pid); err != nil { return err } - ps.resultMsgCh <- Message{ + ps.processMessageFn(Message{ Status: Drop, setID: uint64(setIdx), PeerID: pid, - } + }) // TODO: figure out the condition of connection refuse. if reason == RefusedDrop { @@ -641,20 +656,14 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e } // start handles all the action for the peerSet. -func (ps *PeerSet) start(ctx context.Context, actionQueue chan action) { - ps.actionQueue = actionQueue - - ps.resultMsgCh = make(chan Message, msgChanSize) +func (ps *PeerSet) start(ctx context.Context, processMessageFn func(Message)) { + ps.processMessageFn = processMessageFn go ps.doWork(ctx) } func (ps *PeerSet) doWork(ctx context.Context) { ticker := time.NewTicker(ps.nextPeriodicAllocSlots) - - defer func() { - ticker.Stop() - close(ps.resultMsgCh) - }() + defer ticker.Stop() for { select { @@ -668,40 +677,6 @@ func (ps *PeerSet) doWork(ctx context.Context) { logger.Warnf("failed to do action on peerSet: %s", err) } } - case act, ok := <-ps.actionQueue: - if !ok { - return - } - - var err error - switch act.actionCall { - case addReservedPeer: - err = ps.addReservedPeers(act.setID, act.peers...) - case removeReservedPeer: - err = ps.removeReservedPeers(act.setID, act.peers...) - case setReservedPeers: - // TODO: this is not used yet, might required to implement RPC Call for this. - err = ps.setReservedPeer(act.setID, act.peers...) - case setReservedOnly: - // TODO: not yet implemented (#1888) - err = fmt.Errorf("not implemented yet") - case reportPeer: - err = ps.reportPeer(act.reputation, act.peers...) - case addToPeerSet: - err = ps.addPeer(act.setID, act.peers) - case removeFromPeerSet: - err = ps.removePeer(act.setID, act.peers...) - case incoming: - err = ps.incoming(act.setID, act.peers...) - case sortedPeers: - act.resultPeersCh <- ps.peerState.sortedPeers(act.setID) - case disconnect: - err = ps.disconnect(act.setID, UnknownDrop, act.peers...) - } - - if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", act, err) - } } } } diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index 3c094734c8..a7e958b0d7 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -4,6 +4,7 @@ package peerset import ( + "sync" "testing" "time" @@ -11,10 +12,70 @@ import ( "github.com/stretchr/testify/require" ) +type assertMessageParam struct { + expect interface{} + got interface{} +} + +type mockProcessMessage struct { + callCounter int + executionCounter int + expects map[int]assertMessageParam + mockMutex sync.Mutex +} + +func newMockProcessMessage() *mockProcessMessage { + return &mockProcessMessage{ + expects: make(map[int]assertMessageParam), + } +} + +func (mock *mockProcessMessage) ExpectByCall(expected interface{}) { + mock.mockMutex.Lock() + defer mock.mockMutex.Unlock() + + mock.callCounter++ + mock.expects[mock.callCounter] = assertMessageParam{ + expect: expected, + } +} + +func (mock *mockProcessMessage) ProcessMessage() func(Message) { + return func(m Message) { + mock.mockMutex.Lock() + defer mock.mockMutex.Unlock() + + mock.executionCounter++ + assert, has := mock.expects[mock.executionCounter] + if has { + assert.got = m + mock.expects[mock.executionCounter] = assert + } + } +} + +func (mock *mockProcessMessage) Assert(t *testing.T) { + mock.mockMutex.Lock() + defer mock.mockMutex.Unlock() + + // the number of calls to processMessage should be the same we expect + require.Equal(t, mock.callCounter, mock.executionCounter) + + // follow the order we defined + for i := 1; i <= mock.callCounter; i++ { + assert, has := mock.expects[i] + require.True(t, has) + require.Equal(t, assert.expect, assert.got) + } +} + func TestPeerSetBanned(t *testing.T) { t.Parallel() - handler := newTestPeerSet(t, 25, 25, nil, nil, false) + mock := newMockProcessMessage() + processMessageFn := mock.ProcessMessage() + + handler := newTestPeerSet(t, 25, 25, nil, nil, false, processMessageFn) ps := handler.peerSet require.Equal(t, unknownPeer, ps.peerState.peerStatus(0, peer1)) @@ -25,29 +86,36 @@ func TestPeerSetBanned(t *testing.T) { // we ban a node by setting its reputation under the threshold. rpc := newReputationChange(BannedThresholdValue-1, "") + + mock.ExpectByCall(Message{Status: Drop, setID: 0x0, PeerID: "testPeer1"}) // we need one for the message to be processed. handler.ReportPeer(rpc, peer1) time.Sleep(time.Millisecond * 100) - checkMessageStatus(t, <-ps.resultMsgCh, Drop) - + mock.ExpectByCall(Message{Status: Reject, setID: 0x0, PeerID: "testPeer1"}) // check that an incoming connection from that node gets refused. handler.Incoming(0, peer1) - checkMessageStatus(t, <-ps.resultMsgCh, Reject) // wait a bit for the node's reputation to go above the threshold. time.Sleep(time.Millisecond * 1200) + mock.ExpectByCall(Message{Status: Accept, setID: 0x0, PeerID: "testPeer1"}) // try again. This time the node should be accepted. handler.Incoming(0, peer1) - require.NoError(t, err) - checkMessageStatus(t, <-ps.resultMsgCh, Accept) + + mock.Assert(t) } func TestAddReservedPeers(t *testing.T) { t.Parallel() - handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false) + mock := newMockProcessMessage() + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: bootNode}) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) + processMessageFn := mock.ProcessMessage() + + handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false, processMessageFn) ps := handler.peerSet handler.AddReservedPeer(0, reservedPeer) @@ -55,66 +123,61 @@ func TestAddReservedPeers(t *testing.T) { time.Sleep(time.Millisecond * 200) - expectedMsgs := []Message{ - {Status: Connect, setID: 0, PeerID: bootNode}, - {Status: Connect, setID: 0, PeerID: reservedPeer}, - {Status: Connect, setID: 0, PeerID: reservedPeer2}, - } - require.Equal(t, uint32(1), ps.peerState.sets[0].numOut) - require.Equal(t, 3, len(ps.resultMsgCh)) + require.Equal(t, 3, mock.executionCounter) - for i := 0; ; i++ { - if len(ps.resultMsgCh) == 0 { - break - } - msg := <-ps.resultMsgCh - require.Equal(t, expectedMsgs[i], msg) - } + mock.Assert(t) } func TestPeerSetIncoming(t *testing.T) { t.Parallel() - handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, []peer.ID{}, false) - ps := handler.peerSet + mock := newMockProcessMessage() + processMessageFn := mock.ProcessMessage() - // connect message will be added ingoing queue for bootnode. - checkMessageStatus(t, <-ps.resultMsgCh, Connect) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: bootNode}) + mock.ExpectByCall(Message{Status: Accept, setID: 0, PeerID: incomingPeer}) + mock.ExpectByCall(Message{Status: Accept, setID: 0, PeerID: incoming2}) + mock.ExpectByCall(Message{Status: Reject, setID: 0, PeerID: incoming3}) - handler.Incoming(0, incomingPeer) - checkMessageStatus(t, <-ps.resultMsgCh, Accept) + handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, + []peer.ID{}, false, processMessageFn) + handler.Incoming(0, incomingPeer) handler.Incoming(0, incoming2) - checkMessageStatus(t, <-ps.resultMsgCh, Accept) - handler.Incoming(0, incoming3) - checkMessageStatus(t, <-ps.resultMsgCh, Reject) + + mock.Assert(t) } func TestPeerSetDiscovered(t *testing.T) { t.Parallel() - handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false) + mock := newMockProcessMessage() + processMessageFn := mock.ProcessMessage() - ps := handler.peerSet + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: discovered1}) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: discovered2}) + + handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processMessageFn) handler.AddPeer(0, discovered1) + handler.AddPeer(0, discovered1) - handler.AddPeer(0, discovered2) - time.Sleep(200 * time.Millisecond) + handler.AddPeer(0, discovered2) - require.Equal(t, 3, len(ps.resultMsgCh)) - for len(ps.resultMsgCh) == 0 { - checkMessageStatus(t, <-ps.resultMsgCh, Connect) - } + mock.Assert(t) } func TestReAllocAfterBanned(t *testing.T) { t.Parallel() - handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false) + mock := newMockProcessMessage() + processMessageFn := mock.ProcessMessage() + + handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false, processMessageFn) ps := handler.peerSet // adding peer1 with incoming slot. @@ -126,60 +189,72 @@ func TestReAllocAfterBanned(t *testing.T) { // We ban a node by setting its reputation under the threshold. rep := newReputationChange(BannedThresholdValue-1, "") + + mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: peer1}) // we need one for the message to be processed. handler.ReportPeer(rep, peer1) time.Sleep(time.Millisecond * 100) - checkMessageStatus(t, <-ps.resultMsgCh, Drop) - // Check that an incoming connection from that node gets refused. - + mock.ExpectByCall(Message{Status: Reject, setID: 0, PeerID: peer1}) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: peer1}) handler.Incoming(0, peer1) - checkMessageStatus(t, <-ps.resultMsgCh, Reject) + time.Sleep(time.Second * 2) - time.Sleep(time.Millisecond * 100) - checkMessageStatus(t, <-ps.resultMsgCh, Connect) + mock.Assert(t) } func TestRemovePeer(t *testing.T) { t.Parallel() - handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, nil, false) + mock := newMockProcessMessage() + processMessageFn := mock.ProcessMessage() + + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: "testDiscovered1"}) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: "testDiscovered2"}) + handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, + nil, false, processMessageFn) + ps := handler.peerSet + require.Equal(t, 2, mock.executionCounter) - require.Equal(t, 2, len(ps.resultMsgCh)) - for len(ps.resultMsgCh) != 0 { - checkMessageStatus(t, <-ps.resultMsgCh, Connect) - } + time.Sleep(time.Millisecond * 500) + mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) + mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) handler.RemovePeer(0, discovered1, discovered2) - time.Sleep(200 * time.Millisecond) - - require.Equal(t, 2, len(ps.resultMsgCh)) - for len(ps.resultMsgCh) != 0 { - checkMessageStatus(t, <-ps.resultMsgCh, Drop) - } require.Equal(t, 0, len(ps.peerState.nodes)) + + mock.Assert(t) } func TestSetReservePeer(t *testing.T) { t.Parallel() - handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, true) + mock := newMockProcessMessage() + processMessageFn := mock.ProcessMessage() + + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) + handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, + true, processMessageFn) + ps := handler.peerSet - require.Equal(t, 2, len(ps.resultMsgCh)) - for len(ps.resultMsgCh) != 0 { - checkMessageStatus(t, <-ps.resultMsgCh, Connect) - } + require.Equal(t, 2, mock.executionCounter) + + mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: "newRsrPeer"}) + mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: reservedPeer2}) newRsrPeerSet := peer.IDSlice{reservedPeer, peer.ID("newRsrPeer")} + // add newRsrPeer but remove reservedPeer2 handler.SetReservedPeer(0, newRsrPeerSet...) - time.Sleep(200 * time.Millisecond) require.Equal(t, len(newRsrPeerSet), len(ps.reservedNode)) for _, p := range newRsrPeerSet { require.Contains(t, ps.reservedNode, p) } + + mock.Assert(t) } diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index e062183fd0..d3edd23fae 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -88,19 +88,11 @@ func newNode(n int) *node { } } -func (n *node) getReputation() Reputation { - return n.rep -} - func (n *node) addReputation(modifier Reputation) Reputation { n.rep = n.rep.add(modifier) return n.rep } -func (n *node) setReputation(modifier Reputation) { - n.rep = modifier -} - // PeersState struct contains a list of nodes, where each node // has a reputation and is either connected to us or not type PeersState struct { @@ -116,7 +108,6 @@ type PeersState struct { func (ps *PeersState) getNode(p peer.ID) (*node, error) { ps.peerStateRWMutex.RLock() defer ps.peerStateRWMutex.RUnlock() - if n, ok := ps.nodes[p]; ok { return n, nil } @@ -228,6 +219,23 @@ func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { return peerIDs } +func (ps *PeersState) updateReputationByTick(peerID peer.ID) (after Reputation, err error) { + ps.peerStateRWMutex.Lock() + defer ps.peerStateRWMutex.Unlock() + + node, has := ps.nodes[peerID] + if !has { + return 0, ErrPeerDoesNotExist + } + + after = reputationTick(node.rep) + + node.rep = after + ps.nodes[peerID] = node + + return after, nil +} + func (ps *PeersState) addReputation(peerID peer.ID, change ReputationChange) ( newReputation Reputation, err error) { @@ -269,18 +277,12 @@ func (ps *PeersState) highestNotConnectedPeer(set int) peer.ID { } func (ps *PeersState) hasFreeOutgoingSlot(set int) bool { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() - return ps.sets[set].numOut < ps.sets[set].maxOut } // Note: that it is possible for numIn to be strictly superior to the max, in case we were // connected to reserved node then marked them as not reserved. func (ps *PeersState) hasFreeIncomingSlot(set int) bool { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() - return ps.sets[set].numIn >= ps.sets[set].maxIn } @@ -417,7 +419,7 @@ func (ps *PeersState) forgetPeer(set int, peerID peer.ID) error { node.state[set] = notMember } - if node.getReputation() != 0 { + if node.rep != 0 { return nil } diff --git a/dot/peerset/peerstate_test.go b/dot/peerset/peerstate_test.go index 3b58544f38..3263d16566 100644 --- a/dot/peerset/peerstate_test.go +++ b/dot/peerset/peerstate_test.go @@ -150,33 +150,34 @@ func TestHighestNotConnectedPeer(t *testing.T) { state.discover(0, peer1) n, err := state.getNode(peer1) require.NoError(t, err) - n.setReputation(50) + + n.rep = 50 state.nodes[peer1] = n - require.Equal(t, Reputation(50), state.nodes[peer1].getReputation()) + require.Equal(t, Reputation(50), state.nodes[peer1].rep) require.Equal(t, unknownPeer, state.peerStatus(0, peer2)) state.discover(0, peer2) n, err = state.getNode(peer2) require.NoError(t, err) - n.setReputation(25) + n.rep = 25 state.nodes[peer2] = n // peer1 still has the highest reputation require.Equal(t, peer1, state.highestNotConnectedPeer(0)) - require.Equal(t, Reputation(25), state.nodes[peer2].getReputation()) + require.Equal(t, Reputation(25), state.nodes[peer2].rep) require.Equal(t, notConnectedPeer, state.peerStatus(0, peer2)) n, err = state.getNode(peer2) require.NoError(t, err) - n.setReputation(75) + n.rep = 75 state.nodes[peer2] = n require.Equal(t, peer2, state.highestNotConnectedPeer(0)) - require.Equal(t, Reputation(75), state.nodes[peer2].getReputation()) + require.Equal(t, Reputation(75), state.nodes[peer2].rep) require.Equal(t, notConnectedPeer, state.peerStatus(0, peer2)) err = state.tryAcceptIncoming(0, peer2) @@ -191,7 +192,7 @@ func TestHighestNotConnectedPeer(t *testing.T) { require.Equal(t, notConnectedPeer, state.peerStatus(0, peer1)) n, err = state.getNode(peer1) require.NoError(t, err) - n.setReputation(100) + n.rep = 100 state.nodes[peer1] = n require.Equal(t, peer1, state.highestNotConnectedPeer(0)) diff --git a/dot/peerset/reputation.go b/dot/peerset/reputation.go index 56851e8858..2004bf13e9 100644 --- a/dot/peerset/reputation.go +++ b/dot/peerset/reputation.go @@ -7,8 +7,10 @@ type Reputation int32 // add handles overflow and underflow condition while adding two Reputation values. func (r Reputation) add(num Reputation) Reputation { - if num > 0 && r > math.MaxInt32-num { - return math.MaxInt32 + if num > 0 { + if r > math.MaxInt32-num { + return math.MaxInt32 + } } else if r < math.MinInt32-num { return math.MinInt32 } @@ -17,8 +19,10 @@ func (r Reputation) add(num Reputation) Reputation { // sub handles underflow condition while subtracting two Reputation values. func (r Reputation) sub(num Reputation) Reputation { - if num < 0 && r > math.MaxInt32+num { - return math.MaxInt32 + if num < 0 { + if r > math.MaxInt32+num { + return math.MaxInt32 + } } else if r < math.MinInt32+num { return math.MinInt32 } diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index 3c1c88d2a0..4d5fa6b580 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -25,7 +25,8 @@ const ( peer2 = peer.ID("testPeer2") ) -func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, reservedPeers []peer.ID, reservedOnly bool) *Handler { +func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, + reservedPeers []peer.ID, reservedOnly bool, processMessage func(Message)) *Handler { t.Helper() con := &ConfigSet{ Set: []*config{ @@ -41,7 +42,7 @@ func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, reservedPeers []pee handler, err := NewPeerSetHandler(con) require.NoError(t, err) - handler.Start(context.Background()) + handler.Start(context.Background(), processMessage) handler.AddPeer(0, bootNodes...) handler.AddReservedPeer(0, reservedPeers...) From 98cb4bc51053cbbbc60c79ea945fe66a3fb8d8e8 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 3 Feb 2022 17:14:26 -0400 Subject: [PATCH 03/50] chore: remove unneeded stop and ctx --- dot/network/service.go | 2 -- dot/network/state.go | 1 - dot/peerset/handler.go | 12 +----------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/dot/network/service.go b/dot/network/service.go index 4f9b4b51a2..4c331516a4 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -508,8 +508,6 @@ func (s *Service) Stop() error { logger.Errorf("Failed to close host: %s", err) } - s.host.cm.peerSetHandler.Stop() - // check if closeCh is closed, if not, close it. mainloop: for { diff --git a/dot/network/state.go b/dot/network/state.go index 907ce32c19..201d870c91 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -48,7 +48,6 @@ type TransactionHandler interface { // PeerSetHandler is the interface used by the connection manager to handle peerset. type PeerSetHandler interface { Start(context.Context, func(peerset.Message)) - Stop() ReportPeer(peerset.ReputationChange, ...peer.ID) PeerAdd PeerRemove diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 5866107e5b..3b3ae3b3c9 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -11,10 +11,7 @@ import ( // Handler manages peerSet. type Handler struct { - actionQueue chan<- action - peerSet *PeerSet - closeCh chan struct{} - + peerSet *PeerSet cancelCtx context.CancelFunc } @@ -112,8 +109,6 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { // Start starts peerSet processing func (h *Handler) Start(ctx context.Context, processMessageFn func(Message)) { - ctx, cancel := context.WithCancel(ctx) - h.cancelCtx = cancel h.peerSet.start(ctx, processMessageFn) } @@ -121,8 +116,3 @@ func (h *Handler) Start(ctx context.Context, processMessageFn func(Message)) { func (h *Handler) SortedPeers(setIdx int) peer.IDSlice { return h.peerSet.peerState.sortedPeers(setIdx) } - -// Stop closes the actionQueue and result message chan. -func (h *Handler) Stop() { - h.cancelCtx() -} From a6e97fd2d73eb17b2112efec849858eb5c65fdf7 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 3 Feb 2022 17:35:34 -0400 Subject: [PATCH 04/50] chore: remove unused cancel ctx --- dot/peerset/handler.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 3b3ae3b3c9..176528279b 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -11,8 +11,7 @@ import ( // Handler manages peerSet. type Handler struct { - peerSet *PeerSet - cancelCtx context.CancelFunc + peerSet *PeerSet } // NewPeerSetHandler creates a new *peerset.Handler. From d8f67009f114c80d32368b051eda8b5db9af297c Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 4 Feb 2022 10:46:28 -0400 Subject: [PATCH 05/50] chore: keep the same log informations --- dot/peerset/handler.go | 37 +++++++++++++++++++++++++++++-------- dot/peerset/peerset.go | 21 --------------------- dot/peerset/test_helpers.go | 7 ------- 3 files changed, 29 insertions(+), 36 deletions(-) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 176528279b..a5a55c6f46 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -5,10 +5,14 @@ package peerset import ( "context" + "fmt" + "strings" "github.com/libp2p/go-libp2p-core/peer" ) +const logStringPattern = "call=%s, set-id=%d, reputation change %v, peers=[%s]" + // Handler manages peerSet. type Handler struct { peerSet *PeerSet @@ -36,7 +40,8 @@ func (h *Handler) SetReservedOnlyPeer(setID int, peers ...peer.ID) { func (h *Handler) AddReservedPeer(setID int, peers ...peer.ID) { err := h.peerSet.addReservedPeers(setID, peers...) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", addReservedPeer, err) + msg := fmt.Sprintf(logStringPattern, addReservedPeer, setID, nil, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -44,7 +49,8 @@ func (h *Handler) AddReservedPeer(setID int, peers ...peer.ID) { func (h *Handler) RemoveReservedPeer(setID int, peers ...peer.ID) { err := h.peerSet.removeReservedPeers(setID, peers...) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", removeReservedPeer, err) + msg := fmt.Sprintf(logStringPattern, removeReservedPeer, setID, nil, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -53,7 +59,8 @@ func (h *Handler) SetReservedPeer(setID int, peers ...peer.ID) { // TODO: this is not used yet, might required to implement RPC Call for this. err := h.peerSet.setReservedPeer(setID, peers...) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", setReservedPeers, err) + msg := fmt.Sprintf(logStringPattern, setReservedPeers, setID, nil, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -61,7 +68,8 @@ func (h *Handler) SetReservedPeer(setID int, peers ...peer.ID) { func (h *Handler) AddPeer(setID int, peers ...peer.ID) { err := h.peerSet.addPeer(setID, peers) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", addToPeerSet, err) + msg := fmt.Sprintf(logStringPattern, addToPeerSet, setID, nil, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -69,7 +77,8 @@ func (h *Handler) AddPeer(setID int, peers ...peer.ID) { func (h *Handler) RemovePeer(setID int, peers ...peer.ID) { err := h.peerSet.removePeer(setID, peers...) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", removeFromPeerSet, err) + msg := fmt.Sprintf(logStringPattern, removeFromPeerSet, setID, nil, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -77,7 +86,8 @@ func (h *Handler) RemovePeer(setID int, peers ...peer.ID) { func (h *Handler) ReportPeer(rep ReputationChange, peers ...peer.ID) { err := h.peerSet.reportPeer(rep, peers...) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", reportPeer, err) + msg := fmt.Sprintf(logStringPattern, reportPeer, 0, rep, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -85,7 +95,8 @@ func (h *Handler) ReportPeer(rep ReputationChange, peers ...peer.ID) { func (h *Handler) Incoming(setID int, peers ...peer.ID) { err := h.peerSet.incoming(setID, peers...) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", incoming, err) + msg := fmt.Sprintf(logStringPattern, incoming, setID, nil, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -93,7 +104,8 @@ func (h *Handler) Incoming(setID int, peers ...peer.ID) { func (h *Handler) DisconnectPeer(setID int, peers ...peer.ID) { err := h.peerSet.disconnect(setID, UnknownDrop, peers...) if err != nil { - logger.Errorf("failed to do action %s on peerSet: %s", disconnect, err) + msg := fmt.Sprintf(logStringPattern, disconnect, setID, nil, stringfyPeers(peers)) + logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -115,3 +127,12 @@ func (h *Handler) Start(ctx context.Context, processMessageFn func(Message)) { func (h *Handler) SortedPeers(setIdx int) peer.IDSlice { return h.peerSet.peerState.sortedPeers(setIdx) } + +func stringfyPeers(peers peer.IDSlice) string { + peersStrings := make([]string, len(peers)) + for i := range peers { + peersStrings[i] = peers[i].String() + } + + return strings.Join(peersStrings, ", ") +} diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index c5dc31ab85..6df05900c7 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -6,7 +6,6 @@ package peerset import ( "context" "fmt" - "strings" "sync" "time" @@ -24,8 +23,6 @@ const ( // forgetAfterTime amount of time between the moment we disconnect // from a node and the moment we remove it from the list. forgetAfterTime = time.Second * 3600 // one hour - // default channel size for peerSet. - msgChanSize = 100 ) // ActionReceiver represents the enum value for action to be performed on peerSet @@ -81,24 +78,6 @@ func (a ActionReceiver) String() string { } } -// action struct stores the action type and required parameters to perform action -type action struct { - actionCall ActionReceiver - setID int - reputation ReputationChange - peers peer.IDSlice - resultPeersCh chan peer.IDSlice -} - -func (a action) String() string { - peersStrings := make([]string, len(a.peers)) - for i := range a.peers { - peersStrings[i] = a.peers[i].String() - } - return fmt.Sprintf("call=%s, set-id=%d, reputation change %v, peers=[%s]", - a.actionCall.String(), a.setID, a.reputation, strings.Join(peersStrings, ", ")) -} - // Status represents the enum value for Message type Status uint8 diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index 4d5fa6b580..bba9a16e0e 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -63,10 +63,3 @@ func newTestPeerState(t *testing.T, maxIn, maxOut uint32) *PeersState { return state } - -func checkMessageStatus(t *testing.T, m interface{}, expectedStatus Status) { - t.Helper() - msg, ok := m.(Message) - require.True(t, ok) - require.Equal(t, expectedStatus, msg.Status) -} From a93fafe4a8ce17ea0b06f557c5a118773f56f9b2 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 4 Feb 2022 10:47:31 -0400 Subject: [PATCH 06/50] chore: resolve lint --- dot/peerset/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index a5a55c6f46..355801c5d3 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -30,7 +30,7 @@ func NewPeerSetHandler(cfg *ConfigSet) (*Handler, error) { }, nil } -// AddReservedPeer adds reserved peer into peerSet. +// SetReservedOnlyPeer not yet implemented func (h *Handler) SetReservedOnlyPeer(setID int, peers ...peer.ID) { // TODO: not yet implemented (#1888) logger.Errorf("failed to do action %s on peerSet: not implemented yet", setReservedOnly) From 55dca9cc4f733de6627a17bc2e8e79565d0324aa Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 7 Feb 2022 09:27:44 -0400 Subject: [PATCH 07/50] chore: define struct inside function --- dot/peerset/peerstate.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index d3edd23fae..75b55bcbe3 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -179,11 +179,6 @@ func (ps *PeersState) peers() []peer.ID { return peerIDs } -type connectedPeerReputation struct { - peerID peer.ID - reputation Reputation -} - // sortedPeers returns the list of peers we are connected to of a specific set. func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { ps.peerStateRWMutex.RLock() @@ -194,6 +189,11 @@ func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { return nil } + type connectedPeerReputation struct { + peerID peer.ID + reputation Reputation + } + connectedPeersReps := make([]connectedPeerReputation, 0) for peerID, node := range ps.nodes { From f0c0e107a2e55f354a7b0454ebb41d93263ca52f Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 7 Feb 2022 09:30:31 -0400 Subject: [PATCH 08/50] chore: use sync.Mutex composition --- dot/peerset/peerset.go | 12 ++++---- dot/peerset/peerstate.go | 62 ++++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 6df05900c7..eae0b41c74 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -115,8 +115,8 @@ func newReputationChange(value Reputation, reason string) ReputationChange { // PeerSet is a container for all the components of a peerSet. type PeerSet struct { - peerSetMutex sync.Mutex - peerState *PeersState + sync.Mutex + peerState *PeersState reservedNode map[peer.ID]struct{} // TODO: this will be useful for reserved only mode @@ -213,8 +213,8 @@ func reputationTick(reput Reputation) Reputation { // updateTime updates the value of latestTimeUpdate and performs all the updates that // happen over time, such as Reputation increases for staying connected. func (ps *PeerSet) updateTime() error { - ps.peerSetMutex.Lock() - defer ps.peerSetMutex.Unlock() + ps.Lock() + defer ps.Unlock() currTime := time.Now() // identify the time difference between current time and last update time for peer reputation in seconds. @@ -547,14 +547,14 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { var nodeReputation Reputation - state.peerStateRWMutex.RLock() + state.RLock() node, has := state.nodes[pid] if has { nodeReputation = node.rep } - state.peerStateRWMutex.RUnlock() + state.RUnlock() switch { case nodeReputation < BannedThresholdValue: diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index 75b55bcbe3..44ef1b3f95 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -102,12 +102,12 @@ type PeersState struct { // since, single Info can also manage the flow. sets []Info - peerStateRWMutex sync.RWMutex + sync.RWMutex } func (ps *PeersState) getNode(p peer.ID) (*node, error) { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() + ps.RLock() + defer ps.RUnlock() if n, ok := ps.nodes[p]; ok { return n, nil } @@ -149,8 +149,8 @@ func (ps *PeersState) getSetLength() int { // peerStatus returns the status of peer based on its connection state // i.e. connectedPeer, notConnectedPeer or unknownPeer. func (ps *PeersState) peerStatus(set int, peerID peer.ID) string { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() + ps.RLock() + defer ps.RUnlock() node, has := ps.nodes[peerID] if !has { @@ -169,8 +169,8 @@ func (ps *PeersState) peerStatus(set int, peerID peer.ID) string { // peers return the list of all the peers we know of. func (ps *PeersState) peers() []peer.ID { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() + ps.RLock() + defer ps.RUnlock() peerIDs := make([]peer.ID, 0, len(ps.nodes)) for k := range ps.nodes { @@ -181,8 +181,8 @@ func (ps *PeersState) peers() []peer.ID { // sortedPeers returns the list of peers we are connected to of a specific set. func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() + ps.RLock() + defer ps.RUnlock() if len(ps.sets) == 0 || len(ps.sets) < idx { logger.Debug("peer state doesn't have info for the provided index") @@ -220,8 +220,8 @@ func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { } func (ps *PeersState) updateReputationByTick(peerID peer.ID) (after Reputation, err error) { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() node, has := ps.nodes[peerID] if !has { @@ -239,8 +239,8 @@ func (ps *PeersState) updateReputationByTick(peerID peer.ID) (after Reputation, func (ps *PeersState) addReputation(peerID peer.ID, change ReputationChange) ( newReputation Reputation, err error) { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() node, has := ps.nodes[peerID] if !has { @@ -255,8 +255,8 @@ func (ps *PeersState) addReputation(peerID peer.ID, change ReputationChange) ( // highestNotConnectedPeer returns the peer with the highest Reputation and that we are not connected to. func (ps *PeersState) highestNotConnectedPeer(set int) peer.ID { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() + ps.RLock() + defer ps.RUnlock() maxRep := math.MinInt32 var higestPeerID peer.ID @@ -289,8 +289,8 @@ func (ps *PeersState) hasFreeIncomingSlot(set int) bool { // addNoSlotNode adds a node to the list of nodes that don't occupy slots. // has no effect if the node was already in the group. func (ps *PeersState) addNoSlotNode(idx int, peerID peer.ID) error { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() if _, ok := ps.sets[idx].noSlotNodes[peerID]; ok { logger.Debugf("peer %s already exists in no slot node", peerID) @@ -316,8 +316,8 @@ func (ps *PeersState) addNoSlotNode(idx int, peerID peer.ID) error { } func (ps *PeersState) removeNoSlotNode(idx int, peerID peer.ID) error { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() if _, ok := ps.sets[idx].noSlotNodes[peerID]; !ok { logger.Debugf("peer %s is not in no-slot node map", peerID) @@ -344,8 +344,8 @@ func (ps *PeersState) removeNoSlotNode(idx int, peerID peer.ID) error { // disconnect updates the node status to the notConnected state. // It should be called only when the node is in connected state. func (ps *PeersState) disconnect(idx int, peerID peer.ID) error { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() info := ps.sets[idx] node, has := ps.nodes[peerID] @@ -376,8 +376,8 @@ func (ps *PeersState) disconnect(idx int, peerID peer.ID) error { // discover takes input for set id and create a node and insert in the list. // the initial Reputation of the peer will be 0 and ingoing notMember state. func (ps *PeersState) discover(set int, peerID peer.ID) { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() numSet := len(ps.sets) @@ -390,8 +390,8 @@ func (ps *PeersState) discover(set int, peerID peer.ID) { } func (ps *PeersState) lastConnectedAndDiscovered(set int, peerID peer.ID) (time.Time, error) { - ps.peerStateRWMutex.RLock() - defer ps.peerStateRWMutex.RUnlock() + ps.RLock() + defer ps.RUnlock() node, has := ps.nodes[peerID] if !has { @@ -407,8 +407,8 @@ func (ps *PeersState) lastConnectedAndDiscovered(set int, peerID peer.ID) (time. // forgetPeer removes the peer with reputation 0 from the peerSet. func (ps *PeersState) forgetPeer(set int, peerID peer.ID) error { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() node, has := ps.nodes[peerID] if !has { @@ -444,8 +444,8 @@ func (ps *PeersState) forgetPeer(set int, peerID peer.ID) error { // If the slots are full, the node stays "not connected" and we return the error ErrOutgoingSlotsUnavailable. // non slot occupying nodes don't count towards the number of slots. func (ps *PeersState) tryOutgoing(setID int, peerID peer.ID) error { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() _, isNoSlotNode := ps.sets[setID].noSlotNodes[peerID] @@ -471,8 +471,8 @@ func (ps *PeersState) tryOutgoing(setID int, peerID peer.ID) error { // If the slots are full, the node stays "not connected" and we return Err. // non slot occupying nodes don't count towards the number of slots. func (ps *PeersState) tryAcceptIncoming(setID int, peerID peer.ID) error { - ps.peerStateRWMutex.Lock() - defer ps.peerStateRWMutex.Unlock() + ps.Lock() + defer ps.Unlock() var isNoSlotOccupied bool if _, ok := ps.sets[setID].noSlotNodes[peerID]; ok { From 1f39a56d1db9a815aafcd91667d4ea5c4b818a3f Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 7 Feb 2022 10:59:16 -0400 Subject: [PATCH 09/50] chore: adding license --- dot/peerset/reputation.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dot/peerset/reputation.go b/dot/peerset/reputation.go index 2004bf13e9..053101b24d 100644 --- a/dot/peerset/reputation.go +++ b/dot/peerset/reputation.go @@ -1,3 +1,6 @@ +// Copyright 2022 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package peerset import "math" From 5366804802acfba98a92a3bd370586fb653f02f3 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 7 Feb 2022 11:03:05 -0400 Subject: [PATCH 10/50] chore: keep `Reputation` at `peerset` --- dot/peerset/peerset.go | 28 ++++++++++++++++++++++++++++ dot/peerset/reputation.go | 33 --------------------------------- 2 files changed, 28 insertions(+), 33 deletions(-) delete mode 100644 dot/peerset/reputation.go diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index eae0b41c74..03b793876c 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -6,6 +6,7 @@ package peerset import ( "context" "fmt" + "math" "sync" "time" @@ -101,6 +102,33 @@ type Message struct { PeerID peer.ID } +// Reputation represents reputation value of the node +type Reputation int32 + +// add handles overflow and underflow condition while adding two Reputation values. +func (r Reputation) add(num Reputation) Reputation { + if num > 0 { + if r > math.MaxInt32-num { + return math.MaxInt32 + } + } else if r < math.MinInt32-num { + return math.MinInt32 + } + return r + num +} + +// sub handles underflow condition while subtracting two Reputation values. +func (r Reputation) sub(num Reputation) Reputation { + if num < 0 { + if r > math.MaxInt32+num { + return math.MaxInt32 + } + } else if r < math.MinInt32+num { + return math.MinInt32 + } + return r - num +} + // ReputationChange is description of a reputation adjustment for a node type ReputationChange struct { // PeerReputation value diff --git a/dot/peerset/reputation.go b/dot/peerset/reputation.go deleted file mode 100644 index 053101b24d..0000000000 --- a/dot/peerset/reputation.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2022 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package peerset - -import "math" - -// Reputation represents reputation value of the node -type Reputation int32 - -// add handles overflow and underflow condition while adding two Reputation values. -func (r Reputation) add(num Reputation) Reputation { - if num > 0 { - if r > math.MaxInt32-num { - return math.MaxInt32 - } - } else if r < math.MinInt32-num { - return math.MinInt32 - } - return r + num -} - -// sub handles underflow condition while subtracting two Reputation values. -func (r Reputation) sub(num Reputation) Reputation { - if num < 0 { - if r > math.MaxInt32+num { - return math.MaxInt32 - } - } else if r < math.MinInt32+num { - return math.MinInt32 - } - return r - num -} From 3695853f6dead0a3c3fb638ddf6c6b4ed351cdb5 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 4 Mar 2022 13:18:17 -0400 Subject: [PATCH 11/50] chore: replace fun signature to interface implementation --- dot/network/service.go | 4 +- dot/network/state.go | 2 +- dot/peerset/handler.go | 10 +- dot/peerset/mock_message_processor_test.go | 46 ++++++ dot/peerset/peerset.go | 74 ++++++--- dot/peerset/peerset_test.go | 173 ++++++--------------- dot/peerset/test_helpers.go | 6 +- 7 files changed, 154 insertions(+), 161 deletions(-) create mode 100644 dot/peerset/mock_message_processor_test.go diff --git a/dot/network/service.go b/dot/network/service.go index 13f8ad25b5..e0028fdd21 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -690,14 +690,14 @@ func (s *Service) ReportPeer(change peerset.ReputationChange, p peer.ID) { } func (s *Service) startPeerSetHandler() { - s.host.cm.peerSetHandler.Start(s.ctx, s.processMessage) + s.host.cm.peerSetHandler.Start(s.ctx, s) // wait for peerSetHandler to start. if !s.noBootstrap { s.host.bootstrap() } } -func (s *Service) processMessage(msg peerset.Message) { +func (s *Service) Process(msg peerset.Message) { peerID := msg.PeerID if peerID == "" { logger.Errorf("found empty peer id in peerset message") diff --git a/dot/network/state.go b/dot/network/state.go index 201d870c91..f36af0ec8d 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -47,7 +47,7 @@ type TransactionHandler interface { // PeerSetHandler is the interface used by the connection manager to handle peerset. type PeerSetHandler interface { - Start(context.Context, func(peerset.Message)) + Start(context.Context, peerset.MessageProcessor) ReportPeer(peerset.ReputationChange, ...peer.ID) PeerAdd PeerRemove diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 355801c5d3..1eaca6b167 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -45,7 +45,7 @@ func (h *Handler) AddReservedPeer(setID int, peers ...peer.ID) { } } -// RemoveReservedPeer remove reserved peer from peerSet. +// RemoveReservedPeer removes reserved peer from peerSet. func (h *Handler) RemoveReservedPeer(setID int, peers ...peer.ID) { err := h.peerSet.removeReservedPeers(setID, peers...) if err != nil { @@ -54,7 +54,7 @@ func (h *Handler) RemoveReservedPeer(setID int, peers ...peer.ID) { } } -// SetReservedPeer set the reserve peer into peerSet +// SetReservedPeer sets the reserve peer into peerSet func (h *Handler) SetReservedPeer(setID int, peers ...peer.ID) { // TODO: this is not used yet, might required to implement RPC Call for this. err := h.peerSet.setReservedPeer(setID, peers...) @@ -119,11 +119,11 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { } // Start starts peerSet processing -func (h *Handler) Start(ctx context.Context, processMessageFn func(Message)) { - h.peerSet.start(ctx, processMessageFn) +func (h *Handler) Start(ctx context.Context, processor MessageProcessor) { + h.peerSet.start(ctx, processor) } -// SortedPeers return chan for sorted connected peer in the peerSet. +// SortedPeers returns a sorted peer ID slice for connected peers in the peerSet. func (h *Handler) SortedPeers(setIdx int) peer.IDSlice { return h.peerSet.peerState.sortedPeers(setIdx) } diff --git a/dot/peerset/mock_message_processor_test.go b/dot/peerset/mock_message_processor_test.go new file mode 100644 index 0000000000..91799591fa --- /dev/null +++ b/dot/peerset/mock_message_processor_test.go @@ -0,0 +1,46 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ChainSafe/gossamer/dot/peerset (interfaces: MessageProcessor) + +// Package peerset is a generated GoMock package. +package peerset + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockMessageProcessor is a mock of MessageProcessor interface. +type MockMessageProcessor struct { + ctrl *gomock.Controller + recorder *MockMessageProcessorMockRecorder +} + +// MockMessageProcessorMockRecorder is the mock recorder for MockMessageProcessor. +type MockMessageProcessorMockRecorder struct { + mock *MockMessageProcessor +} + +// NewMockMessageProcessor creates a new mock instance. +func NewMockMessageProcessor(ctrl *gomock.Controller) *MockMessageProcessor { + mock := &MockMessageProcessor{ctrl: ctrl} + mock.recorder = &MockMessageProcessorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMessageProcessor) EXPECT() *MockMessageProcessorMockRecorder { + return m.recorder +} + +// Process mocks base method. +func (m *MockMessageProcessor) Process(arg0 Message) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Process", arg0) +} + +// Process indicates an expected call of Process. +func (mr *MockMessageProcessorMockRecorder) Process(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Process", reflect.TypeOf((*MockMessageProcessor)(nil).Process), arg0) +} diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 03b793876c..857d1e2075 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -141,6 +141,12 @@ func newReputationChange(value Reputation, reason string) ReputationChange { return ReputationChange{value, reason} } +// MessageProcessor interface allows network layer to receive and +// process messages from the peerstate layer +type MessageProcessor interface { + Process(Message) +} + // PeerSet is a container for all the components of a peerSet. type PeerSet struct { sync.Mutex @@ -158,7 +164,7 @@ type PeerSet struct { // second, to match the period of the Reputation updates. nextPeriodicAllocSlots time.Duration - processMessageFn func(Message) + processor MessageProcessor } // config is configuration of a single set. @@ -324,11 +330,13 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { return err } - ps.processMessageFn(Message{ + dropMessage := Message{ Status: Drop, setID: uint64(i), PeerID: pid, - }) + } + + ps.processor.Process(dropMessage) if err = ps.allocSlots(i); err != nil { return err @@ -371,12 +379,13 @@ func (ps *PeerSet) allocSlots(setIdx int) error { return err } - ps.processMessageFn(Message{ + connectMessage := Message{ Status: Connect, setID: uint64(setIdx), PeerID: reservePeer, - }) + } + ps.processor.Process(connectMessage) } // nothing more to do if we're in reserved mode. @@ -401,11 +410,13 @@ func (ps *PeerSet) allocSlots(setIdx int) error { break } - ps.processMessageFn(Message{ + connectMessage := Message{ Status: Connect, setID: uint64(setIdx), PeerID: peerID, - }) + } + + ps.processor.Process(connectMessage) logger.Debugf("Sent connect message to peer %s", peerID) } @@ -458,11 +469,13 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { return err } - ps.processMessageFn(Message{ + dropMessage := Message{ Status: Drop, setID: uint64(setID), PeerID: peerID, - }) + } + + ps.processor.Process(dropMessage) } } @@ -516,11 +529,13 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { } if status := ps.peerState.peerStatus(setID, pid); status == connectedPeer { - ps.processMessageFn(Message{ + dropMessage := Message{ Status: Drop, setID: uint64(setID), PeerID: pid, - }) + } + + ps.processor.Process(dropMessage) // disconnect and forget err := ps.peerState.disconnect(setID, pid) @@ -552,11 +567,13 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { for _, pid := range peers { if ps.isReservedOnly { if _, ok := ps.reservedNode[pid]; !ok { - ps.processMessageFn(Message{ + rejectMessage := Message{ Status: Reject, setID: uint64(setID), PeerID: pid, - }) + } + + ps.processor.Process(rejectMessage) continue } } @@ -577,34 +594,38 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { state.RLock() node, has := state.nodes[pid] - if has { nodeReputation = node.rep } - state.RUnlock() switch { case nodeReputation < BannedThresholdValue: - ps.processMessageFn(Message{ + rejectMessage := Message{ Status: Reject, setID: uint64(setID), PeerID: pid, - }) + } + + ps.processor.Process(rejectMessage) case state.tryAcceptIncoming(setID, pid) != nil: - ps.processMessageFn(Message{ + rejectMessage := Message{ Status: Reject, setID: uint64(setID), PeerID: pid, - }) + } + + ps.processor.Process(rejectMessage) default: logger.Debugf("incoming connection accepted from peer %s", pid) - ps.processMessageFn(Message{ + acceptMessage := Message{ Status: Accept, setID: uint64(setID), PeerID: pid, - }) + } + + ps.processor.Process(acceptMessage) } } @@ -645,11 +666,14 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e if err = state.disconnect(setIdx, pid); err != nil { return err } - ps.processMessageFn(Message{ + + dropMessage := Message{ Status: Drop, setID: uint64(setIdx), PeerID: pid, - }) + } + + ps.processor.Process(dropMessage) // TODO: figure out the condition of connection refuse. if reason == RefusedDrop { @@ -663,8 +687,8 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e } // start handles all the action for the peerSet. -func (ps *PeerSet) start(ctx context.Context, processMessageFn func(Message)) { - ps.processMessageFn = processMessageFn +func (ps *PeerSet) start(ctx context.Context, processor MessageProcessor) { + ps.processor = processor go ps.doWork(ctx) } diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index a7e958b0d7..d2199ecf66 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -4,78 +4,24 @@ package peerset import ( - "sync" "testing" "time" + "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" ) -type assertMessageParam struct { - expect interface{} - got interface{} -} - -type mockProcessMessage struct { - callCounter int - executionCounter int - expects map[int]assertMessageParam - mockMutex sync.Mutex -} - -func newMockProcessMessage() *mockProcessMessage { - return &mockProcessMessage{ - expects: make(map[int]assertMessageParam), - } -} - -func (mock *mockProcessMessage) ExpectByCall(expected interface{}) { - mock.mockMutex.Lock() - defer mock.mockMutex.Unlock() - - mock.callCounter++ - mock.expects[mock.callCounter] = assertMessageParam{ - expect: expected, - } -} - -func (mock *mockProcessMessage) ProcessMessage() func(Message) { - return func(m Message) { - mock.mockMutex.Lock() - defer mock.mockMutex.Unlock() - - mock.executionCounter++ - assert, has := mock.expects[mock.executionCounter] - if has { - assert.got = m - mock.expects[mock.executionCounter] = assert - } - } -} - -func (mock *mockProcessMessage) Assert(t *testing.T) { - mock.mockMutex.Lock() - defer mock.mockMutex.Unlock() - - // the number of calls to processMessage should be the same we expect - require.Equal(t, mock.callCounter, mock.executionCounter) - - // follow the order we defined - for i := 1; i <= mock.callCounter; i++ { - assert, has := mock.expects[i] - require.True(t, has) - require.Equal(t, assert.expect, assert.got) - } -} - func TestPeerSetBanned(t *testing.T) { t.Parallel() - mock := newMockProcessMessage() - processMessageFn := mock.ProcessMessage() + ctrl := gomock.NewController(t) + processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Drop, setID: 0x0, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Reject, setID: 0x0, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Accept, setID: 0x0, PeerID: "testPeer1"}) - handler := newTestPeerSet(t, 25, 25, nil, nil, false, processMessageFn) + handler := newTestPeerSet(t, 25, 25, nil, nil, false, processor) ps := handler.peerSet require.Equal(t, unknownPeer, ps.peerState.peerStatus(0, peer1)) @@ -87,35 +33,30 @@ func TestPeerSetBanned(t *testing.T) { // we ban a node by setting its reputation under the threshold. rpc := newReputationChange(BannedThresholdValue-1, "") - mock.ExpectByCall(Message{Status: Drop, setID: 0x0, PeerID: "testPeer1"}) // we need one for the message to be processed. handler.ReportPeer(rpc, peer1) time.Sleep(time.Millisecond * 100) - mock.ExpectByCall(Message{Status: Reject, setID: 0x0, PeerID: "testPeer1"}) // check that an incoming connection from that node gets refused. handler.Incoming(0, peer1) // wait a bit for the node's reputation to go above the threshold. time.Sleep(time.Millisecond * 1200) - mock.ExpectByCall(Message{Status: Accept, setID: 0x0, PeerID: "testPeer1"}) // try again. This time the node should be accepted. handler.Incoming(0, peer1) - - mock.Assert(t) } func TestAddReservedPeers(t *testing.T) { t.Parallel() - mock := newMockProcessMessage() - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: bootNode}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) - processMessageFn := mock.ProcessMessage() + ctrl := gomock.NewController(t) + processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: bootNode}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) - handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false, processMessageFn) + handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false, processor) ps := handler.peerSet handler.AddReservedPeer(0, reservedPeer) @@ -124,60 +65,52 @@ func TestAddReservedPeers(t *testing.T) { time.Sleep(time.Millisecond * 200) require.Equal(t, uint32(1), ps.peerState.sets[0].numOut) - require.Equal(t, 3, mock.executionCounter) - - mock.Assert(t) } func TestPeerSetIncoming(t *testing.T) { t.Parallel() - mock := newMockProcessMessage() - processMessageFn := mock.ProcessMessage() - - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: bootNode}) - mock.ExpectByCall(Message{Status: Accept, setID: 0, PeerID: incomingPeer}) - mock.ExpectByCall(Message{Status: Accept, setID: 0, PeerID: incoming2}) - mock.ExpectByCall(Message{Status: Reject, setID: 0, PeerID: incoming3}) + ctrl := gomock.NewController(t) + processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: bootNode}) + processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incomingPeer}) + processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incoming2}) + processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: incoming3}) handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, - []peer.ID{}, false, processMessageFn) + []peer.ID{}, false, processor) handler.Incoming(0, incomingPeer) handler.Incoming(0, incoming2) handler.Incoming(0, incoming3) - - mock.Assert(t) } func TestPeerSetDiscovered(t *testing.T) { t.Parallel() - mock := newMockProcessMessage() - processMessageFn := mock.ProcessMessage() + ctrl := gomock.NewController(t) + processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered1}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered2}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: discovered1}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: discovered2}) - - handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processMessageFn) + handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processor) handler.AddPeer(0, discovered1) - handler.AddPeer(0, discovered1) - handler.AddPeer(0, discovered2) - - mock.Assert(t) } func TestReAllocAfterBanned(t *testing.T) { t.Parallel() - mock := newMockProcessMessage() - processMessageFn := mock.ProcessMessage() + ctrl := gomock.NewController(t) + processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: peer1}) + processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: peer1}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: peer1}) - handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false, processMessageFn) + handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false, processor) ps := handler.peerSet // adding peer1 with incoming slot. @@ -190,63 +123,53 @@ func TestReAllocAfterBanned(t *testing.T) { // We ban a node by setting its reputation under the threshold. rep := newReputationChange(BannedThresholdValue-1, "") - mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: peer1}) // we need one for the message to be processed. handler.ReportPeer(rep, peer1) time.Sleep(time.Millisecond * 100) // Check that an incoming connection from that node gets refused. - mock.ExpectByCall(Message{Status: Reject, setID: 0, PeerID: peer1}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: peer1}) + handler.Incoming(0, peer1) time.Sleep(time.Second * 2) - mock.Assert(t) } func TestRemovePeer(t *testing.T) { t.Parallel() - mock := newMockProcessMessage() - processMessageFn := mock.ProcessMessage() + ctrl := gomock.NewController(t) + processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "testDiscovered1"}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "testDiscovered2"}) + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: "testDiscovered1"}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: "testDiscovered2"}) handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, - nil, false, processMessageFn) + nil, false, processor) ps := handler.peerSet - require.Equal(t, 2, mock.executionCounter) - time.Sleep(time.Millisecond * 500) - mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) - mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) handler.RemovePeer(0, discovered1, discovered2) require.Equal(t, 0, len(ps.peerState.nodes)) - - mock.Assert(t) } func TestSetReservePeer(t *testing.T) { t.Parallel() - mock := newMockProcessMessage() - processMessageFn := mock.ProcessMessage() + ctrl := gomock.NewController(t) + processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "newRsrPeer"}) + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: reservedPeer2}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, - true, processMessageFn) + true, processor) ps := handler.peerSet - require.Equal(t, 2, mock.executionCounter) - - mock.ExpectByCall(Message{Status: Connect, setID: 0, PeerID: "newRsrPeer"}) - mock.ExpectByCall(Message{Status: Drop, setID: 0, PeerID: reservedPeer2}) - newRsrPeerSet := peer.IDSlice{reservedPeer, peer.ID("newRsrPeer")} // add newRsrPeer but remove reservedPeer2 handler.SetReservedPeer(0, newRsrPeerSet...) @@ -255,6 +178,4 @@ func TestSetReservePeer(t *testing.T) { for _, p := range newRsrPeerSet { require.Contains(t, ps.reservedNode, p) } - - mock.Assert(t) } diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index bba9a16e0e..42028d44c0 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -25,8 +25,10 @@ const ( peer2 = peer.ID("testPeer2") ) +//go:generate mockgen -destination=mock_message_processor_test.go -package $GOPACKAGE . MessageProcessor + func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, - reservedPeers []peer.ID, reservedOnly bool, processMessage func(Message)) *Handler { + reservedPeers []peer.ID, reservedOnly bool, processor MessageProcessor) *Handler { t.Helper() con := &ConfigSet{ Set: []*config{ @@ -42,7 +44,7 @@ func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, handler, err := NewPeerSetHandler(con) require.NoError(t, err) - handler.Start(context.Background(), processMessage) + handler.Start(context.Background(), processor) handler.AddPeer(0, bootNodes...) handler.AddReservedPeer(0, reservedPeers...) From 36ab22e8b52d10946b34011d97d830b38e79b597 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 4 Mar 2022 13:21:32 -0400 Subject: [PATCH 12/50] chore: move comment --- dot/peerset/peerset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 857d1e2075..68ec158056 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -563,9 +563,9 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { return err } - // This is for reserved only mode. for _, pid := range peers { if ps.isReservedOnly { + // This is for reserved only mode. if _, ok := ps.reservedNode[pid]; !ok { rejectMessage := Message{ Status: Reject, From fc0cf713dc62873b958e9ff95e23aa5d8a37b48f Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 4 Mar 2022 13:25:59 -0400 Subject: [PATCH 13/50] chore: improve test description --- dot/peerset/peerset_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index d2199ecf66..20981074f5 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -12,14 +12,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestPeerSetBanned(t *testing.T) { +func Test_Ban_Reject_Accept_Peer(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Drop, setID: 0x0, PeerID: "testPeer1"}) - processor.EXPECT().Process(Message{Status: Reject, setID: 0x0, PeerID: "testPeer1"}) - processor.EXPECT().Process(Message{Status: Accept, setID: 0x0, PeerID: "testPeer1"}) handler := newTestPeerSet(t, 25, 25, nil, nil, false, processor) @@ -34,16 +31,19 @@ func TestPeerSetBanned(t *testing.T) { rpc := newReputationChange(BannedThresholdValue-1, "") // we need one for the message to be processed. + processor.EXPECT().Process(Message{Status: Drop, setID: 0x0, PeerID: "testPeer1"}) handler.ReportPeer(rpc, peer1) time.Sleep(time.Millisecond * 100) // check that an incoming connection from that node gets refused. + processor.EXPECT().Process(Message{Status: Reject, setID: 0x0, PeerID: "testPeer1"}) handler.Incoming(0, peer1) // wait a bit for the node's reputation to go above the threshold. time.Sleep(time.Millisecond * 1200) // try again. This time the node should be accepted. + processor.EXPECT().Process(Message{Status: Accept, setID: 0x0, PeerID: "testPeer1"}) handler.Incoming(0, peer1) } From 147575c75adfe91901519e6b057e07150a97172e Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 4 Mar 2022 13:30:55 -0400 Subject: [PATCH 14/50] chore: improve test exepect function calls --- dot/peerset/peerset_test.go | 39 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index 20981074f5..2237a53896 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -72,16 +72,18 @@ func TestPeerSetIncoming(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: bootNode}) - processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incomingPeer}) - processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incoming2}) - processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: incoming3}) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: bootNode}) handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, []peer.ID{}, false, processor) + processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incomingPeer}) handler.Incoming(0, incomingPeer) + + processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incoming2}) handler.Incoming(0, incoming2) + + processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: incoming3}) handler.Incoming(0, incoming3) } @@ -90,14 +92,15 @@ func TestPeerSetDiscovered(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered1}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered2}) - handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processor) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) handler.AddPeer(0, discovered1) + + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered1}) handler.AddPeer(0, discovered1) + + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered2}) handler.AddPeer(0, discovered2) } @@ -106,10 +109,6 @@ func TestReAllocAfterBanned(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: peer1}) - processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: peer1}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: peer1}) - handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false, processor) ps := handler.peerSet @@ -121,14 +120,16 @@ func TestReAllocAfterBanned(t *testing.T) { } // We ban a node by setting its reputation under the threshold. + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: peer1}) rep := newReputationChange(BannedThresholdValue-1, "") // we need one for the message to be processed. + processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: peer1}) handler.ReportPeer(rep, peer1) time.Sleep(time.Millisecond * 100) // Check that an incoming connection from that node gets refused. - + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: peer1}) handler.Incoming(0, peer1) time.Sleep(time.Second * 2) @@ -139,17 +140,17 @@ func TestRemovePeer(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "testDiscovered1"}) processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "testDiscovered2"}) - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) - handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, nil, false, processor) ps := handler.peerSet time.Sleep(time.Millisecond * 500) + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) handler.RemovePeer(0, discovered1, discovered2) require.Equal(t, 0, len(ps.peerState.nodes)) @@ -160,11 +161,9 @@ func TestSetReservePeer(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "newRsrPeer"}) - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: reservedPeer2}) - handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, true, processor) @@ -172,6 +171,8 @@ func TestSetReservePeer(t *testing.T) { newRsrPeerSet := peer.IDSlice{reservedPeer, peer.ID("newRsrPeer")} // add newRsrPeer but remove reservedPeer2 + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "newRsrPeer"}) + processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: reservedPeer2}) handler.SetReservedPeer(0, newRsrPeerSet...) require.Equal(t, len(newRsrPeerSet), len(ps.reservedNode)) From 8e693b59c32dafb0bd92233eaf0d8059e26990dd Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 8 Mar 2022 10:03:24 -0400 Subject: [PATCH 15/50] chore: add comment to network.Process exported function --- dot/network/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dot/network/service.go b/dot/network/service.go index e0028fdd21..87f495a96a 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -697,6 +697,7 @@ func (s *Service) startPeerSetHandler() { } } +// Process will connect, drop or reject a peer based on a peerset message func (s *Service) Process(msg peerset.Message) { peerID := msg.PeerID if peerID == "" { From a340adca5059ab6527a5006aca3c63fa4e6a4f4a Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 11 Mar 2022 14:15:49 -0400 Subject: [PATCH 16/50] chore: fix TestPeerSetDiscovered unit test --- dot/peerset/peerset_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index 2237a53896..e159ad9a32 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -92,13 +92,13 @@ func TestPeerSetDiscovered(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processor) processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - handler.AddPeer(0, discovered1) + handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processor) processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered1}) handler.AddPeer(0, discovered1) + handler.AddPeer(0, discovered1) processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered2}) handler.AddPeer(0, discovered2) From 39e1974b11bf1dbf7674bb0db5989e66f189dcc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Fri, 11 Mar 2022 17:37:48 -0400 Subject: [PATCH 17/50] chore: fix typo at comment Co-authored-by: Quentin McGaw --- dot/peerset/peerset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 68ec158056..ac6680ac27 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -141,7 +141,7 @@ func newReputationChange(value Reputation, reason string) ReputationChange { return ReputationChange{value, reason} } -// MessageProcessor interface allows network layer to receive and +// MessageProcessor interface allows the network layer to receive and // process messages from the peerstate layer type MessageProcessor interface { Process(Message) From 3773559145e025e3475371a2e39e14e2aaf5f1ea Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 11 Mar 2022 17:49:47 -0400 Subject: [PATCH 18/50] chore: wrap errors and add String to ReputationChange --- dot/peerset/handler.go | 20 +++++++++--------- dot/peerset/peerset.go | 47 ++++++++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 1eaca6b167..acfa660dcd 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -11,7 +11,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -const logStringPattern = "call=%s, set-id=%d, reputation change %v, peers=[%s]" +const logStringPattern = "call=%s, set-id=%d, reputation change %s, peers=[%s]" // Handler manages peerSet. type Handler struct { @@ -40,7 +40,7 @@ func (h *Handler) SetReservedOnlyPeer(setID int, peers ...peer.ID) { func (h *Handler) AddReservedPeer(setID int, peers ...peer.ID) { err := h.peerSet.addReservedPeers(setID, peers...) if err != nil { - msg := fmt.Sprintf(logStringPattern, addReservedPeer, setID, nil, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, addReservedPeer, setID, "", stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -49,17 +49,17 @@ func (h *Handler) AddReservedPeer(setID int, peers ...peer.ID) { func (h *Handler) RemoveReservedPeer(setID int, peers ...peer.ID) { err := h.peerSet.removeReservedPeers(setID, peers...) if err != nil { - msg := fmt.Sprintf(logStringPattern, removeReservedPeer, setID, nil, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, removeReservedPeer, setID, "", stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } // SetReservedPeer sets the reserve peer into peerSet func (h *Handler) SetReservedPeer(setID int, peers ...peer.ID) { - // TODO: this is not used yet, might required to implement RPC Call for this. + // TODO: this is not used yet, it might be required to implement an RPC Call for this. err := h.peerSet.setReservedPeer(setID, peers...) if err != nil { - msg := fmt.Sprintf(logStringPattern, setReservedPeers, setID, nil, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, setReservedPeers, setID, "", stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -68,7 +68,7 @@ func (h *Handler) SetReservedPeer(setID int, peers ...peer.ID) { func (h *Handler) AddPeer(setID int, peers ...peer.ID) { err := h.peerSet.addPeer(setID, peers) if err != nil { - msg := fmt.Sprintf(logStringPattern, addToPeerSet, setID, nil, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, addToPeerSet, setID, "", stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -77,7 +77,7 @@ func (h *Handler) AddPeer(setID int, peers ...peer.ID) { func (h *Handler) RemovePeer(setID int, peers ...peer.ID) { err := h.peerSet.removePeer(setID, peers...) if err != nil { - msg := fmt.Sprintf(logStringPattern, removeFromPeerSet, setID, nil, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, removeFromPeerSet, setID, "", stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -86,7 +86,7 @@ func (h *Handler) RemovePeer(setID int, peers ...peer.ID) { func (h *Handler) ReportPeer(rep ReputationChange, peers ...peer.ID) { err := h.peerSet.reportPeer(rep, peers...) if err != nil { - msg := fmt.Sprintf(logStringPattern, reportPeer, 0, rep, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, reportPeer, 0, rep.String(), stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -95,7 +95,7 @@ func (h *Handler) ReportPeer(rep ReputationChange, peers ...peer.ID) { func (h *Handler) Incoming(setID int, peers ...peer.ID) { err := h.peerSet.incoming(setID, peers...) if err != nil { - msg := fmt.Sprintf(logStringPattern, incoming, setID, nil, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, incoming, setID, "", stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } @@ -104,7 +104,7 @@ func (h *Handler) Incoming(setID int, peers ...peer.ID) { func (h *Handler) DisconnectPeer(setID int, peers ...peer.ID) { err := h.peerSet.disconnect(setID, UnknownDrop, peers...) if err != nil { - msg := fmt.Sprintf(logStringPattern, disconnect, setID, nil, stringfyPeers(peers)) + msg := fmt.Sprintf(logStringPattern, disconnect, setID, "", stringfyPeers(peers)) logger.Errorf("failed to do action %s on peerSet: %s", msg, err) } } diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 68ec158056..d20e0358e5 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -137,6 +137,10 @@ type ReputationChange struct { Reason string } +func (r *ReputationChange) String() string { + return fmt.Sprintf("value: %d, reason: %s", r.Value, r.Reason) +} + func newReputationChange(value Reputation, reason string) ReputationChange { return ReputationChange{value, reason} } @@ -264,7 +268,7 @@ func (ps *PeerSet) updateTime() error { for _, peerID := range ps.peerState.peers() { after, err := ps.peerState.updateReputationByTick(peerID) if err != nil { - return err + return fmt.Errorf("cannot update reputation by tick: %w", err) } if after != 0 { @@ -280,7 +284,7 @@ func (ps *PeerSet) updateTime() error { lastDiscoveredTime, err := ps.peerState.lastConnectedAndDiscovered(set, peerID) if err != nil { - return err + return fmt.Errorf("cannot get last connected peer: %w", err) } if lastDiscoveredTime.Add(forgetAfterTime).Second() >= currTime.Second() { @@ -290,7 +294,7 @@ func (ps *PeerSet) updateTime() error { // forget peer removes the peer from the list of members of the set. err = ps.peerState.forgetPeer(set, peerID) if err != nil { - return err + return fmt.Errorf("cannot forget peer %s from set %d: %w", peerID, set, err) } } } @@ -305,13 +309,13 @@ func (ps *PeerSet) updateTime() error { func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { // we want reputations to be up-to-date before adjusting them. if err := ps.updateTime(); err != nil { - return err + return fmt.Errorf("cannot update time: %w", err) } for _, pid := range peers { rep, err := ps.peerState.addReputation(pid, change) if err != nil { - return err + return fmt.Errorf("cannot add reputation (%s) to peer %s: %w", pid, change.String(), err) } if rep >= BannedThresholdValue { @@ -327,7 +331,7 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { // disconnect peer err = ps.peerState.disconnect(i, pid) if err != nil { - return err + return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, i, err) } dropMessage := Message{ @@ -339,7 +343,7 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { ps.processor.Process(dropMessage) if err = ps.allocSlots(i); err != nil { - return err + return fmt.Errorf("could not allocate slots: %w", err) } } } @@ -350,7 +354,7 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { func (ps *PeerSet) allocSlots(setIdx int) error { err := ps.updateTime() if err != nil { - return err + return fmt.Errorf("cannot update time: %w", err) } peerState := ps.peerState @@ -366,7 +370,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { var node *node node, err = ps.peerState.getNode(reservePeer) if err != nil { - return err + return fmt.Errorf("cannot get node using peer id %s: %w", reservePeer, err) } if node.rep < BannedThresholdValue { @@ -376,7 +380,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { } if err = peerState.tryOutgoing(setIdx, reservePeer); err != nil { - return err + return fmt.Errorf("cannot set peer %s from set %d as outgoing: %w", reservePeer, setIdx, err) } connectMessage := Message{ @@ -466,7 +470,7 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { if ps.peerState.peerStatus(setID, peerID) == connectedPeer { err := ps.peerState.disconnect(setID, peerID) if err != nil { - return err + return fmt.Errorf("cannot disconnect peer %s at set %d: %w", peerID, setID, err) } dropMessage := Message{ @@ -483,8 +487,11 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { } func (ps *PeerSet) setReservedPeer(setID int, peers ...peer.ID) error { - toInsert, toRemove := make([]peer.ID, 0, len(peers)), make([]peer.ID, 0, len(peers)) + toInsert := make([]peer.ID, 0, len(peers)) + toRemove := make([]peer.ID, 0, len(peers)) + peerIDMap := make(map[peer.ID]struct{}, len(peers)) + for _, pid := range peers { peerIDMap[pid] = struct{}{} if _, ok := ps.reservedNode[pid]; ok { @@ -515,7 +522,7 @@ func (ps *PeerSet) addPeer(setID int, peers peer.IDSlice) error { ps.peerState.discover(setID, pid) if err := ps.allocSlots(setID); err != nil { - return err + return fmt.Errorf("could not allocate slots: %w", err) } } return nil @@ -540,15 +547,15 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { // disconnect and forget err := ps.peerState.disconnect(setID, pid) if err != nil { - return err + return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, setID, err) } if err = ps.peerState.forgetPeer(setID, pid); err != nil { - return err + return fmt.Errorf("cannot forget peer %s from set %d: %w", pid, setID, err) } } else if status == notConnectedPeer { if err := ps.peerState.forgetPeer(setID, pid); err != nil { - return err + return fmt.Errorf("cannot forget peer %s from set %d: %w", pid, setID, err) } } } @@ -560,7 +567,7 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { // connected to this peer. func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { if err := ps.updateTime(); err != nil { - return err + return fmt.Errorf("cannot update time: %w", err) } for _, pid := range peers { @@ -649,7 +656,7 @@ const ( func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) error { err := ps.updateTime() if err != nil { - return err + return fmt.Errorf("cannot update time: %w", err) } state := ps.peerState @@ -664,7 +671,7 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e state.nodes[pid] = n if err = state.disconnect(setIdx, pid); err != nil { - return err + return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, setIdx, err) } dropMessage := Message{ @@ -678,7 +685,7 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e // TODO: figure out the condition of connection refuse. if reason == RefusedDrop { if err = ps.removePeer(setIdx, pid); err != nil { - return err + return fmt.Errorf("cannot remove peer %s at set %d: %w", pid, setIdx, err) } } } From 78a5468d5aac1aa75bf82581fe41f4d5a8b32190 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 11 Mar 2022 17:50:49 -0400 Subject: [PATCH 19/50] chore: remove unneeded comment --- dot/peerset/peerset.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 71b7fbad3e..70704626dc 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -572,7 +572,6 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { for _, pid := range peers { if ps.isReservedOnly { - // This is for reserved only mode. if _, ok := ps.reservedNode[pid]; !ok { rejectMessage := Message{ Status: Reject, From 50ef6cb1a13c87c89b31e27a12befa5f64e3b857 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 14 Mar 2022 13:12:59 -0400 Subject: [PATCH 20/50] chore: implement MessageProcessor interface at network/host --- devnet/alice.Dockerfile | 6 ++--- devnet/bob.Dockerfile | 9 ++++---- devnet/docker-compose.yml | 8 ++----- devnet/substrate_bob.Dockerfile | 8 +++---- dot/network/host.go | 36 +++++++++++++++++++++++++++++ dot/network/service.go | 40 ++------------------------------- dot/network/state.go | 3 ++- dot/peerset/handler.go | 8 +++++-- dot/peerset/peerset.go | 3 +-- dot/peerset/test_helpers.go | 2 +- 10 files changed, 62 insertions(+), 61 deletions(-) diff --git a/devnet/alice.Dockerfile b/devnet/alice.Dockerfile index f0a026a31b..62e5cc1ce2 100644 --- a/devnet/alice.Dockerfile +++ b/devnet/alice.Dockerfile @@ -11,7 +11,7 @@ ARG DD_API_KEY=somekey ENV DD_API_KEY=${DD_API_KEY} -RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" +#RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" WORKDIR /gossamer @@ -34,10 +34,10 @@ ARG METRICS_NAMESPACE=gossamer.local.devnet WORKDIR /gossamer/devnet -RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml +#RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml WORKDIR /gossamer -ENTRYPOINT service datadog-agent start && gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubdns=alice --port 7001 +ENTRYPOINT gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubdns=alice --port 7001 --log-network debug EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp 6060/tcp diff --git a/devnet/bob.Dockerfile b/devnet/bob.Dockerfile index 973f25e5be..1858a3a79a 100644 --- a/devnet/bob.Dockerfile +++ b/devnet/bob.Dockerfile @@ -12,7 +12,7 @@ ARG DD_API_KEY=somekey ENV DD_API_KEY=${DD_API_KEY} -RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" +#RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" WORKDIR /gossamer @@ -36,15 +36,16 @@ ARG METRICS_NAMESPACE=gossamer.local.devnet WORKDIR /gossamer/devnet -RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml +#RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml WORKDIR /gossamer -ENTRYPOINT service datadog-agent start && gossamer --key=${key} \ +ENTRYPOINT gossamer --key=${key} \ --bootnodes=/dns/alice/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU \ --publish-metrics \ --rpc \ --port 7001 \ - --pubdns=${key} + --pubdns=${key} \ + --log-network debug EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp 6060/tcp diff --git a/devnet/docker-compose.yml b/devnet/docker-compose.yml index 1e268bbf0e..65f372c969 100644 --- a/devnet/docker-compose.yml +++ b/devnet/docker-compose.yml @@ -18,7 +18,7 @@ services: platform: linux/amd64 build: context: ./.. - dockerfile: devnet/bob.Dockerfile + dockerfile: devnet/substrate_bob.Dockerfile args: DD_API_KEY: ${DD_API_KEY} key: bob @@ -29,14 +29,12 @@ services: - 8540 - 9876 - 6060 - depends_on: - - alice charlie: platform: linux/amd64 build: context: ./.. - dockerfile: devnet/bob.Dockerfile + dockerfile: devnet/substrate_bob.Dockerfile args: DD_API_KEY: ${DD_API_KEY} key: charlie @@ -46,5 +44,3 @@ services: - 8546 - 8540 - 9876 - depends_on: - - alice diff --git a/devnet/substrate_bob.Dockerfile b/devnet/substrate_bob.Dockerfile index e9847f97ca..aba2c949a8 100644 --- a/devnet/substrate_bob.Dockerfile +++ b/devnet/substrate_bob.Dockerfile @@ -33,17 +33,17 @@ RUN apt update && apt install -y curl && rm -r /var/cache/* /var/lib/apt/lists/* WORKDIR /cross-client -RUN curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh --output install_script.sh && \ - chmod +x ./install_script.sh +#RUN curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh --output install_script.sh && \ +# chmod +x ./install_script.sh -RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" ./install_script.sh +#RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" ./install_script.sh COPY --from=openmetrics /devnet/conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/ USER polkadot COPY ./devnet/chain ./chain/ -ENTRYPOINT service datadog-agent start && /usr/bin/polkadot \ +ENTRYPOINT /usr/bin/polkadot \ --bootnodes /dns/alice/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU \ --chain chain/$CHAIN/genesis-raw.json \ --port 7001 \ diff --git a/dot/network/host.go b/dot/network/host.go index 1dbdda5d9c..b2e6f40127 100644 --- a/dot/network/host.go +++ b/dot/network/host.go @@ -223,6 +223,7 @@ func newHost(ctx context.Context, cfg *Config) (*host, error) { } cm.host = host + cm.peerSetHandler.SetMessageProcessor(host) return host, nil } @@ -437,3 +438,38 @@ func (h *host) closeProtocolStream(pID protocol.ID, p peer.ID) { } } } + +// Process will connect, drop or reject a peer based on a peerset message +func (h *host) Process(msg peerset.Message) { + peerID := msg.PeerID + if peerID == "" { + logger.Errorf("found empty peer id in peerset message") + return + } + switch msg.Status { + case peerset.Connect: + addrInfo := h.h.Peerstore().PeerInfo(peerID) + if len(addrInfo.Addrs) == 0 { + var err error + addrInfo, err = h.discovery.findPeer(peerID) + if err != nil { + logger.Warnf("failed to find peer id %s: %s", peerID, err) + return + } + } + + err := h.connect(addrInfo) + if err != nil { + logger.Warnf("failed to open connection for peer %s: %s", peerID, err) + return + } + logger.Debugf("connection successful with peer %s", peerID) + case peerset.Drop, peerset.Reject: + err := h.closePeer(peerID) + if err != nil { + logger.Warnf("failed to close connection with peer %s: %s", peerID, err) + return + } + logger.Debugf("connection dropped successfully for peer %s", peerID) + } +} diff --git a/dot/network/service.go b/dot/network/service.go index f7275e1f16..3f0e89ee7c 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -135,8 +135,7 @@ type Service struct { blockResponseBuf []byte blockResponseBufMu sync.Mutex - - telemetry telemetry.Client + telemetry telemetry.Client } // NewService creates a new network service from the configuration and message channels @@ -674,44 +673,9 @@ func (s *Service) ReportPeer(change peerset.ReputationChange, p peer.ID) { } func (s *Service) startPeerSetHandler() { - s.host.cm.peerSetHandler.Start(s.ctx, s) + s.host.cm.peerSetHandler.Start(s.ctx) // wait for peerSetHandler to start. if !s.noBootstrap { s.host.bootstrap() } } - -// Process will connect, drop or reject a peer based on a peerset message -func (s *Service) Process(msg peerset.Message) { - peerID := msg.PeerID - if peerID == "" { - logger.Errorf("found empty peer id in peerset message") - return - } - switch msg.Status { - case peerset.Connect: - addrInfo := s.host.h.Peerstore().PeerInfo(peerID) - if len(addrInfo.Addrs) == 0 { - var err error - addrInfo, err = s.host.discovery.findPeer(peerID) - if err != nil { - logger.Warnf("failed to find peer id %s: %s", peerID, err) - return - } - } - - err := s.host.connect(addrInfo) - if err != nil { - logger.Warnf("failed to open connection for peer %s: %s", peerID, err) - return - } - logger.Debugf("connection successful with peer %s", peerID) - case peerset.Drop, peerset.Reject: - err := s.host.closePeer(peerID) - if err != nil { - logger.Warnf("failed to close connection with peer %s: %s", peerID, err) - return - } - logger.Debugf("connection dropped successfully for peer %s", peerID) - } -} diff --git a/dot/network/state.go b/dot/network/state.go index f309181173..8cf32945da 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -46,7 +46,8 @@ type TransactionHandler interface { // PeerSetHandler is the interface used by the connection manager to handle peerset. type PeerSetHandler interface { - Start(context.Context, peerset.MessageProcessor) + SetMessageProcessor(peerset.MessageProcessor) + Start(context.Context) ReportPeer(peerset.ReputationChange, ...peer.ID) PeerAdd PeerRemove diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index acfa660dcd..3ed600be2d 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -118,9 +118,13 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { return n.rep, nil } +func (h *Handler) SetMessageProcessor(processor MessageProcessor) { + h.peerSet.processor = processor +} + // Start starts peerSet processing -func (h *Handler) Start(ctx context.Context, processor MessageProcessor) { - h.peerSet.start(ctx, processor) +func (h *Handler) Start(ctx context.Context) { + h.peerSet.start(ctx) } // SortedPeers returns a sorted peer ID slice for connected peers in the peerSet. diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 70704626dc..6a5fd3b548 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -693,8 +693,7 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e } // start handles all the action for the peerSet. -func (ps *PeerSet) start(ctx context.Context, processor MessageProcessor) { - ps.processor = processor +func (ps *PeerSet) start(ctx context.Context) { go ps.doWork(ctx) } diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index 42028d44c0..e648a396eb 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -44,7 +44,7 @@ func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, handler, err := NewPeerSetHandler(con) require.NoError(t, err) - handler.Start(context.Background(), processor) + handler.Start(context.Background()) handler.AddPeer(0, bootNodes...) handler.AddReservedPeer(0, reservedPeers...) From 4f285eeb3c85548a3ac7a52922f09cd0843715e6 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 14 Mar 2022 13:19:45 -0400 Subject: [PATCH 21/50] chore: rename `doWork` to `periodicallyAllocateSlots` --- devnet/alice.Dockerfile | 6 +++--- devnet/bob.Dockerfile | 9 ++++----- devnet/docker-compose.yml | 8 ++++++-- devnet/substrate_bob.Dockerfile | 8 ++++---- dot/peerset/handler.go | 2 +- dot/peerset/peerset.go | 12 ++++++------ dot/peerset/peerstate.go | 16 ++++++++-------- dot/peerset/peerstate_test.go | 14 +++++++------- 8 files changed, 39 insertions(+), 36 deletions(-) diff --git a/devnet/alice.Dockerfile b/devnet/alice.Dockerfile index 62e5cc1ce2..f0a026a31b 100644 --- a/devnet/alice.Dockerfile +++ b/devnet/alice.Dockerfile @@ -11,7 +11,7 @@ ARG DD_API_KEY=somekey ENV DD_API_KEY=${DD_API_KEY} -#RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" +RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" WORKDIR /gossamer @@ -34,10 +34,10 @@ ARG METRICS_NAMESPACE=gossamer.local.devnet WORKDIR /gossamer/devnet -#RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml +RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml WORKDIR /gossamer -ENTRYPOINT gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubdns=alice --port 7001 --log-network debug +ENTRYPOINT service datadog-agent start && gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubdns=alice --port 7001 EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp 6060/tcp diff --git a/devnet/bob.Dockerfile b/devnet/bob.Dockerfile index 1858a3a79a..973f25e5be 100644 --- a/devnet/bob.Dockerfile +++ b/devnet/bob.Dockerfile @@ -12,7 +12,7 @@ ARG DD_API_KEY=somekey ENV DD_API_KEY=${DD_API_KEY} -#RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" +RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)" WORKDIR /gossamer @@ -36,16 +36,15 @@ ARG METRICS_NAMESPACE=gossamer.local.devnet WORKDIR /gossamer/devnet -#RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml +RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml WORKDIR /gossamer -ENTRYPOINT gossamer --key=${key} \ +ENTRYPOINT service datadog-agent start && gossamer --key=${key} \ --bootnodes=/dns/alice/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU \ --publish-metrics \ --rpc \ --port 7001 \ - --pubdns=${key} \ - --log-network debug + --pubdns=${key} EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp 6060/tcp diff --git a/devnet/docker-compose.yml b/devnet/docker-compose.yml index 65f372c969..1e268bbf0e 100644 --- a/devnet/docker-compose.yml +++ b/devnet/docker-compose.yml @@ -18,7 +18,7 @@ services: platform: linux/amd64 build: context: ./.. - dockerfile: devnet/substrate_bob.Dockerfile + dockerfile: devnet/bob.Dockerfile args: DD_API_KEY: ${DD_API_KEY} key: bob @@ -29,12 +29,14 @@ services: - 8540 - 9876 - 6060 + depends_on: + - alice charlie: platform: linux/amd64 build: context: ./.. - dockerfile: devnet/substrate_bob.Dockerfile + dockerfile: devnet/bob.Dockerfile args: DD_API_KEY: ${DD_API_KEY} key: charlie @@ -44,3 +46,5 @@ services: - 8546 - 8540 - 9876 + depends_on: + - alice diff --git a/devnet/substrate_bob.Dockerfile b/devnet/substrate_bob.Dockerfile index aba2c949a8..e9847f97ca 100644 --- a/devnet/substrate_bob.Dockerfile +++ b/devnet/substrate_bob.Dockerfile @@ -33,17 +33,17 @@ RUN apt update && apt install -y curl && rm -r /var/cache/* /var/lib/apt/lists/* WORKDIR /cross-client -#RUN curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh --output install_script.sh && \ -# chmod +x ./install_script.sh +RUN curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh --output install_script.sh && \ + chmod +x ./install_script.sh -#RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" ./install_script.sh +RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" ./install_script.sh COPY --from=openmetrics /devnet/conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/ USER polkadot COPY ./devnet/chain ./chain/ -ENTRYPOINT /usr/bin/polkadot \ +ENTRYPOINT service datadog-agent start && /usr/bin/polkadot \ --bootnodes /dns/alice/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU \ --chain chain/$CHAIN/genesis-raw.json \ --port 7001 \ diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 3ed600be2d..2ae588e019 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -115,7 +115,7 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { if err != nil { return 0, err } - return n.rep, nil + return n.reputation, nil } func (h *Handler) SetMessageProcessor(processor MessageProcessor) { diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 6a5fd3b548..970814d6d7 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -373,9 +373,9 @@ func (ps *PeerSet) allocSlots(setIdx int) error { return fmt.Errorf("cannot get node using peer id %s: %w", reservePeer, err) } - if node.rep < BannedThresholdValue { + if node.reputation < BannedThresholdValue { logger.Warnf("reputation is lower than banned threshold value, reputation: %d, banned threshold value: %d", - node.rep, BannedThresholdValue) + node.reputation, BannedThresholdValue) break } @@ -404,7 +404,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { } n := peerState.nodes[peerID] - if n.rep < BannedThresholdValue { + if n.reputation < BannedThresholdValue { logger.Critical("highest rated peer is below bannedThresholdValue") break } @@ -601,7 +601,7 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { state.RLock() node, has := state.nodes[pid] if has { - nodeReputation = node.rep + nodeReputation = node.reputation } state.RUnlock() @@ -694,10 +694,10 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e // start handles all the action for the peerSet. func (ps *PeerSet) start(ctx context.Context) { - go ps.doWork(ctx) + go ps.periodicallyAllocateSlots(ctx) } -func (ps *PeerSet) doWork(ctx context.Context) { +func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { ticker := time.NewTicker(ps.nextPeriodicAllocSlots) defer ticker.Stop() diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index 44ef1b3f95..c691555b20 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -69,7 +69,7 @@ type node struct { lastConnected []time.Time // Reputation of the node, between int32 MIN and int32 MAX. - rep Reputation + reputation Reputation } // newNode creates a node with n number of sets and 0 reputation. @@ -89,8 +89,8 @@ func newNode(n int) *node { } func (n *node) addReputation(modifier Reputation) Reputation { - n.rep = n.rep.add(modifier) - return n.rep + n.reputation = n.reputation.add(modifier) + return n.reputation } // PeersState struct contains a list of nodes, where each node @@ -202,7 +202,7 @@ func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { if isPeerConnected(state) { connectedPeersReps = append(connectedPeersReps, connectedPeerReputation{ peerID: peerID, - reputation: node.rep, + reputation: node.reputation, }) } } @@ -228,9 +228,9 @@ func (ps *PeersState) updateReputationByTick(peerID peer.ID) (after Reputation, return 0, ErrPeerDoesNotExist } - after = reputationTick(node.rep) + after = reputationTick(node.reputation) - node.rep = after + node.reputation = after ps.nodes[peerID] = node return after, nil @@ -266,7 +266,7 @@ func (ps *PeersState) highestNotConnectedPeer(set int) peer.ID { continue } - val := int(node.rep) + val := int(node.reputation) if val >= maxRep { maxRep = val higestPeerID = peerID @@ -419,7 +419,7 @@ func (ps *PeersState) forgetPeer(set int, peerID peer.ID) error { node.state[set] = notMember } - if node.rep != 0 { + if node.reputation != 0 { return nil } diff --git a/dot/peerset/peerstate_test.go b/dot/peerset/peerstate_test.go index 3263d16566..bb0395a776 100644 --- a/dot/peerset/peerstate_test.go +++ b/dot/peerset/peerstate_test.go @@ -151,33 +151,33 @@ func TestHighestNotConnectedPeer(t *testing.T) { n, err := state.getNode(peer1) require.NoError(t, err) - n.rep = 50 + n.reputation = 50 state.nodes[peer1] = n - require.Equal(t, Reputation(50), state.nodes[peer1].rep) + require.Equal(t, Reputation(50), state.nodes[peer1].reputation) require.Equal(t, unknownPeer, state.peerStatus(0, peer2)) state.discover(0, peer2) n, err = state.getNode(peer2) require.NoError(t, err) - n.rep = 25 + n.reputation = 25 state.nodes[peer2] = n // peer1 still has the highest reputation require.Equal(t, peer1, state.highestNotConnectedPeer(0)) - require.Equal(t, Reputation(25), state.nodes[peer2].rep) + require.Equal(t, Reputation(25), state.nodes[peer2].reputation) require.Equal(t, notConnectedPeer, state.peerStatus(0, peer2)) n, err = state.getNode(peer2) require.NoError(t, err) - n.rep = 75 + n.reputation = 75 state.nodes[peer2] = n require.Equal(t, peer2, state.highestNotConnectedPeer(0)) - require.Equal(t, Reputation(75), state.nodes[peer2].rep) + require.Equal(t, Reputation(75), state.nodes[peer2].reputation) require.Equal(t, notConnectedPeer, state.peerStatus(0, peer2)) err = state.tryAcceptIncoming(0, peer2) @@ -192,7 +192,7 @@ func TestHighestNotConnectedPeer(t *testing.T) { require.Equal(t, notConnectedPeer, state.peerStatus(0, peer1)) n, err = state.getNode(peer1) require.NoError(t, err) - n.rep = 100 + n.reputation = 100 state.nodes[peer1] = n require.Equal(t, peer1, state.highestNotConnectedPeer(0)) From 237cfe9f2d0ede7dedbbe457a5cdc31468c157f1 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 14 Mar 2022 14:17:49 -0400 Subject: [PATCH 22/50] chore: set processor at testing --- dot/peerset/handler.go | 1 + dot/peerset/test_helpers.go | 1 + 2 files changed, 2 insertions(+) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 2ae588e019..0e465dd0be 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -118,6 +118,7 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { return n.reputation, nil } +// SetMessageProcessor set a processor to process peerset messages func (h *Handler) SetMessageProcessor(processor MessageProcessor) { h.peerSet.processor = processor } diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index e648a396eb..04798ba6fa 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -44,6 +44,7 @@ func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, handler, err := NewPeerSetHandler(con) require.NoError(t, err) + handler.SetMessageProcessor(processor) handler.Start(context.Background()) handler.AddPeer(0, bootNodes...) From d02fafce3360819d60065b3463f0779529f0f559 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 14 Mar 2022 19:06:24 -0400 Subject: [PATCH 23/50] chore: address comments --- dot/network/connmgr_test.go | 10 +++---- dot/network/discovery_test.go | 4 +-- dot/network/host.go | 46 +++++++++++++++---------------- dot/network/host_test.go | 12 ++++---- dot/network/light_test.go | 2 +- dot/network/mdns.go | 4 +-- dot/network/mdns_test.go | 4 +-- dot/network/notifications_test.go | 14 +++++----- dot/network/service.go | 8 +++--- dot/network/service_test.go | 6 ++-- dot/network/sync.go | 6 ++-- dot/peerset/handler.go | 2 +- dot/peerset/peerset.go | 23 ++++++---------- 13 files changed, 67 insertions(+), 74 deletions(-) diff --git a/dot/network/connmgr_test.go b/dot/network/connmgr_test.go index ac2c8ff5c9..1ed6001816 100644 --- a/dot/network/connmgr_test.go +++ b/dot/network/connmgr_test.go @@ -82,12 +82,12 @@ func TestMaxPeers(t *testing.T) { continue } - n.host.h.Peerstore().AddAddrs(ainfo.ID, ainfo.Addrs, peerstore.PermanentAddrTTL) + n.host.p2pHost.Peerstore().AddAddrs(ainfo.ID, ainfo.Addrs, peerstore.PermanentAddrTTL) n.host.cm.peerSetHandler.AddPeer(0, ainfo.ID) } time.Sleep(200 * time.Millisecond) - p := nodes[0].host.h.Peerstore().Peers() + p := nodes[0].host.p2pHost.Peerstore().Peers() require.LessOrEqual(t, max, len(p)) } @@ -152,7 +152,7 @@ func TestPersistentPeers(t *testing.T) { time.Sleep(time.Millisecond * 600) // B should have connected to A during bootstrap - conns := nodeB.host.h.Network().ConnsToPeer(nodeA.host.id()) + conns := nodeB.host.p2pHost.Network().ConnsToPeer(nodeA.host.id()) require.NotEqual(t, 0, len(conns)) // if A disconnects from B, B should reconnect @@ -160,7 +160,7 @@ func TestPersistentPeers(t *testing.T) { time.Sleep(time.Millisecond * 500) - conns = nodeB.host.h.Network().ConnsToPeer(nodeA.host.id()) + conns = nodeB.host.p2pHost.Network().ConnsToPeer(nodeA.host.id()) require.NotEqual(t, 0, len(conns)) } @@ -239,7 +239,7 @@ func TestSetReservedPeer(t *testing.T) { require.Equal(t, 2, node3.host.peerCount()) - node3.host.h.Peerstore().AddAddrs(addrC.ID, addrC.Addrs, peerstore.PermanentAddrTTL) + node3.host.p2pHost.Peerstore().AddAddrs(addrC.ID, addrC.Addrs, peerstore.PermanentAddrTTL) node3.host.cm.peerSetHandler.SetReservedPeer(0, addrC.ID) time.Sleep(200 * time.Millisecond) diff --git a/dot/network/discovery_test.go b/dot/network/discovery_test.go index 3d8b2278ff..9858d9e428 100644 --- a/dot/network/discovery_test.go +++ b/dot/network/discovery_test.go @@ -35,7 +35,7 @@ func newTestDiscovery(t *testing.T, num int) []*discovery { require.NoError(t, err) disc := &discovery{ ctx: srvc.ctx, - h: srvc.host.h, + h: srvc.host.p2pHost, ds: ds, } @@ -200,7 +200,7 @@ func TestBeginDiscovery_ThreeNodes(t *testing.T) { time.Sleep(time.Millisecond * 500) // assert B and C can discover each other - addrs := nodeB.host.h.Peerstore().Addrs(nodeC.host.id()) + addrs := nodeB.host.p2pHost.Peerstore().Addrs(nodeC.host.id()) require.NotEqual(t, 0, len(addrs)) } diff --git a/dot/network/host.go b/dot/network/host.go index b2e6f40127..c998fd2eb4 100644 --- a/dot/network/host.go +++ b/dot/network/host.go @@ -68,7 +68,7 @@ const ( // host wraps libp2p host with network host configuration and services type host struct { ctx context.Context - h libp2phost.Host + p2pHost libp2phost.Host discovery *discovery bootnodes []peer.AddrInfo persistentPeers []peer.AddrInfo @@ -211,7 +211,7 @@ func newHost(ctx context.Context, cfg *Config) (*host, error) { host := &host{ ctx: ctx, - h: h, + p2pHost: h, discovery: discovery, bootnodes: bns, protocolID: pid, @@ -237,14 +237,14 @@ func (h *host) close() error { } // close libp2p host - err = h.h.Close() + err = h.p2pHost.Close() if err != nil { logger.Errorf("Failed to close libp2p host: %s", err) return err } h.closeSync.Do(func() { - err = h.h.Peerstore().Close() + err = h.p2pHost.Peerstore().Close() if err != nil { logger.Errorf("Failed to close libp2p peerstore: %s", err) return @@ -261,28 +261,28 @@ func (h *host) close() error { // registerStreamHandler registers the stream handler for the given protocol id. func (h *host) registerStreamHandler(pid protocol.ID, handler func(libp2pnetwork.Stream)) { - h.h.SetStreamHandler(pid, handler) + h.p2pHost.SetStreamHandler(pid, handler) } // connect connects the host to a specific peer address func (h *host) connect(p peer.AddrInfo) (err error) { - h.h.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) + h.p2pHost.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) ctx, cancel := context.WithTimeout(h.ctx, connectTimeout) defer cancel() - err = h.h.Connect(ctx, p) + err = h.p2pHost.Connect(ctx, p) return err } // bootstrap connects the host to the configured bootnodes func (h *host) bootstrap() { for _, info := range h.persistentPeers { - h.h.Peerstore().AddAddrs(info.ID, info.Addrs, peerstore.PermanentAddrTTL) + h.p2pHost.Peerstore().AddAddrs(info.ID, info.Addrs, peerstore.PermanentAddrTTL) h.cm.peerSetHandler.AddReservedPeer(0, info.ID) } for _, addrInfo := range h.bootnodes { logger.Debugf("bootstrapping to peer %s", addrInfo.ID) - h.h.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL) + h.p2pHost.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL) h.cm.peerSetHandler.AddPeer(0, addrInfo.ID) } } @@ -291,7 +291,7 @@ func (h *host) bootstrap() { // the newly created stream. func (h *host) send(p peer.ID, pid protocol.ID, msg Message) (libp2pnetwork.Stream, error) { // open outbound stream with host protocol id - stream, err := h.h.NewStream(h.ctx, p, pid) + stream, err := h.p2pHost.NewStream(h.ctx, p, pid) if err != nil { logger.Tracef("failed to open new stream with peer %s using protocol %s: %s", p, pid, err) return nil, err @@ -335,12 +335,12 @@ func (h *host) writeToStream(s libp2pnetwork.Stream, msg Message) error { // id returns the host id func (h *host) id() peer.ID { - return h.h.ID() + return h.p2pHost.ID() } // Peers returns connected peers func (h *host) peers() []peer.ID { - return h.h.Network().Peers() + return h.p2pHost.Network().Peers() } // addReservedPeers adds the peers `addrs` to the protected peers list and connects to them @@ -355,7 +355,7 @@ func (h *host) addReservedPeers(addrs ...string) error { if err != nil { return err } - h.h.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL) + h.p2pHost.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL) h.cm.peerSetHandler.AddReservedPeer(0, addrInfo.ID) } @@ -370,7 +370,7 @@ func (h *host) removeReservedPeers(ids ...string) error { return err } h.cm.peerSetHandler.RemoveReservedPeer(0, peerID) - h.h.ConnManager().Unprotect(peerID, "") + h.p2pHost.ConnManager().Unprotect(peerID, "") } return nil @@ -379,7 +379,7 @@ func (h *host) removeReservedPeers(ids ...string) error { // supportsProtocol checks if the protocol is supported by peerID // returns an error if could not get peer protocols func (h *host) supportsProtocol(peerID peer.ID, protocol protocol.ID) (bool, error) { - peerProtocols, err := h.h.Peerstore().SupportsProtocols(peerID, string(protocol)) + peerProtocols, err := h.p2pHost.Peerstore().SupportsProtocols(peerID, string(protocol)) if err != nil { return false, err } @@ -389,21 +389,21 @@ func (h *host) supportsProtocol(peerID peer.ID, protocol protocol.ID) (bool, err // peerCount returns the number of connected peers func (h *host) peerCount() int { - peers := h.h.Network().Peers() + peers := h.p2pHost.Network().Peers() return len(peers) } // addrInfo returns the libp2p peer.AddrInfo of the host func (h *host) addrInfo() peer.AddrInfo { return peer.AddrInfo{ - ID: h.h.ID(), - Addrs: h.h.Addrs(), + ID: h.p2pHost.ID(), + Addrs: h.p2pHost.Addrs(), } } // multiaddrs returns the multiaddresses of the host func (h *host) multiaddrs() (multiaddrs []ma.Multiaddr) { - addrs := h.h.Addrs() + addrs := h.p2pHost.Addrs() for _, addr := range addrs { multiaddr, err := ma.NewMultiaddr(fmt.Sprintf("%s/p2p/%s", addr, h.id())) if err != nil { @@ -416,16 +416,16 @@ func (h *host) multiaddrs() (multiaddrs []ma.Multiaddr) { // protocols returns all protocols currently supported by the node func (h *host) protocols() []string { - return h.h.Mux().Protocols() + return h.p2pHost.Mux().Protocols() } // closePeer closes connection with peer. func (h *host) closePeer(peer peer.ID) error { - return h.h.Network().ClosePeer(peer) + return h.p2pHost.Network().ClosePeer(peer) } func (h *host) closeProtocolStream(pID protocol.ID, p peer.ID) { - connToPeer := h.h.Network().ConnsToPeer(p) + connToPeer := h.p2pHost.Network().ConnsToPeer(p) for _, c := range connToPeer { for _, st := range c.GetStreams() { if st.Protocol() != pID { @@ -448,7 +448,7 @@ func (h *host) Process(msg peerset.Message) { } switch msg.Status { case peerset.Connect: - addrInfo := h.h.Peerstore().PeerInfo(peerID) + addrInfo := h.p2pHost.Peerstore().PeerInfo(peerID) if len(addrInfo.Addrs) == 0 { var err error addrInfo, err = h.discovery.findPeer(peerID) diff --git a/dot/network/host_test.go b/dot/network/host_test.go index 1250b15181..baa812a9c4 100644 --- a/dot/network/host_test.go +++ b/dot/network/host_test.go @@ -170,13 +170,13 @@ func TestBootstrap(t *testing.T) { peerCountA := nodeA.host.peerCount() if peerCountA == 0 { - peerCountA := len(nodeA.host.h.Peerstore().Peers()) + peerCountA := len(nodeA.host.p2pHost.Peerstore().Peers()) require.NotZero(t, peerCountA) } peerCountB := nodeB.host.peerCount() if peerCountB == 0 { - peerCountB := len(nodeB.host.h.Peerstore().Peers()) + peerCountB := len(nodeB.host.p2pHost.Peerstore().Peers()) require.NotZero(t, peerCountB) } } @@ -498,7 +498,7 @@ func Test_RemoveReservedPeers(t *testing.T) { time.Sleep(100 * time.Millisecond) require.Equal(t, 1, nodeA.host.peerCount()) - isProtected := nodeA.host.h.ConnManager().IsProtected(nodeB.host.addrInfo().ID, "") + isProtected := nodeA.host.p2pHost.ConnManager().IsProtected(nodeB.host.addrInfo().ID, "") require.False(t, isProtected) err = nodeA.host.removeReservedPeers("unknown_perr_id") @@ -583,7 +583,7 @@ func TestPeerConnect(t *testing.T) { nodeB.noGossip = true addrInfoB := nodeB.host.addrInfo() - nodeA.host.h.Peerstore().AddAddrs(addrInfoB.ID, addrInfoB.Addrs, peerstore.PermanentAddrTTL) + nodeA.host.p2pHost.Peerstore().AddAddrs(addrInfoB.ID, addrInfoB.Addrs, peerstore.PermanentAddrTTL) nodeA.host.cm.peerSetHandler.AddPeer(0, addrInfoB.ID) time.Sleep(100 * time.Millisecond) @@ -621,7 +621,7 @@ func TestBannedPeer(t *testing.T) { nodeB.noGossip = true addrInfoB := nodeB.host.addrInfo() - nodeA.host.h.Peerstore().AddAddrs(addrInfoB.ID, addrInfoB.Addrs, peerstore.PermanentAddrTTL) + nodeA.host.p2pHost.Peerstore().AddAddrs(addrInfoB.ID, addrInfoB.Addrs, peerstore.PermanentAddrTTL) nodeA.host.cm.peerSetHandler.AddPeer(0, addrInfoB.ID) time.Sleep(100 * time.Millisecond) @@ -674,7 +674,7 @@ func TestPeerReputation(t *testing.T) { nodeB.noGossip = true addrInfoB := nodeB.host.addrInfo() - nodeA.host.h.Peerstore().AddAddrs(addrInfoB.ID, addrInfoB.Addrs, peerstore.PermanentAddrTTL) + nodeA.host.p2pHost.Peerstore().AddAddrs(addrInfoB.ID, addrInfoB.Addrs, peerstore.PermanentAddrTTL) nodeA.host.cm.peerSetHandler.AddPeer(0, addrInfoB.ID) time.Sleep(100 * time.Millisecond) diff --git a/dot/network/light_test.go b/dot/network/light_test.go index 85f26ffd40..7cbc037921 100644 --- a/dot/network/light_test.go +++ b/dot/network/light_test.go @@ -113,7 +113,7 @@ func TestHandleLightMessage_Response(t *testing.T) { } require.NoError(t, err) - stream, err := s.host.h.NewStream(s.ctx, b.host.id(), s.host.protocolID+lightID) + stream, err := s.host.p2pHost.NewStream(s.ctx, b.host.id(), s.host.protocolID+lightID) require.NoError(t, err) // Testing empty request diff --git a/dot/network/mdns.go b/dot/network/mdns.go index fc329218f1..04635087fe 100644 --- a/dot/network/mdns.go +++ b/dot/network/mdns.go @@ -47,7 +47,7 @@ func (m *mdns) start() { // create and start service mdns, err := libp2pdiscovery.NewMdnsService( m.host.ctx, - m.host.h, + m.host.p2pHost, MDNSPeriod, string(m.host.protocolID), ) @@ -89,7 +89,7 @@ func (n Notifee) HandlePeerFound(p peer.AddrInfo) { "Peer %s found using mDNS discovery, with host %s", p.ID, n.host.id()) - n.host.h.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) + n.host.p2pHost.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) // connect to found peer n.host.cm.peerSetHandler.AddPeer(0, p.ID) } diff --git a/dot/network/mdns_test.go b/dot/network/mdns_test.go index 901a57622f..82ce9dd87e 100644 --- a/dot/network/mdns_test.go +++ b/dot/network/mdns_test.go @@ -42,13 +42,13 @@ func TestMDNS(t *testing.T) { if peerCountA == 0 { // check peerstore for disconnected peers - peerCountA := len(nodeA.host.h.Peerstore().Peers()) + peerCountA := len(nodeA.host.p2pHost.Peerstore().Peers()) require.NotZero(t, peerCountA) } if peerCountB == 0 { // check peerstore for disconnected peers - peerCountB := len(nodeB.host.h.Peerstore().Peers()) + peerCountB := len(nodeB.host.p2pHost.Peerstore().Peers()) require.NotZero(t, peerCountB) } } diff --git a/dot/network/notifications_test.go b/dot/network/notifications_test.go index a62f121e4b..224db70c46 100644 --- a/dot/network/notifications_test.go +++ b/dot/network/notifications_test.go @@ -112,7 +112,7 @@ func TestCreateNotificationsMessageHandler_BlockAnnounce(t *testing.T) { } require.NoError(t, err) - stream, err := s.host.h.NewStream(s.ctx, b.host.id(), s.host.protocolID+blockAnnounceID) + stream, err := s.host.p2pHost.NewStream(s.ctx, b.host.id(), s.host.protocolID+blockAnnounceID) require.NoError(t, err) // create info and handler @@ -181,7 +181,7 @@ func TestCreateNotificationsMessageHandler_BlockAnnounceHandshake(t *testing.T) } require.NoError(t, err) - stream, err := s.host.h.NewStream(s.ctx, b.host.id(), s.host.protocolID+blockAnnounceID) + stream, err := s.host.p2pHost.NewStream(s.ctx, b.host.id(), s.host.protocolID+blockAnnounceID) require.NoError(t, err) // try invalid handshake @@ -248,7 +248,7 @@ func Test_HandshakeTimeout(t *testing.T) { info := newNotificationsProtocol(nodeA.host.protocolID+blockAnnounceID, nodeA.getBlockAnnounceHandshake, testHandshakeDecoder, nodeA.validateBlockAnnounceHandshake) - nodeB.host.h.SetStreamHandler(info.protocolID, func(stream libp2pnetwork.Stream) { + nodeB.host.p2pHost.SetStreamHandler(info.protocolID, func(stream libp2pnetwork.Stream) { // should not respond to a handshake message }) @@ -265,7 +265,7 @@ func Test_HandshakeTimeout(t *testing.T) { // clear handshake data from connection handler time.Sleep(time.Millisecond * 100) info.peersData.deleteOutboundHandshakeData(nodeB.host.id()) - connAToB := nodeA.host.h.Network().ConnsToPeer(nodeB.host.id()) + connAToB := nodeA.host.p2pHost.Network().ConnsToPeer(nodeB.host.id()) for _, stream := range connAToB[0].GetStreams() { _ = stream.Close() } @@ -287,7 +287,7 @@ func Test_HandshakeTimeout(t *testing.T) { require.Nil(t, data) // a stream should be open until timeout - connAToB = nodeA.host.h.Network().ConnsToPeer(nodeB.host.id()) + connAToB = nodeA.host.p2pHost.Network().ConnsToPeer(nodeB.host.id()) require.Len(t, connAToB, 1) require.Len(t, connAToB[0].GetStreams(), 1) @@ -299,7 +299,7 @@ func Test_HandshakeTimeout(t *testing.T) { require.Nil(t, data) // stream should be closed - connAToB = nodeA.host.h.Network().ConnsToPeer(nodeB.host.id()) + connAToB = nodeA.host.p2pHost.Network().ConnsToPeer(nodeB.host.id()) require.Len(t, connAToB, 1) require.Len(t, connAToB[0].GetStreams(), 0) } @@ -341,7 +341,7 @@ func TestCreateNotificationsMessageHandler_HandleTransaction(t *testing.T) { require.NoError(t, err) txnProtocolID := srvc1.host.protocolID + transactionsID - stream, err := srvc1.host.h.NewStream(srvc1.ctx, srvc2.host.id(), txnProtocolID) + stream, err := srvc1.host.p2pHost.NewStream(srvc1.ctx, srvc2.host.id(), txnProtocolID) require.NoError(t, err) // create info and handler diff --git a/dot/network/service.go b/dot/network/service.go index 3f0e89ee7c..11647a5095 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -277,7 +277,7 @@ func (s *Service) Start() error { // since this opens block announce streams, it should happen after the protocol is registered // NOTE: this only handles *incoming* connections - s.host.h.Network().SetConnHandler(s.handleConn) + s.host.p2pHost.Network().SetConnHandler(s.handleConn) // this handles all new connections (incoming and outgoing) // it creates a per-protocol mutex for sending outbound handshakes to the peer @@ -341,9 +341,9 @@ func (s *Service) updateMetrics() { return case <-ticker.C: peerCountGauge.Set(float64(s.host.peerCount())) - connectionsGauge.Set(float64(len(s.host.h.Network().Conns()))) + connectionsGauge.Set(float64(len(s.host.p2pHost.Network().Conns()))) nodeLatencyGauge.Set(float64( - s.host.h.Peerstore().LatencyEWMA(s.host.id()).Milliseconds())) + s.host.p2pHost.Peerstore().LatencyEWMA(s.host.id()).Milliseconds())) inboundBlockAnnounceStreamsGauge.Set(float64( s.getNumStreams(BlockAnnounceMsgType, true))) outboundBlockAnnounceStreamsGauge.Set(float64( @@ -357,7 +357,7 @@ func (s *Service) updateMetrics() { } func (s *Service) getTotalStreams(inbound bool) (count int64) { - for _, conn := range s.host.h.Network().Conns() { + for _, conn := range s.host.p2pHost.Network().Conns() { for _, stream := range conn.GetStreams() { streamIsInbound := isInbound(stream) if (streamIsInbound && inbound) || (!streamIsInbound && !inbound) { diff --git a/dot/network/service_test.go b/dot/network/service_test.go index ea6e925398..5877f3d464 100644 --- a/dot/network/service_test.go +++ b/dot/network/service_test.go @@ -269,7 +269,7 @@ func TestBroadcastDuplicateMessage(t *testing.T) { } require.NoError(t, err) - stream, err := nodeA.host.h.NewStream(context.Background(), nodeB.host.id(), nodeB.host.protocolID+blockAnnounceID) + stream, err := nodeA.host.p2pHost.NewStream(context.Background(), nodeB.host.id(), nodeB.host.protocolID+blockAnnounceID) require.NoError(t, err) require.NotNil(t, stream) @@ -361,7 +361,7 @@ func TestPersistPeerStore(t *testing.T) { } require.NoError(t, err) - require.NotEmpty(t, nodeA.host.h.Peerstore().PeerInfo(nodeB.host.id()).Addrs) + require.NotEmpty(t, nodeA.host.p2pHost.Peerstore().PeerInfo(nodeB.host.id()).Addrs) // Stop a node and reinitialise a new node with same base path. err = nodeA.Stop() @@ -369,7 +369,7 @@ func TestPersistPeerStore(t *testing.T) { // Since nodeAA uses the persistent peerstore of nodeA, it should be have nodeB in it's peerstore. nodeAA := createTestService(t, nodeA.cfg) - require.NotEmpty(t, nodeAA.host.h.Peerstore().PeerInfo(nodeB.host.id()).Addrs) + require.NotEmpty(t, nodeAA.host.p2pHost.Peerstore().PeerInfo(nodeB.host.id()).Addrs) } func TestHandleConn(t *testing.T) { diff --git a/dot/network/sync.go b/dot/network/sync.go index d21e7a189a..856447254e 100644 --- a/dot/network/sync.go +++ b/dot/network/sync.go @@ -24,13 +24,13 @@ var ( func (s *Service) DoBlockRequest(to peer.ID, req *BlockRequestMessage) (*BlockResponseMessage, error) { fullSyncID := s.host.protocolID + syncID - s.host.h.ConnManager().Protect(to, "") - defer s.host.h.ConnManager().Unprotect(to, "") + s.host.p2pHost.ConnManager().Protect(to, "") + defer s.host.p2pHost.ConnManager().Unprotect(to, "") ctx, cancel := context.WithTimeout(s.ctx, blockRequestTimeout) defer cancel() - stream, err := s.host.h.NewStream(ctx, to, fullSyncID) + stream, err := s.host.p2pHost.NewStream(ctx, to, fullSyncID) if err != nil { return nil, err } diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 0e465dd0be..fa45fda855 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -125,7 +125,7 @@ func (h *Handler) SetMessageProcessor(processor MessageProcessor) { // Start starts peerSet processing func (h *Handler) Start(ctx context.Context) { - h.peerSet.start(ctx) + go h.peerSet.periodicallyAllocateSlots(ctx) } // SortedPeers returns a sorted peer ID slice for connected peers in the peerSet. diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 970814d6d7..eba7712195 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -367,8 +367,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { peerState.discover(setIdx, reservePeer) } - var node *node - node, err = ps.peerState.getNode(reservePeer) + node, err := ps.peerState.getNode(reservePeer) if err != nil { return fmt.Errorf("cannot get node using peer id %s: %w", reservePeer, err) } @@ -605,34 +604,33 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { } state.RUnlock() + var message Message + switch { case nodeReputation < BannedThresholdValue: - rejectMessage := Message{ + message = Message{ Status: Reject, setID: uint64(setID), PeerID: pid, } - ps.processor.Process(rejectMessage) - case state.tryAcceptIncoming(setID, pid) != nil: - rejectMessage := Message{ + message = Message{ Status: Reject, setID: uint64(setID), PeerID: pid, } - ps.processor.Process(rejectMessage) default: logger.Debugf("incoming connection accepted from peer %s", pid) - acceptMessage := Message{ + message = Message{ Status: Accept, setID: uint64(setID), PeerID: pid, } - - ps.processor.Process(acceptMessage) } + + ps.processor.Process(message) } return nil @@ -692,11 +690,6 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e return ps.allocSlots(setIdx) } -// start handles all the action for the peerSet. -func (ps *PeerSet) start(ctx context.Context) { - go ps.periodicallyAllocateSlots(ctx) -} - func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { ticker := time.NewTicker(ps.nextPeriodicAllocSlots) defer ticker.Stop() From a4adf71621e75209ced4baa52486179ba42e8b21 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 15 Mar 2022 09:13:24 -0400 Subject: [PATCH 24/50] chore: address comments --- dot/peerset/peerset_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index e159ad9a32..47eccd6248 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -131,8 +131,6 @@ func TestReAllocAfterBanned(t *testing.T) { // Check that an incoming connection from that node gets refused. processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: peer1}) handler.Incoming(0, peer1) - time.Sleep(time.Second * 2) - } func TestRemovePeer(t *testing.T) { From 1b49a64a2e86629137f1f77e40fdbe53181264db Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 15 Mar 2022 09:35:33 -0400 Subject: [PATCH 25/50] chore: wrap `ErrPeerDoesNotExist` --- dot/peerset/peerstate.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index c691555b20..5eb5e9b9b1 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -112,7 +112,7 @@ func (ps *PeersState) getNode(p peer.ID) (*node, error) { return n, nil } - return nil, ErrPeerDoesNotExist + return nil, fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, p) } // NewPeerState initiates a new PeersState @@ -194,7 +194,7 @@ func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { reputation Reputation } - connectedPeersReps := make([]connectedPeerReputation, 0) + connectedPeersReps := make([]connectedPeerReputation, 0, len(ps.nodes)) for peerID, node := range ps.nodes { state := node.state[idx] @@ -219,21 +219,21 @@ func (ps *PeersState) sortedPeers(idx int) peer.IDSlice { return peerIDs } -func (ps *PeersState) updateReputationByTick(peerID peer.ID) (after Reputation, err error) { +func (ps *PeersState) updateReputationByTick(peerID peer.ID) (newReputation Reputation, err error) { ps.Lock() defer ps.Unlock() node, has := ps.nodes[peerID] if !has { - return 0, ErrPeerDoesNotExist + return 0, fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } - after = reputationTick(node.reputation) + newReputation = reputationTick(node.reputation) - node.reputation = after + node.reputation = newReputation ps.nodes[peerID] = node - return after, nil + return newReputation, nil } func (ps *PeersState) addReputation(peerID peer.ID, change ReputationChange) ( @@ -244,7 +244,7 @@ func (ps *PeersState) addReputation(peerID peer.ID, change ReputationChange) ( node, has := ps.nodes[peerID] if !has { - return 0, ErrPeerDoesNotExist + return 0, fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } newReputation = node.addReputation(change.Value) @@ -302,7 +302,7 @@ func (ps *PeersState) addNoSlotNode(idx int, peerID peer.ID) error { node, has := ps.nodes[peerID] if !has { - return fmt.Errorf("could not get node for peer id %s: %w", peerID, ErrPeerDoesNotExist) + return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } switch node.state[idx] { @@ -328,7 +328,7 @@ func (ps *PeersState) removeNoSlotNode(idx int, peerID peer.ID) error { node, has := ps.nodes[peerID] if !has { - return fmt.Errorf("could not get node for peer id %s: %w", peerID, ErrPeerDoesNotExist) + return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } switch node.state[idx] { @@ -350,7 +350,7 @@ func (ps *PeersState) disconnect(idx int, peerID peer.ID) error { info := ps.sets[idx] node, has := ps.nodes[peerID] if !has { - return ErrPeerDoesNotExist + return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } _, has = info.noSlotNodes[peerID] @@ -395,7 +395,7 @@ func (ps *PeersState) lastConnectedAndDiscovered(set int, peerID peer.ID) (time. node, has := ps.nodes[peerID] if !has { - return time.Time{}, ErrPeerDoesNotExist + return time.Time{}, fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } if node.state[set] == notConnected { @@ -412,7 +412,7 @@ func (ps *PeersState) forgetPeer(set int, peerID peer.ID) error { node, has := ps.nodes[peerID] if !has { - return ErrPeerDoesNotExist + return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } if node.state[set] != notMember { @@ -455,7 +455,7 @@ func (ps *PeersState) tryOutgoing(setID int, peerID peer.ID) error { node, has := ps.nodes[peerID] if !has { - return ErrPeerDoesNotExist + return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } node.state[setID] = outgoing @@ -487,7 +487,7 @@ func (ps *PeersState) tryAcceptIncoming(setID int, peerID peer.ID) error { node, has := ps.nodes[peerID] if !has { // state inconsistency tryOutgoing on an unknown node - return ErrPeerDoesNotExist + return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } node.state[setID] = ingoing From 9d4c81afc2c1a72fa3deb94b104c9674b5dd03a5 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 15 Mar 2022 15:23:06 -0400 Subject: [PATCH 26/50] chore: addressing comments --- dot/network/service.go | 9 ++------- dot/network/state.go | 1 + dot/peerset/peerset.go | 6 ++++-- dot/peerset/peerstate_test.go | 30 ------------------------------ 4 files changed, 7 insertions(+), 39 deletions(-) diff --git a/dot/network/service.go b/dot/network/service.go index 11647a5095..367ec98c77 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -488,19 +488,14 @@ func (s *Service) Stop() error { } // check if closeCh is closed, if not, close it. -mainloop: for { select { - case _, hasMore := <-s.closeCh: - if !hasMore { - break mainloop - } + case <-s.closeCh: + return nil default: close(s.closeCh) } } - - return nil } // RegisterNotificationsProtocol registers a protocol with the network service with the given handler diff --git a/dot/network/state.go b/dot/network/state.go index 8cf32945da..9b4fe42e10 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -71,5 +71,6 @@ type PeerRemove interface { // Peer is the interface used by the PeerSetHandler to get the peer data from peerSet. type Peer interface { + SortedPeers(idx int) chan peer.IDSlice PeerReputation(peer.ID) (peerset.Reputation, error) } diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index eba7712195..52b0b3ae9f 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -308,7 +308,8 @@ func (ps *PeerSet) updateTime() error { // be disconnected and a drop message for the peer is sent in order to disconnect. func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { // we want reputations to be up-to-date before adjusting them. - if err := ps.updateTime(); err != nil { + err := ps.updateTime() + if err != nil { return fmt.Errorf("cannot update time: %w", err) } @@ -565,7 +566,8 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { // either with a corresponding `Accept` or `Reject`, except if we were already // connected to this peer. func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { - if err := ps.updateTime(); err != nil { + err := ps.updateTime() + if err != nil { return fmt.Errorf("cannot update time: %w", err) } diff --git a/dot/peerset/peerstate_test.go b/dot/peerset/peerstate_test.go index bb0395a776..21405024b4 100644 --- a/dot/peerset/peerstate_test.go +++ b/dot/peerset/peerstate_test.go @@ -197,33 +197,3 @@ func TestHighestNotConnectedPeer(t *testing.T) { require.Equal(t, peer1, state.highestNotConnectedPeer(0)) } - -func TestSortedPeers(t *testing.T) { - t.Parallel() - - const msgChanSize = 1 - state := newTestPeerState(t, 2, 1) - state.nodes[peer1] = newNode(1) - - err := state.addNoSlotNode(0, peer1) - require.NoError(t, err) - - state.discover(0, peer1) - err = state.tryAcceptIncoming(0, peer1) - require.NoError(t, err) - - require.Equal(t, connectedPeer, state.peerStatus(0, peer1)) - - // discover peer2 - state.discover(0, peer2) - // try to make peer2 as an incoming connection. - err = state.tryAcceptIncoming(0, peer2) - require.NoError(t, err) - - require.Equal(t, connectedPeer, state.peerStatus(0, peer1)) - - peerCh := make(chan peer.IDSlice, msgChanSize) - peerCh <- state.sortedPeers(0) - peers := <-peerCh - require.Equal(t, 2, len(peers)) -} From af4380e38edaef0063c64fbe63b7da60d4c59e6e Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 16 Mar 2022 08:41:26 -0400 Subject: [PATCH 27/50] chore: fix PeerStateHandler interface --- dot/network/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/network/state.go b/dot/network/state.go index 9b4fe42e10..f9181554c3 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -71,6 +71,6 @@ type PeerRemove interface { // Peer is the interface used by the PeerSetHandler to get the peer data from peerSet. type Peer interface { - SortedPeers(idx int) chan peer.IDSlice + SortedPeers(idx int) peer.IDSlice PeerReputation(peer.ID) (peerset.Reputation, error) } From c184ccf28e88a1d008e43f9e854ad2850bacd00d Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 16 Mar 2022 09:48:43 -0400 Subject: [PATCH 28/50] chore: improve `Test_Ban_Reject_Accept_Peer` assertions --- dot/peerset/peerset.go | 4 +-- dot/peerset/peerset_test.go | 53 ++++++++++++++++++++++--------------- dot/peerset/test_helpers.go | 4 ++- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 52b0b3ae9f..45003e888e 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -702,8 +702,8 @@ func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { // TODO: log context error? return case <-ticker.C: - l := ps.peerState.getSetLength() - for i := 0; i < l; i++ { + setsAmount := ps.peerState.getSetLength() + for i := 0; i < setsAmount; i++ { if err := ps.allocSlots(i); err != nil { logger.Warnf("failed to do action on peerSet: %s", err) } diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index 47eccd6248..56c9f62e23 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -20,30 +20,45 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { handler := newTestPeerSet(t, 25, 25, nil, nil, false, processor) + const setID = 0 ps := handler.peerSet - require.Equal(t, unknownPeer, ps.peerState.peerStatus(0, peer1)) - ps.peerState.discover(0, peer1) + + require.Equal(t, uint32(0), ps.peerState.sets[setID].numIn) + require.Equal(t, unknownPeer, ps.peerState.peerStatus(setID, peer1)) + + ps.peerState.discover(setID, peer1) // adding peer1 with incoming slot. - err := ps.peerState.tryAcceptIncoming(0, peer1) + err := ps.peerState.tryAcceptIncoming(setID, peer1) require.NoError(t, err) + require.Equal(t, uint32(1), ps.peerState.sets[setID].numIn) + require.Equal(t, connectedPeer, ps.peerState.peerStatus(setID, peer1)) + // we ban a node by setting its reputation under the threshold. rpc := newReputationChange(BannedThresholdValue-1, "") // we need one for the message to be processed. - processor.EXPECT().Process(Message{Status: Drop, setID: 0x0, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Drop, setID: setID, PeerID: "testPeer1"}) + // report peer will disconnect the peer and set the `lastConnected` to time.Now handler.ReportPeer(rpc, peer1) - time.Sleep(time.Millisecond * 100) + require.Equal(t, uint32(0), ps.peerState.sets[setID].numIn) + require.Equal(t, notConnectedPeer, ps.peerState.peerStatus(setID, peer1)) + + disconectedAt := ps.peerState.nodes[peer1].lastConnected // check that an incoming connection from that node gets refused. - processor.EXPECT().Process(Message{Status: Reject, setID: 0x0, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Reject, setID: setID, PeerID: "testPeer1"}) + // incoming should update the lastConnected time handler.Incoming(0, peer1) + triedToConnectAt := ps.peerState.nodes[peer1].lastConnected + require.Greater(t, triedToConnectAt, disconectedAt) + // wait a bit for the node's reputation to go above the threshold. time.Sleep(time.Millisecond * 1200) // try again. This time the node should be accepted. - processor.EXPECT().Process(Message{Status: Accept, setID: 0x0, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Accept, setID: setID, PeerID: "testPeer1"}) handler.Incoming(0, peer1) } @@ -62,8 +77,6 @@ func TestAddReservedPeers(t *testing.T) { handler.AddReservedPeer(0, reservedPeer) handler.AddReservedPeer(0, reservedPeer2) - time.Sleep(time.Millisecond * 200) - require.Equal(t, uint32(1), ps.peerState.sets[0].numOut) } @@ -112,25 +125,24 @@ func TestReAllocAfterBanned(t *testing.T) { handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false, processor) ps := handler.peerSet - // adding peer1 with incoming slot. - if ps.peerState.peerStatus(0, peer1) == unknownPeer { - ps.peerState.discover(0, peer1) - err := ps.peerState.tryAcceptIncoming(0, peer1) - require.NoError(t, err) - } + require.Equal(t, unknownPeer, ps.peerState.peerStatus(0, peer1)) + + ps.peerState.discover(0, peer1) + err := ps.peerState.tryAcceptIncoming(0, peer1) + require.NoError(t, err) // We ban a node by setting its reputation under the threshold. processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: peer1}) rep := newReputationChange(BannedThresholdValue-1, "") - - // we need one for the message to be processed. - processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: peer1}) handler.ReportPeer(rep, peer1) - time.Sleep(time.Millisecond * 100) // Check that an incoming connection from that node gets refused. - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: peer1}) + processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: peer1}) handler.Incoming(0, peer1) + + // wait a bit for the node's reputation to go above the threshold. + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: peer1}) + <-time.After(allocTimeDuration + time.Second) } func TestRemovePeer(t *testing.T) { @@ -145,7 +157,6 @@ func TestRemovePeer(t *testing.T) { nil, false, processor) ps := handler.peerSet - time.Sleep(time.Millisecond * 500) processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index 04798ba6fa..c2a0e1bcbd 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -25,6 +25,8 @@ const ( peer2 = peer.ID("testPeer2") ) +const allocTimeDuration = time.Second * 2 + //go:generate mockgen -destination=mock_message_processor_test.go -package $GOPACKAGE . MessageProcessor func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, @@ -36,7 +38,7 @@ func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, maxInPeers: in, maxOutPeers: out, reservedOnly: reservedOnly, - periodicAllocTime: time.Second * 2, + periodicAllocTime: allocTimeDuration, }, }, } From 5fe401c6d1c948184ea38d1f1628ddc422ed5b6e Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 17 Mar 2022 15:42:27 -0400 Subject: [PATCH 29/50] chore: improve the peerset test assertions --- dot/peerset/peerset.go | 45 ++++---- dot/peerset/peerset_test.go | 205 ++++++++++++++++++++++++++++-------- dot/peerset/peerstate.go | 13 ++- dot/peerset/test_helpers.go | 6 +- 4 files changed, 200 insertions(+), 69 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 45003e888e..66666c856c 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -507,11 +507,17 @@ func (ps *PeerSet) setReservedPeer(setID int, peers ...peer.ID) error { toRemove = append(toRemove, pid) } - if err := ps.addReservedPeers(setID, toInsert...); err != nil { - return err + err := ps.addReservedPeers(setID, toInsert...) + if err != nil { + return fmt.Errorf("cannot add reserved peers: %w", err) + } + + err = ps.removeReservedPeers(setID, toRemove...) + if err != nil { + return fmt.Errorf("cannot remove reserved peers: %w", err) } - return ps.removeReservedPeers(setID, toRemove...) + return nil } func (ps *PeerSet) addPeer(setID int, peers peer.IDSlice) error { @@ -615,20 +621,23 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { setID: uint64(setID), PeerID: pid, } - - case state.tryAcceptIncoming(setID, pid) != nil: - message = Message{ - Status: Reject, - setID: uint64(setID), - PeerID: pid, - } - default: - logger.Debugf("incoming connection accepted from peer %s", pid) - message = Message{ - Status: Accept, - setID: uint64(setID), - PeerID: pid, + err := state.tryAcceptIncoming(setID, pid) + if err != nil { + logger.Errorf("cannot accept incomming peer %pid: %w", pid, err) + + message = Message{ + Status: Reject, + setID: uint64(setID), + PeerID: pid, + } + } else { + logger.Debugf("incoming connection accepted from peer %s", pid) + message = Message{ + Status: Accept, + setID: uint64(setID), + PeerID: pid, + } } } @@ -702,8 +711,8 @@ func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { // TODO: log context error? return case <-ticker.C: - setsAmount := ps.peerState.getSetLength() - for i := 0; i < setsAmount; i++ { + setLen := ps.peerState.getSetLength() + for i := 0; i < setLen; i++ { if err := ps.allocSlots(i); err != nil { logger.Warnf("failed to do action on peerSet: %s", err) } diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index 56c9f62e23..ab605b5aa1 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -12,6 +12,8 @@ import ( "github.com/stretchr/testify/require" ) +const testSetID = 0 + func Test_Ban_Reject_Accept_Peer(t *testing.T) { t.Parallel() @@ -20,34 +22,33 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { handler := newTestPeerSet(t, 25, 25, nil, nil, false, processor) - const setID = 0 ps := handler.peerSet - require.Equal(t, uint32(0), ps.peerState.sets[setID].numIn) - require.Equal(t, unknownPeer, ps.peerState.peerStatus(setID, peer1)) + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + require.Equal(t, unknownPeer, ps.peerState.peerStatus(testSetID, peer1)) - ps.peerState.discover(setID, peer1) + ps.peerState.discover(testSetID, peer1) // adding peer1 with incoming slot. - err := ps.peerState.tryAcceptIncoming(setID, peer1) + err := ps.peerState.tryAcceptIncoming(testSetID, peer1) require.NoError(t, err) - require.Equal(t, uint32(1), ps.peerState.sets[setID].numIn) - require.Equal(t, connectedPeer, ps.peerState.peerStatus(setID, peer1)) + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numIn) + require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, peer1)) // we ban a node by setting its reputation under the threshold. rpc := newReputationChange(BannedThresholdValue-1, "") // we need one for the message to be processed. - processor.EXPECT().Process(Message{Status: Drop, setID: setID, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: "testPeer1"}) // report peer will disconnect the peer and set the `lastConnected` to time.Now handler.ReportPeer(rpc, peer1) - require.Equal(t, uint32(0), ps.peerState.sets[setID].numIn) - require.Equal(t, notConnectedPeer, ps.peerState.peerStatus(setID, peer1)) + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + require.Equal(t, notConnectedPeer, ps.peerState.peerStatus(testSetID, peer1)) disconectedAt := ps.peerState.nodes[peer1].lastConnected // check that an incoming connection from that node gets refused. - processor.EXPECT().Process(Message{Status: Reject, setID: setID, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Reject, setID: testSetID, PeerID: "testPeer1"}) // incoming should update the lastConnected time handler.Incoming(0, peer1) @@ -58,7 +59,7 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { time.Sleep(time.Millisecond * 1200) // try again. This time the node should be accepted. - processor.EXPECT().Process(Message{Status: Accept, setID: setID, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Accept, setID: testSetID, PeerID: "testPeer1"}) handler.Incoming(0, peer1) } @@ -67,17 +68,49 @@ func TestAddReservedPeers(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: bootNode}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: bootNode}) handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false, processor) ps := handler.peerSet - handler.AddReservedPeer(0, reservedPeer) - handler.AddReservedPeer(0, reservedPeer2) + _, exists := ps.peerState.nodes[bootNode] + require.True(t, exists) + + require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, bootNode)) + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + + reservedPeers := []struct { + peerID peer.ID + }{ + { + peerID: reservedPeer, + }, + { + peerID: reservedPeer2, + }, + } + + for _, tt := range reservedPeers { + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: tt.peerID}) + handler.AddReservedPeer(testSetID, tt.peerID) - require.Equal(t, uint32(1), ps.peerState.sets[0].numOut) + _, exists = ps.reservedNode[tt.peerID] + require.True(t, exists) + + _, exists = ps.peerState.sets[testSetID].noSlotNodes[tt.peerID] + require.True(t, exists) + + node, exists := ps.peerState.nodes[tt.peerID] + require.True(t, exists) + require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, tt.peerID)) + require.Equal(t, outgoing, node.state[testSetID]) + + // peers in noSlotNodes maps should not increase the + // numIn and numOut count + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + } } func TestPeerSetIncoming(t *testing.T) { @@ -86,18 +119,63 @@ func TestPeerSetIncoming(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: bootNode}) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: bootNode}) handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, []peer.ID{}, false, processor) - processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incomingPeer}) - handler.Incoming(0, incomingPeer) + ps := handler.peerSet + + require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, bootNode)) + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + + incomingPeers := []struct { + pid peer.ID + expectedStatus Status + expectedNumIn uint32 + // hasFreeIncomingSlot indicates the next slots + // are only available to noSlotNodes nodes + hasFreeIncomingSlot bool + }{ + { + pid: incomingPeer, + expectedStatus: Accept, + expectedNumIn: 1, + hasFreeIncomingSlot: false, + }, + { + pid: incoming2, + expectedStatus: Accept, + expectedNumIn: 2, + hasFreeIncomingSlot: true, + }, + { + pid: incoming3, + expectedStatus: Reject, + expectedNumIn: 2, + hasFreeIncomingSlot: true, + }, + } + + for _, tt := range incomingPeers { + processor.EXPECT().Process(Message{Status: tt.expectedStatus, setID: testSetID, PeerID: tt.pid}) - processor.EXPECT().Process(Message{Status: Accept, setID: 0, PeerID: incoming2}) - handler.Incoming(0, incoming2) + // all the incoming peers are unknow before calling the Incoming method + status := ps.peerState.peerStatus(testSetID, tt.pid) + require.Equal(t, unknownPeer, status) - processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: incoming3}) - handler.Incoming(0, incoming3) + handler.Incoming(testSetID, tt.pid) + + _, exists := ps.peerState.nodes[tt.pid] + require.True(t, exists) + + freeSlots := ps.peerState.hasFreeIncomingSlot(testSetID) + require.Equal(t, tt.hasFreeIncomingSlot, freeSlots) + + require.Equal(t, tt.expectedNumIn, ps.peerState.sets[testSetID].numIn) + // incoming peers should not chang the numOut count + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + } } func TestPeerSetDiscovered(t *testing.T) { @@ -109,12 +187,33 @@ func TestPeerSetDiscovered(t *testing.T) { processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processor) + ps := handler.peerSet + + _, isReservedNode := ps.reservedNode[reservedPeer] + require.True(t, isReservedNode) + + _, isNoSlotNode := ps.peerState.sets[testSetID].noSlotNodes[reservedPeer] + require.True(t, isNoSlotNode) + + // reserved nodes should not increase the numOut count + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numOut) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered1}) handler.AddPeer(0, discovered1) handler.AddPeer(0, discovered1) + _, exitsts := ps.peerState.nodes[discovered1] + require.True(t, exitsts) + + // AddPeer called twice with the same peer ID should not increase the numOut count + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered2}) handler.AddPeer(0, discovered2) + + _, exitsts = ps.peerState.nodes[discovered2] + require.True(t, exitsts) + require.Equal(t, uint32(2), ps.peerState.sets[testSetID].numOut) } func TestReAllocAfterBanned(t *testing.T) { @@ -125,24 +224,44 @@ func TestReAllocAfterBanned(t *testing.T) { handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false, processor) ps := handler.peerSet - require.Equal(t, unknownPeer, ps.peerState.peerStatus(0, peer1)) + require.Equal(t, unknownPeer, ps.peerState.peerStatus(testSetID, peer1)) - ps.peerState.discover(0, peer1) - err := ps.peerState.tryAcceptIncoming(0, peer1) + ps.peerState.discover(testSetID, peer1) + err := ps.peerState.tryAcceptIncoming(testSetID, peer1) require.NoError(t, err) + // accepting the income peer which is not in the reserved peers + // should increase the numIn count by 1 + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numIn) + // We ban a node by setting its reputation under the threshold. - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: peer1}) + processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: peer1}) rep := newReputationChange(BannedThresholdValue-1, "") handler.ReportPeer(rep, peer1) + // banning a incoming peer should decrease the numIn count by 1 + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + node := ps.peerState.nodes[peer1] + require.Equal(t, notConnected, node.state[testSetID]) + + // when the peer1 was banned we updated its lastConnected field to time.Now() + lastTimeConnected := node.lastConnected[testSetID] + // Check that an incoming connection from that node gets refused. - processor.EXPECT().Process(Message{Status: Reject, setID: 0, PeerID: peer1}) - handler.Incoming(0, peer1) + processor.EXPECT().Process(Message{Status: Reject, setID: testSetID, PeerID: peer1}) + handler.Incoming(testSetID, peer1) + + // when calling Incoming method the peer1 is with status notConnectedPeer + // so we update its lastConnected field to time.Now() + node = ps.peerState.nodes[peer1] + currentLastTimeConnected := node.lastConnected[testSetID] + require.True(t, lastTimeConnected.Before(currentLastTimeConnected)) // wait a bit for the node's reputation to go above the threshold. - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: peer1}) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: peer1}) <-time.After(allocTimeDuration + time.Second) + + require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) } func TestRemovePeer(t *testing.T) { @@ -151,18 +270,21 @@ func TestRemovePeer(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "testDiscovered1"}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "testDiscovered2"}) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: "testDiscovered1"}) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: "testDiscovered2"}) handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, nil, false, processor) ps := handler.peerSet + require.Equal(t, 2, len(ps.peerState.nodes)) + require.Equal(t, uint32(2), ps.peerState.sets[testSetID].numOut) processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) - handler.RemovePeer(0, discovered1, discovered2) + handler.RemovePeer(testSetID, discovered1, discovered2) require.Equal(t, 0, len(ps.peerState.nodes)) + require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numOut) } func TestSetReservePeer(t *testing.T) { @@ -171,20 +293,21 @@ func TestSetReservePeer(t *testing.T) { ctrl := gomock.NewController(t) processor := NewMockMessageProcessor(ctrl) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer2}) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: reservedPeer}) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: reservedPeer2}) handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, true, processor) ps := handler.peerSet + require.Equal(t, 2, len(ps.reservedNode)) newRsrPeerSet := peer.IDSlice{reservedPeer, peer.ID("newRsrPeer")} // add newRsrPeer but remove reservedPeer2 - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: "newRsrPeer"}) - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: reservedPeer2}) - handler.SetReservedPeer(0, newRsrPeerSet...) + processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: "newRsrPeer"}) + processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: reservedPeer2}) + handler.SetReservedPeer(testSetID, newRsrPeerSet...) - require.Equal(t, len(newRsrPeerSet), len(ps.reservedNode)) + require.Equal(t, 2, len(ps.reservedNode)) for _, p := range newRsrPeerSet { require.Contains(t, ps.reservedNode, p) } diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index 5eb5e9b9b1..4b60da1eb5 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -259,7 +259,7 @@ func (ps *PeersState) highestNotConnectedPeer(set int) peer.ID { defer ps.RUnlock() maxRep := math.MinInt32 - var higestPeerID peer.ID + var highestPeerID peer.ID for peerID, node := range ps.nodes { if node.state[set] != notConnected { @@ -269,11 +269,11 @@ func (ps *PeersState) highestNotConnectedPeer(set int) peer.ID { val := int(node.reputation) if val >= maxRep { maxRep = val - higestPeerID = peerID + highestPeerID = peerID } } - return higestPeerID + return highestPeerID } func (ps *PeersState) hasFreeOutgoingSlot(set int) bool { @@ -302,6 +302,7 @@ func (ps *PeersState) addNoSlotNode(idx int, peerID peer.ID) error { node, has := ps.nodes[peerID] if !has { + fmt.Println("error here") return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } @@ -459,6 +460,7 @@ func (ps *PeersState) tryOutgoing(setID int, peerID peer.ID) error { } node.state[setID] = outgoing + if !isNoSlotNode { ps.sets[setID].numOut++ } @@ -474,10 +476,7 @@ func (ps *PeersState) tryAcceptIncoming(setID int, peerID peer.ID) error { ps.Lock() defer ps.Unlock() - var isNoSlotOccupied bool - if _, ok := ps.sets[setID].noSlotNodes[peerID]; ok { - isNoSlotOccupied = true - } + _, isNoSlotOccupied := ps.sets[setID].noSlotNodes[peerID] // if slot is not available and the node is not a reserved node then error if ps.hasFreeIncomingSlot(setID) && !isNoSlotOccupied { diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index c2a0e1bcbd..f8383a9bcf 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -29,14 +29,14 @@ const allocTimeDuration = time.Second * 2 //go:generate mockgen -destination=mock_message_processor_test.go -package $GOPACKAGE . MessageProcessor -func newTestPeerSet(t *testing.T, in, out uint32, bootNodes, +func newTestPeerSet(t *testing.T, maxIn, maxOut uint32, bootNodes, reservedPeers []peer.ID, reservedOnly bool, processor MessageProcessor) *Handler { t.Helper() con := &ConfigSet{ Set: []*config{ { - maxInPeers: in, - maxOutPeers: out, + maxInPeers: maxIn, + maxOutPeers: maxOut, reservedOnly: reservedOnly, periodicAllocTime: allocTimeDuration, }, From ca7f6768d74694430bc453f40361ed08781693aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Thu, 17 Mar 2022 15:49:59 -0400 Subject: [PATCH 30/50] Update dot/network/state.go Co-authored-by: Quentin McGaw --- dot/network/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/network/state.go b/dot/network/state.go index f9181554c3..5027566a11 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -71,6 +71,6 @@ type PeerRemove interface { // Peer is the interface used by the PeerSetHandler to get the peer data from peerSet. type Peer interface { - SortedPeers(idx int) peer.IDSlice PeerReputation(peer.ID) (peerset.Reputation, error) + SortedPeers(idx int) peer.IDSlice } From 60791f1e14ac4d89601167f79d0a36755ef32974 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 17 Mar 2022 15:56:27 -0400 Subject: [PATCH 31/50] chore: fix lll lint issue --- dot/network/service_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dot/network/service_test.go b/dot/network/service_test.go index 5877f3d464..a358ec6c3d 100644 --- a/dot/network/service_test.go +++ b/dot/network/service_test.go @@ -269,7 +269,8 @@ func TestBroadcastDuplicateMessage(t *testing.T) { } require.NoError(t, err) - stream, err := nodeA.host.p2pHost.NewStream(context.Background(), nodeB.host.id(), nodeB.host.protocolID+blockAnnounceID) + stream, err := nodeA.host.p2pHost.NewStream(context.Background(), + nodeB.host.id(), nodeB.host.protocolID+blockAnnounceID) require.NoError(t, err) require.NotNil(t, stream) From 4856eb27772fbd2f4e5c338fb78e1cb5fe9fc9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Thu, 17 Mar 2022 15:57:40 -0400 Subject: [PATCH 32/50] Update dot/peerset/handler.go Co-authored-by: Quentin McGaw --- dot/peerset/handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index fa45fda855..ba35981af5 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -118,7 +118,8 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { return n.reputation, nil } -// SetMessageProcessor set a processor to process peerset messages +// SetMessageProcessor sets the peerset processor of the handler +// to process peerset messages. func (h *Handler) SetMessageProcessor(processor MessageProcessor) { h.peerSet.processor = processor } From ad060fe56fa2627587626e0e4a0fde63b9eab8b4 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 17 Mar 2022 17:03:18 -0400 Subject: [PATCH 33/50] chore: implementing the reject case in the incoming peer connections --- dot/network/service.go | 26 +++++++++++++++++++++- dot/network/state.go | 2 +- dot/peerset/handler.go | 9 +++----- dot/peerset/peerset.go | 43 ++++++++++++++++--------------------- dot/peerset/test_helpers.go | 4 ++-- 5 files changed, 50 insertions(+), 34 deletions(-) diff --git a/dot/network/service.go b/dot/network/service.go index 367ec98c77..8311eafdd1 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -439,7 +439,31 @@ func (s *Service) sentBlockIntervalTelemetry() { func (s *Service) handleConn(conn libp2pnetwork.Conn) { // TODO: currently we only have one set so setID is 0, change this once we have more set in peerSet. - s.host.cm.peerSetHandler.Incoming(0, conn.RemotePeer()) + results, err := s.host.cm.peerSetHandler.Incoming(0, conn.RemotePeer()) + if err != nil { + logger.Errorf("cannot accept incoming peer: %w", err) + err := conn.Close() + if err != nil { + logger.Errorf("cannot close connection with peer: %w", err) + } + return + } + + if len(results) == 0 { + logger.Errorf("got an empty incoming status list") + return + } + + incomingPeerResult := results[0] + switch incomingPeerResult { + case peerset.Drop, peerset.Reject: + logger.Warnf("connection rejected with peer %s", conn.RemotePeer()) + err := conn.Close() + if err != nil { + logger.Errorf("cannot close connection with peer: %w", err) + } + return + } // exchange BlockAnnounceHandshake with peer so we can start to // sync if necessary. diff --git a/dot/network/state.go b/dot/network/state.go index 5027566a11..08fa1f9bb1 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -56,7 +56,7 @@ type PeerSetHandler interface { // PeerAdd is the interface used by the PeerSetHandler to add peers in peerSet. type PeerAdd interface { - Incoming(int, ...peer.ID) + Incoming(int, ...peer.ID) ([]peerset.Status, error) AddReservedPeer(int, ...peer.ID) AddPeer(int, ...peer.ID) SetReservedPeer(int, ...peer.ID) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index fa45fda855..b69b3a6eac 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -92,12 +92,9 @@ func (h *Handler) ReportPeer(rep ReputationChange, peers ...peer.ID) { } // Incoming calls when we have an incoming connection from peer. -func (h *Handler) Incoming(setID int, peers ...peer.ID) { - err := h.peerSet.incoming(setID, peers...) - if err != nil { - msg := fmt.Sprintf(logStringPattern, incoming, setID, "", stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) - } +func (h *Handler) Incoming(setID int, peers ...peer.ID) (status []Status, err error) { + peersStatus, err := h.peerSet.incoming(setID, peers...) + return peersStatus, err } // DisconnectPeer calls for disconnecting a connection from peer. diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 66666c856c..d03eb720db 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -571,22 +571,26 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { // incoming indicates that we have received an incoming connection. Must be answered // either with a corresponding `Accept` or `Reject`, except if we were already // connected to this peer. -func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { +func (ps *PeerSet) incoming(setID int, peers ...peer.ID) ([]Status, error) { err := ps.updateTime() if err != nil { - return fmt.Errorf("cannot update time: %w", err) + return nil, fmt.Errorf("cannot update time: %w", err) } - for _, pid := range peers { + peersStatus := make([]Status, len(peers)) + + for idx, pid := range peers { if ps.isReservedOnly { - if _, ok := ps.reservedNode[pid]; !ok { - rejectMessage := Message{ + _, has := ps.reservedNode[pid] + if !has { + message := Message{ Status: Reject, setID: uint64(setID), PeerID: pid, } - ps.processor.Process(rejectMessage) + peersStatus[idx] = message.Status + ps.processor.Process(message) continue } } @@ -612,39 +616,30 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { } state.RUnlock() - var message Message + message := Message{ + setID: uint64(setID), + PeerID: pid, + } switch { case nodeReputation < BannedThresholdValue: - message = Message{ - Status: Reject, - setID: uint64(setID), - PeerID: pid, - } + message.Status = Reject default: err := state.tryAcceptIncoming(setID, pid) if err != nil { logger.Errorf("cannot accept incomming peer %pid: %w", pid, err) - - message = Message{ - Status: Reject, - setID: uint64(setID), - PeerID: pid, - } + message.Status = Reject } else { logger.Debugf("incoming connection accepted from peer %s", pid) - message = Message{ - Status: Accept, - setID: uint64(setID), - PeerID: pid, - } + message.Status = Accept } } ps.processor.Process(message) + peersStatus[idx] = message.Status } - return nil + return peersStatus, nil } // DropReason represents reason for disconnection of the peer diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index f8383a9bcf..7fe8f6cdb3 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -25,13 +25,13 @@ const ( peer2 = peer.ID("testPeer2") ) -const allocTimeDuration = time.Second * 2 +const allocTimeDuration = 2 * time.Second //go:generate mockgen -destination=mock_message_processor_test.go -package $GOPACKAGE . MessageProcessor - func newTestPeerSet(t *testing.T, maxIn, maxOut uint32, bootNodes, reservedPeers []peer.ID, reservedOnly bool, processor MessageProcessor) *Handler { t.Helper() + con := &ConfigSet{ Set: []*config{ { From 2c870d28bd6f1b0951c0e1b8d61b966be353b79b Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 18 Mar 2022 10:29:47 -0400 Subject: [PATCH 34/50] chore: fix race conditions in test assertion --- dot/peerset/peerset_test.go | 158 ++++++++++++++++++++++++++---------- 1 file changed, 113 insertions(+), 45 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index ab605b5aa1..db8a3ba3f2 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -24,7 +24,7 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { ps := handler.peerSet - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) require.Equal(t, unknownPeer, ps.peerState.peerStatus(testSetID, peer1)) ps.peerState.discover(testSetID, peer1) @@ -32,34 +32,37 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { err := ps.peerState.tryAcceptIncoming(testSetID, peer1) require.NoError(t, err) - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numIn) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 1) require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, peer1)) // we ban a node by setting its reputation under the threshold. rpc := newReputationChange(BannedThresholdValue-1, "") // we need one for the message to be processed. - processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: peer1}) // report peer will disconnect the peer and set the `lastConnected` to time.Now handler.ReportPeer(rpc, peer1) - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) require.Equal(t, notConnectedPeer, ps.peerState.peerStatus(testSetID, peer1)) - disconectedAt := ps.peerState.nodes[peer1].lastConnected + lastDisconectedAt := ps.peerState.nodes[peer1].lastConnected[testSetID] + + // simple wait to ensure the triedToConnectAt will be greater than the lastDisconectedAt + time.Sleep(100 * time.Millisecond) // check that an incoming connection from that node gets refused. - processor.EXPECT().Process(Message{Status: Reject, setID: testSetID, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Reject, setID: testSetID, PeerID: peer1}) // incoming should update the lastConnected time handler.Incoming(0, peer1) - triedToConnectAt := ps.peerState.nodes[peer1].lastConnected - require.Greater(t, triedToConnectAt, disconectedAt) + triedToConnectAt := ps.peerState.nodes[peer1].lastConnected[testSetID] + require.True(t, lastDisconectedAt.Before(triedToConnectAt)) // wait a bit for the node's reputation to go above the threshold. time.Sleep(time.Millisecond * 1200) // try again. This time the node should be accepted. - processor.EXPECT().Process(Message{Status: Accept, setID: testSetID, PeerID: "testPeer1"}) + processor.EXPECT().Process(Message{Status: Accept, setID: testSetID, PeerID: peer1}) handler.Incoming(0, peer1) } @@ -73,12 +76,11 @@ func TestAddReservedPeers(t *testing.T) { handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false, processor) ps := handler.peerSet - _, exists := ps.peerState.nodes[bootNode] - require.True(t, exists) + checkNodePeerExists(t, ps.peerState, bootNode) require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, bootNode)) - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) reservedPeers := []struct { peerID peer.ID @@ -95,21 +97,16 @@ func TestAddReservedPeers(t *testing.T) { processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: tt.peerID}) handler.AddReservedPeer(testSetID, tt.peerID) - _, exists = ps.reservedNode[tt.peerID] - require.True(t, exists) - - _, exists = ps.peerState.sets[testSetID].noSlotNodes[tt.peerID] - require.True(t, exists) + checkReservedNodePeerExists(t, ps, tt.peerID) + checkPeerIsInNoSlotsNode(t, ps.peerState, tt.peerID, testSetID) - node, exists := ps.peerState.nodes[tt.peerID] - require.True(t, exists) require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, tt.peerID)) - require.Equal(t, outgoing, node.state[testSetID]) + checkNodePeerMembershipState(t, ps.peerState, tt.peerID, testSetID, outgoing) // peers in noSlotNodes maps should not increase the // numIn and numOut count - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) } } @@ -126,8 +123,8 @@ func TestPeerSetIncoming(t *testing.T) { ps := handler.peerSet require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, bootNode)) - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) incomingPeers := []struct { pid peer.ID @@ -166,15 +163,14 @@ func TestPeerSetIncoming(t *testing.T) { handler.Incoming(testSetID, tt.pid) - _, exists := ps.peerState.nodes[tt.pid] - require.True(t, exists) + checkNodePeerExists(t, ps.peerState, tt.pid) freeSlots := ps.peerState.hasFreeIncomingSlot(testSetID) require.Equal(t, tt.hasFreeIncomingSlot, freeSlots) - require.Equal(t, tt.expectedNumIn, ps.peerState.sets[testSetID].numIn) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, tt.expectedNumIn) // incoming peers should not chang the numOut count - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) } } @@ -196,24 +192,22 @@ func TestPeerSetDiscovered(t *testing.T) { require.True(t, isNoSlotNode) // reserved nodes should not increase the numOut count - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 0) processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered1}) handler.AddPeer(0, discovered1) handler.AddPeer(0, discovered1) - _, exitsts := ps.peerState.nodes[discovered1] - require.True(t, exitsts) + checkNodePeerExists(t, ps.peerState, discovered1) // AddPeer called twice with the same peer ID should not increase the numOut count - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered2}) handler.AddPeer(0, discovered2) - _, exitsts = ps.peerState.nodes[discovered2] - require.True(t, exitsts) - require.Equal(t, uint32(2), ps.peerState.sets[testSetID].numOut) + checkNodePeerExists(t, ps.peerState, discovered2) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 2) } func TestReAllocAfterBanned(t *testing.T) { @@ -232,7 +226,7 @@ func TestReAllocAfterBanned(t *testing.T) { // accepting the income peer which is not in the reserved peers // should increase the numIn count by 1 - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numIn) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 1) // We ban a node by setting its reputation under the threshold. processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: peer1}) @@ -240,9 +234,12 @@ func TestReAllocAfterBanned(t *testing.T) { handler.ReportPeer(rep, peer1) // banning a incoming peer should decrease the numIn count by 1 - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numIn) - node := ps.peerState.nodes[peer1] - require.Equal(t, notConnected, node.state[testSetID]) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) + + checkNodePeerMembershipState(t, ps.peerState, peer1, testSetID, notConnected) + + node, exists := getNodePeer(t, ps.peerState, peer1) + require.True(t, exists) // when the peer1 was banned we updated its lastConnected field to time.Now() lastTimeConnected := node.lastConnected[testSetID] @@ -252,8 +249,10 @@ func TestReAllocAfterBanned(t *testing.T) { handler.Incoming(testSetID, peer1) // when calling Incoming method the peer1 is with status notConnectedPeer - // so we update its lastConnected field to time.Now() - node = ps.peerState.nodes[peer1] + // so we update its lastConnected field to time.Now() again + node, exists = getNodePeer(t, ps.peerState, peer1) + require.True(t, exists) + currentLastTimeConnected := node.lastConnected[testSetID] require.True(t, lastTimeConnected.Before(currentLastTimeConnected)) @@ -261,7 +260,7 @@ func TestReAllocAfterBanned(t *testing.T) { processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: peer1}) <-time.After(allocTimeDuration + time.Second) - require.Equal(t, uint32(1), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) } func TestRemovePeer(t *testing.T) { @@ -277,14 +276,14 @@ func TestRemovePeer(t *testing.T) { ps := handler.peerSet require.Equal(t, 2, len(ps.peerState.nodes)) - require.Equal(t, uint32(2), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 2) processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) handler.RemovePeer(testSetID, discovered1, discovered2) require.Equal(t, 0, len(ps.peerState.nodes)) - require.Equal(t, uint32(0), ps.peerState.sets[testSetID].numOut) + checkPeerStateSetNumOut(t, ps.peerState, testSetID, 0) } func TestSetReservePeer(t *testing.T) { @@ -312,3 +311,72 @@ func TestSetReservePeer(t *testing.T) { require.Contains(t, ps.reservedNode, p) } } + +func getNodePeer(t *testing.T, ps *PeersState, pid peer.ID) (node, bool) { + ps.RLock() + defer ps.RUnlock() + + n, exists := ps.nodes[pid] + if !exists { + return node{}, false + } + + return *n, exists +} + +func checkNodePeerMembershipState(t *testing.T, ps *PeersState, pid peer.ID, + setID int, ms MembershipState) { + t.Helper() + + ps.RLock() + defer ps.RUnlock() + + node, exists := ps.nodes[pid] + + require.True(t, exists) + require.Equal(t, ms, node.state[setID]) +} + +func checkNodePeerExists(t *testing.T, ps *PeersState, pid peer.ID) { + t.Helper() + + ps.RLock() + defer ps.RUnlock() + + _, exists := ps.nodes[pid] + require.True(t, exists) +} + +func checkReservedNodePeerExists(t *testing.T, ps *PeerSet, pid peer.ID) { + t.Helper() + + ps.Lock() + defer ps.Unlock() + + _, exists := ps.reservedNode[pid] + require.True(t, exists) +} + +func checkPeerIsInNoSlotsNode(t *testing.T, ps *PeersState, pid peer.ID, setID int) { + ps.RLock() + defer ps.RUnlock() + + _, exists := ps.sets[setID].noSlotNodes[pid] + require.True(t, exists) +} + +func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNumOut uint32) { + ps.RLock() + defer ps.RUnlock() + + gotNumOut := ps.sets[testSetID].numOut + require.Equal(t, expectedNumOut, gotNumOut) +} + +func checkPeerStateSetNumIn(t *testing.T, ps *PeersState, setID int, expectedNumIn uint32) { + ps.RLock() + defer ps.RUnlock() + + gotNumIn := ps.sets[testSetID].numIn + require.Equal(t, expectedNumIn, gotNumIn) +} From 9dcb3480bcfdc2345363ebaf43256c3fa6ee9485 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 18 Mar 2022 10:30:40 -0400 Subject: [PATCH 35/50] chore: exec `go fmt ./...` --- dot/peerset/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 2a521d3358..de7832bf08 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -116,7 +116,7 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { } // SetMessageProcessor sets the peerset processor of the handler -// to process peerset messages. +// to process peerset messages. func (h *Handler) SetMessageProcessor(processor MessageProcessor) { h.peerSet.processor = processor } From c72694904a5257d21b41828e9095e417d85fde43 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 18 Mar 2022 10:44:00 -0400 Subject: [PATCH 36/50] chore: add `//nolint:unparam` --- dot/peerset/peerset_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index db8a3ba3f2..d45da0940d 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -238,11 +238,11 @@ func TestReAllocAfterBanned(t *testing.T) { checkNodePeerMembershipState(t, ps.peerState, peer1, testSetID, notConnected) - node, exists := getNodePeer(t, ps.peerState, peer1) + n, exists := getNodePeer(t, ps.peerState, peer1) require.True(t, exists) // when the peer1 was banned we updated its lastConnected field to time.Now() - lastTimeConnected := node.lastConnected[testSetID] + lastTimeConnected := n.lastConnected[testSetID] // Check that an incoming connection from that node gets refused. processor.EXPECT().Process(Message{Status: Reject, setID: testSetID, PeerID: peer1}) @@ -250,10 +250,10 @@ func TestReAllocAfterBanned(t *testing.T) { // when calling Incoming method the peer1 is with status notConnectedPeer // so we update its lastConnected field to time.Now() again - node, exists = getNodePeer(t, ps.peerState, peer1) + n, exists = getNodePeer(t, ps.peerState, peer1) require.True(t, exists) - currentLastTimeConnected := node.lastConnected[testSetID] + currentLastTimeConnected := n.lastConnected[testSetID] require.True(t, lastTimeConnected.Before(currentLastTimeConnected)) // wait a bit for the node's reputation to go above the threshold. @@ -313,6 +313,8 @@ func TestSetReservePeer(t *testing.T) { } func getNodePeer(t *testing.T, ps *PeersState, pid peer.ID) (node, bool) { + t.Helper() + ps.RLock() defer ps.RUnlock() @@ -365,6 +367,7 @@ func checkPeerIsInNoSlotsNode(t *testing.T, ps *PeersState, pid peer.ID, setID i require.True(t, exists) } +//nolint:unparam func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNumOut uint32) { ps.RLock() defer ps.RUnlock() @@ -373,6 +376,7 @@ func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNu require.Equal(t, expectedNumOut, gotNumOut) } +//nolint:unparam func checkPeerStateSetNumIn(t *testing.T, ps *PeersState, setID int, expectedNumIn uint32) { ps.RLock() defer ps.RUnlock() From 7d2f7c708995ee5b42447f9c20fd71ecd7800d2a Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 30 Mar 2022 12:30:54 -0400 Subject: [PATCH 37/50] chore: use channels with protected maps --- dot/network/host.go | 36 --------- dot/network/service.go | 78 ++++++++++++------ dot/network/state.go | 6 +- dot/peerset/handler.go | 131 ++++++++++++++++-------------- dot/peerset/peerset.go | 113 +++++++++++++++++++++----- dot/peerset/peerset_test.go | 154 +++++++++++++++++++----------------- dot/peerset/test_helpers.go | 10 ++- 7 files changed, 313 insertions(+), 215 deletions(-) diff --git a/dot/network/host.go b/dot/network/host.go index c998fd2eb4..9f626438bb 100644 --- a/dot/network/host.go +++ b/dot/network/host.go @@ -223,7 +223,6 @@ func newHost(ctx context.Context, cfg *Config) (*host, error) { } cm.host = host - cm.peerSetHandler.SetMessageProcessor(host) return host, nil } @@ -438,38 +437,3 @@ func (h *host) closeProtocolStream(pID protocol.ID, p peer.ID) { } } } - -// Process will connect, drop or reject a peer based on a peerset message -func (h *host) Process(msg peerset.Message) { - peerID := msg.PeerID - if peerID == "" { - logger.Errorf("found empty peer id in peerset message") - return - } - switch msg.Status { - case peerset.Connect: - addrInfo := h.p2pHost.Peerstore().PeerInfo(peerID) - if len(addrInfo.Addrs) == 0 { - var err error - addrInfo, err = h.discovery.findPeer(peerID) - if err != nil { - logger.Warnf("failed to find peer id %s: %s", peerID, err) - return - } - } - - err := h.connect(addrInfo) - if err != nil { - logger.Warnf("failed to open connection for peer %s: %s", peerID, err) - return - } - logger.Debugf("connection successful with peer %s", peerID) - case peerset.Drop, peerset.Reject: - err := h.closePeer(peerID) - if err != nil { - logger.Warnf("failed to close connection with peer %s: %s", peerID, err) - return - } - logger.Debugf("connection dropped successfully for peer %s", peerID) - } -} diff --git a/dot/network/service.go b/dot/network/service.go index 8311eafdd1..9f6ba2779f 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -439,31 +439,7 @@ func (s *Service) sentBlockIntervalTelemetry() { func (s *Service) handleConn(conn libp2pnetwork.Conn) { // TODO: currently we only have one set so setID is 0, change this once we have more set in peerSet. - results, err := s.host.cm.peerSetHandler.Incoming(0, conn.RemotePeer()) - if err != nil { - logger.Errorf("cannot accept incoming peer: %w", err) - err := conn.Close() - if err != nil { - logger.Errorf("cannot close connection with peer: %w", err) - } - return - } - - if len(results) == 0 { - logger.Errorf("got an empty incoming status list") - return - } - - incomingPeerResult := results[0] - switch incomingPeerResult { - case peerset.Drop, peerset.Reject: - logger.Warnf("connection rejected with peer %s", conn.RemotePeer()) - err := conn.Close() - if err != nil { - logger.Errorf("cannot close connection with peer: %w", err) - } - return - } + s.host.cm.peerSetHandler.Incoming(0, conn.RemotePeer()) // exchange BlockAnnounceHandshake with peer so we can start to // sync if necessary. @@ -697,4 +673,56 @@ func (s *Service) startPeerSetHandler() { if !s.noBootstrap { s.host.bootstrap() } + + go s.startProcessingMsg() +} + +func (s *Service) processMessage(msg peerset.Message) { + peerID := msg.PeerID + if peerID == "" { + logger.Errorf("found empty peer id in peerset message") + return + } + switch msg.Status { + case peerset.Connect: + addrInfo := s.host.p2pHost.Peerstore().PeerInfo(peerID) + if len(addrInfo.Addrs) == 0 { + var err error + addrInfo, err = s.host.discovery.findPeer(peerID) + if err != nil { + logger.Warnf("failed to find peer id %s: %s", peerID, err) + return + } + } + + err := s.host.connect(addrInfo) + if err != nil { + logger.Warnf("failed to open connection for peer %s: %s", peerID, err) + return + } + logger.Debugf("connection successful with peer %s", peerID) + case peerset.Drop, peerset.Reject: + err := s.host.closePeer(peerID) + if err != nil { + logger.Warnf("failed to close connection with peer %s: %s", peerID, err) + return + } + logger.Debugf("connection dropped successfully for peer %s", peerID) + } +} + +func (s *Service) startProcessingMsg() { + msgCh := s.host.cm.peerSetHandler.Messages() + for { + select { + case <-s.ctx.Done(): + return + case msg, ok := <-msgCh: + if !ok { + return + } + + s.processMessage(msg) + } + } } diff --git a/dot/network/state.go b/dot/network/state.go index 08fa1f9bb1..75c66c8810 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -46,7 +46,6 @@ type TransactionHandler interface { // PeerSetHandler is the interface used by the connection manager to handle peerset. type PeerSetHandler interface { - SetMessageProcessor(peerset.MessageProcessor) Start(context.Context) ReportPeer(peerset.ReputationChange, ...peer.ID) PeerAdd @@ -56,7 +55,7 @@ type PeerSetHandler interface { // PeerAdd is the interface used by the PeerSetHandler to add peers in peerSet. type PeerAdd interface { - Incoming(int, ...peer.ID) ([]peerset.Status, error) + Incoming(int, ...peer.ID) AddReservedPeer(int, ...peer.ID) AddPeer(int, ...peer.ID) SetReservedPeer(int, ...peer.ID) @@ -72,5 +71,6 @@ type PeerRemove interface { // Peer is the interface used by the PeerSetHandler to get the peer data from peerSet. type Peer interface { PeerReputation(peer.ID) (peerset.Reputation, error) - SortedPeers(idx int) peer.IDSlice + SortedPeers(idx int) chan peer.IDSlice + Messages() chan peerset.Message } diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index de7832bf08..adcb12e00a 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -5,16 +5,16 @@ package peerset import ( "context" - "fmt" - "strings" "github.com/libp2p/go-libp2p-core/peer" ) -const logStringPattern = "call=%s, set-id=%d, reputation change %s, peers=[%s]" - // Handler manages peerSet. type Handler struct { + actionQueue chan<- action + closeCh chan struct{} + cancelCtx context.CancelFunc + peerSet *PeerSet } @@ -30,79 +30,80 @@ func NewPeerSetHandler(cfg *ConfigSet) (*Handler, error) { }, nil } -// SetReservedOnlyPeer not yet implemented -func (h *Handler) SetReservedOnlyPeer(setID int, peers ...peer.ID) { - // TODO: not yet implemented (#1888) - logger.Errorf("failed to do action %s on peerSet: not implemented yet", setReservedOnly) -} - // AddReservedPeer adds reserved peer into peerSet. func (h *Handler) AddReservedPeer(setID int, peers ...peer.ID) { - err := h.peerSet.addReservedPeers(setID, peers...) - if err != nil { - msg := fmt.Sprintf(logStringPattern, addReservedPeer, setID, "", stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) + h.actionQueue <- action{ + actionCall: addReservedPeer, + setID: setID, + peers: peers, } } -// RemoveReservedPeer removes reserved peer from peerSet. +// RemoveReservedPeer remove reserved peer from peerSet. func (h *Handler) RemoveReservedPeer(setID int, peers ...peer.ID) { - err := h.peerSet.removeReservedPeers(setID, peers...) - if err != nil { - msg := fmt.Sprintf(logStringPattern, removeReservedPeer, setID, "", stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) + h.actionQueue <- action{ + actionCall: removeReservedPeer, + setID: setID, + peers: peers, } } -// SetReservedPeer sets the reserve peer into peerSet +// SetReservedPeer set the reserve peer into peerSet func (h *Handler) SetReservedPeer(setID int, peers ...peer.ID) { - // TODO: this is not used yet, it might be required to implement an RPC Call for this. - err := h.peerSet.setReservedPeer(setID, peers...) - if err != nil { - msg := fmt.Sprintf(logStringPattern, setReservedPeers, setID, "", stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) + h.actionQueue <- action{ + actionCall: setReservedPeers, + setID: setID, + peers: peers, } } // AddPeer adds peer to peerSet. func (h *Handler) AddPeer(setID int, peers ...peer.ID) { - err := h.peerSet.addPeer(setID, peers) - if err != nil { - msg := fmt.Sprintf(logStringPattern, addToPeerSet, setID, "", stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) + h.actionQueue <- action{ + actionCall: addToPeerSet, + setID: setID, + peers: peers, } } // RemovePeer removes peer from peerSet. func (h *Handler) RemovePeer(setID int, peers ...peer.ID) { - err := h.peerSet.removePeer(setID, peers...) - if err != nil { - msg := fmt.Sprintf(logStringPattern, removeFromPeerSet, setID, "", stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) + h.actionQueue <- action{ + actionCall: removeFromPeerSet, + setID: setID, + peers: peers, } } // ReportPeer reports ReputationChange according to the peer behaviour. func (h *Handler) ReportPeer(rep ReputationChange, peers ...peer.ID) { - err := h.peerSet.reportPeer(rep, peers...) - if err != nil { - msg := fmt.Sprintf(logStringPattern, reportPeer, 0, rep.String(), stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) + h.actionQueue <- action{ + actionCall: reportPeer, + reputation: rep, + peers: peers, } } // Incoming calls when we have an incoming connection from peer. -func (h *Handler) Incoming(setID int, peers ...peer.ID) (status []Status, err error) { - peersStatus, err := h.peerSet.incoming(setID, peers...) - return peersStatus, err +func (h *Handler) Incoming(setID int, peers ...peer.ID) { + h.actionQueue <- action{ + actionCall: incoming, + peers: peers, + setID: setID, + } +} + +// Messages return result message chan. +func (h *Handler) Messages() chan Message { + return h.peerSet.resultMsgCh } // DisconnectPeer calls for disconnecting a connection from peer. func (h *Handler) DisconnectPeer(setID int, peers ...peer.ID) { - err := h.peerSet.disconnect(setID, UnknownDrop, peers...) - if err != nil { - msg := fmt.Sprintf(logStringPattern, disconnect, setID, "", stringfyPeers(peers)) - logger.Errorf("failed to do action %s on peerSet: %s", msg, err) + h.actionQueue <- action{ + actionCall: disconnect, + setID: setID, + peers: peers, } } @@ -115,27 +116,37 @@ func (h *Handler) PeerReputation(peerID peer.ID) (Reputation, error) { return n.reputation, nil } -// SetMessageProcessor sets the peerset processor of the handler -// to process peerset messages. -func (h *Handler) SetMessageProcessor(processor MessageProcessor) { - h.peerSet.processor = processor -} - // Start starts peerSet processing func (h *Handler) Start(ctx context.Context) { - go h.peerSet.periodicallyAllocateSlots(ctx) -} + ctx, cancel := context.WithCancel(ctx) + h.cancelCtx = cancel + + actionCh := make(chan action, msgChanSize) + h.closeCh = make(chan struct{}) + h.actionQueue = actionCh -// SortedPeers returns a sorted peer ID slice for connected peers in the peerSet. -func (h *Handler) SortedPeers(setIdx int) peer.IDSlice { - return h.peerSet.peerState.sortedPeers(setIdx) + h.peerSet.start(ctx, actionCh) } -func stringfyPeers(peers peer.IDSlice) string { - peersStrings := make([]string, len(peers)) - for i := range peers { - peersStrings[i] = peers[i].String() +// SortedPeers return chan for sorted connected peer in the peerSet. +func (h *Handler) SortedPeers(setIdx int) chan peer.IDSlice { + resultPeersCh := make(chan peer.IDSlice) + h.actionQueue <- action{ + actionCall: sortedPeers, + resultPeersCh: resultPeersCh, + setID: setIdx, } - return strings.Join(peersStrings, ", ") + return resultPeersCh +} + +// Stop closes the actionQueue and result message chan. +func (h *Handler) Stop() { + select { + case <-h.closeCh: + default: + h.cancelCtx() + close(h.closeCh) + close(h.actionQueue) + } } diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index d03eb720db..d5f3b49032 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "math" + "strings" "sync" "time" @@ -24,6 +25,8 @@ const ( // forgetAfterTime amount of time between the moment we disconnect // from a node and the moment we remove it from the list. forgetAfterTime = time.Second * 3600 // one hour + + msgChanSize = 100 ) // ActionReceiver represents the enum value for action to be performed on peerSet @@ -79,6 +82,24 @@ func (a ActionReceiver) String() string { } } +// action struct stores the action type and required parameters to perform action +type action struct { + actionCall ActionReceiver + setID int + reputation ReputationChange + peers peer.IDSlice + resultPeersCh chan peer.IDSlice +} + +func (a action) String() string { + peersStrings := make([]string, len(a.peers)) + for i := range a.peers { + peersStrings[i] = a.peers[i].String() + } + return fmt.Sprintf("{call=%s, set-id=%d, reputation change %v, peers=[%s]", + a.actionCall.String(), a.setID, a.reputation, strings.Join(peersStrings, ", ")) +} + // Status represents the enum value for Message type Status uint8 @@ -156,10 +177,12 @@ type PeerSet struct { sync.Mutex peerState *PeersState + reservedLock sync.RWMutex reservedNode map[peer.ID]struct{} // TODO: this will be useful for reserved only mode // this is for future purpose if reserved-only flag is enabled (#1888). isReservedOnly bool + resultMsgCh chan Message // time when the PeerSet was created. created time.Time // last time when we updated the reputations of connected nodes. @@ -167,8 +190,7 @@ type PeerSet struct { // next time to do a periodic call to allocSlots with all Set. This is done once two // second, to match the period of the Reputation updates. nextPeriodicAllocSlots time.Duration - - processor MessageProcessor + actionQueue <-chan action } // config is configuration of a single set. @@ -341,7 +363,7 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { PeerID: pid, } - ps.processor.Process(dropMessage) + ps.resultMsgCh <- dropMessage if err = ps.allocSlots(i); err != nil { return fmt.Errorf("could not allocate slots: %w", err) @@ -389,7 +411,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { PeerID: reservePeer, } - ps.processor.Process(connectMessage) + ps.resultMsgCh <- connectMessage } // nothing more to do if we're in reserved mode. @@ -420,7 +442,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { PeerID: peerID, } - ps.processor.Process(connectMessage) + ps.resultMsgCh <- connectMessage logger.Debugf("Sent connect message to peer %s", peerID) } @@ -428,6 +450,9 @@ func (ps *PeerSet) allocSlots(setIdx int) error { } func (ps *PeerSet) addReservedPeers(setID int, peers ...peer.ID) error { + ps.reservedLock.Lock() + defer ps.reservedLock.Unlock() + for _, peerID := range peers { if _, ok := ps.reservedNode[peerID]; ok { logger.Debugf("peer %s already exists in peerSet", peerID) @@ -448,6 +473,9 @@ func (ps *PeerSet) addReservedPeers(setID int, peers ...peer.ID) error { } func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { + ps.reservedLock.Lock() + defer ps.reservedLock.Unlock() + for _, peerID := range peers { if _, ok := ps.reservedNode[peerID]; !ok { logger.Debugf("peer %s doesn't exist in the peerSet", peerID) @@ -479,7 +507,7 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { PeerID: peerID, } - ps.processor.Process(dropMessage) + ps.resultMsgCh <- dropMessage } } @@ -548,7 +576,7 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { PeerID: pid, } - ps.processor.Process(dropMessage) + ps.resultMsgCh <- dropMessage // disconnect and forget err := ps.peerState.disconnect(setID, pid) @@ -571,15 +599,13 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { // incoming indicates that we have received an incoming connection. Must be answered // either with a corresponding `Accept` or `Reject`, except if we were already // connected to this peer. -func (ps *PeerSet) incoming(setID int, peers ...peer.ID) ([]Status, error) { +func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { err := ps.updateTime() if err != nil { - return nil, fmt.Errorf("cannot update time: %w", err) + return fmt.Errorf("cannot update time: %w", err) } - peersStatus := make([]Status, len(peers)) - - for idx, pid := range peers { + for _, pid := range peers { if ps.isReservedOnly { _, has := ps.reservedNode[pid] if !has { @@ -589,8 +615,7 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) ([]Status, error) { PeerID: pid, } - peersStatus[idx] = message.Status - ps.processor.Process(message) + ps.resultMsgCh <- message continue } } @@ -635,11 +660,10 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) ([]Status, error) { } } - ps.processor.Process(message) - peersStatus[idx] = message.Status + ps.resultMsgCh <- message } - return peersStatus, nil + return nil } // DropReason represents reason for disconnection of the peer @@ -683,7 +707,7 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e PeerID: pid, } - ps.processor.Process(dropMessage) + ps.resultMsgCh <- dropMessage // TODO: figure out the condition of connection refuse. if reason == RefusedDrop { @@ -696,6 +720,59 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e return ps.allocSlots(setIdx) } +// start handles all the action for the peerSet. +func (ps *PeerSet) start(ctx context.Context, actionQueue chan action) { + ps.actionQueue = actionQueue + ps.resultMsgCh = make(chan Message, msgChanSize) + + go ps.listenAction(ctx) + go ps.periodicallyAllocateSlots(ctx) +} + +func (ps *PeerSet) listenAction(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case act, ok := <-ps.actionQueue: + + if !ok { + return + } + + var err error + switch act.actionCall { + case addReservedPeer: + err = ps.addReservedPeers(act.setID, act.peers...) + case removeReservedPeer: + err = ps.removeReservedPeers(act.setID, act.peers...) + case setReservedPeers: + // TODO: this is not used yet, might required to implement RPC Call for this. + err = ps.setReservedPeer(act.setID, act.peers...) + case setReservedOnly: + // TODO: not yet implemented (#1888) + err = fmt.Errorf("not implemented yet") + case reportPeer: + err = ps.reportPeer(act.reputation, act.peers...) + case addToPeerSet: + err = ps.addPeer(act.setID, act.peers) + case removeFromPeerSet: + err = ps.removePeer(act.setID, act.peers...) + case incoming: + err = ps.incoming(act.setID, act.peers...) + case sortedPeers: + act.resultPeersCh <- ps.peerState.sortedPeers(act.setID) + case disconnect: + err = ps.disconnect(act.setID, UnknownDrop, act.peers...) + } + + if err != nil { + logger.Errorf("failed to do action %s on peerSet: %s", act, err) + } + } + } +} + func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { ticker := time.NewTicker(ps.nextPeriodicAllocSlots) defer ticker.Stop() diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index d45da0940d..c55eb17719 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" ) @@ -17,10 +16,7 @@ const testSetID = 0 func Test_Ban_Reject_Accept_Peer(t *testing.T) { t.Parallel() - ctrl := gomock.NewController(t) - processor := NewMockMessageProcessor(ctrl) - - handler := newTestPeerSet(t, 25, 25, nil, nil, false, processor) + handler := newTestPeerSet(t, 25, 25, nil, nil, false) ps := handler.peerSet @@ -39,21 +35,20 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { rpc := newReputationChange(BannedThresholdValue-1, "") // we need one for the message to be processed. - processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: peer1}) // report peer will disconnect the peer and set the `lastConnected` to time.Now handler.ReportPeer(rpc, peer1) + + time.Sleep(100 * time.Millisecond) + checkMessageStatus(t, <-ps.resultMsgCh, Drop) + checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) require.Equal(t, notConnectedPeer, ps.peerState.peerStatus(testSetID, peer1)) - lastDisconectedAt := ps.peerState.nodes[peer1].lastConnected[testSetID] - // simple wait to ensure the triedToConnectAt will be greater than the lastDisconectedAt - time.Sleep(100 * time.Millisecond) - // check that an incoming connection from that node gets refused. - processor.EXPECT().Process(Message{Status: Reject, setID: testSetID, PeerID: peer1}) // incoming should update the lastConnected time handler.Incoming(0, peer1) + checkMessageStatus(t, <-ps.resultMsgCh, Reject) triedToConnectAt := ps.peerState.nodes[peer1].lastConnected[testSetID] require.True(t, lastDisconectedAt.Before(triedToConnectAt)) @@ -62,18 +57,13 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { time.Sleep(time.Millisecond * 1200) // try again. This time the node should be accepted. - processor.EXPECT().Process(Message{Status: Accept, setID: testSetID, PeerID: peer1}) handler.Incoming(0, peer1) + checkMessageStatus(t, <-ps.resultMsgCh, Accept) } func TestAddReservedPeers(t *testing.T) { t.Parallel() - - ctrl := gomock.NewController(t) - processor := NewMockMessageProcessor(ctrl) - - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: bootNode}) - handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false, processor) + handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false) ps := handler.peerSet checkNodePeerExists(t, ps.peerState, bootNode) @@ -94,8 +84,8 @@ func TestAddReservedPeers(t *testing.T) { } for _, tt := range reservedPeers { - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: tt.peerID}) handler.AddReservedPeer(testSetID, tt.peerID) + time.Sleep(time.Millisecond * 100) checkReservedNodePeerExists(t, ps, tt.peerID) checkPeerIsInNoSlotsNode(t, ps.peerState, tt.peerID, testSetID) @@ -108,19 +98,31 @@ func TestAddReservedPeers(t *testing.T) { checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) } + + expectedMsgs := []Message{ + {Status: Connect, setID: 0, PeerID: bootNode}, + {Status: Connect, setID: 0, PeerID: reservedPeer}, + {Status: Connect, setID: 0, PeerID: reservedPeer2}, + } + + require.Equal(t, 3, len(ps.resultMsgCh)) + + for i := 0; ; i++ { + if len(ps.resultMsgCh) == 0 { + break + } + msg := <-ps.resultMsgCh + require.Equal(t, expectedMsgs[i], msg) + } } func TestPeerSetIncoming(t *testing.T) { t.Parallel() - - ctrl := gomock.NewController(t) - processor := NewMockMessageProcessor(ctrl) - - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: bootNode}) handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, - []peer.ID{}, false, processor) + []peer.ID{}, false) ps := handler.peerSet + checkMessageStatus(t, <-ps.resultMsgCh, Connect) require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, bootNode)) checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) @@ -155,13 +157,13 @@ func TestPeerSetIncoming(t *testing.T) { } for _, tt := range incomingPeers { - processor.EXPECT().Process(Message{Status: tt.expectedStatus, setID: testSetID, PeerID: tt.pid}) // all the incoming peers are unknow before calling the Incoming method status := ps.peerState.peerStatus(testSetID, tt.pid) require.Equal(t, unknownPeer, status) handler.Incoming(testSetID, tt.pid) + time.Sleep(time.Millisecond * 100) checkNodePeerExists(t, ps.peerState, tt.pid) @@ -171,22 +173,18 @@ func TestPeerSetIncoming(t *testing.T) { checkPeerStateSetNumIn(t, ps.peerState, testSetID, tt.expectedNumIn) // incoming peers should not chang the numOut count checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) + + checkMessageStatus(t, <-ps.resultMsgCh, tt.expectedStatus) } } func TestPeerSetDiscovered(t *testing.T) { t.Parallel() - - ctrl := gomock.NewController(t) - processor := NewMockMessageProcessor(ctrl) - - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: reservedPeer}) - handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false, processor) + handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false) ps := handler.peerSet - _, isReservedNode := ps.reservedNode[reservedPeer] - require.True(t, isReservedNode) + checkReservedNodePeerExists(t, ps, reservedPeer) _, isNoSlotNode := ps.peerState.sets[testSetID].noSlotNodes[reservedPeer] require.True(t, isNoSlotNode) @@ -194,28 +192,26 @@ func TestPeerSetDiscovered(t *testing.T) { // reserved nodes should not increase the numOut count checkPeerStateSetNumOut(t, ps.peerState, testSetID, 0) - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered1}) handler.AddPeer(0, discovered1) handler.AddPeer(0, discovered1) - - checkNodePeerExists(t, ps.peerState, discovered1) - - // AddPeer called twice with the same peer ID should not increase the numOut count - checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) - - processor.EXPECT().Process(Message{Status: Connect, setID: 0, PeerID: discovered2}) handler.AddPeer(0, discovered2) + time.Sleep(200 * time.Millisecond) + + checkNodePeerExists(t, ps.peerState, discovered1) checkNodePeerExists(t, ps.peerState, discovered2) + // AddPeer called twice with the same peer ID should not increase the numOut count checkPeerStateSetNumOut(t, ps.peerState, testSetID, 2) + + require.Equal(t, 3, len(ps.resultMsgCh)) + for len(ps.resultMsgCh) == 0 { + checkMessageStatus(t, <-ps.resultMsgCh, Connect) + } } func TestReAllocAfterBanned(t *testing.T) { t.Parallel() - - ctrl := gomock.NewController(t) - processor := NewMockMessageProcessor(ctrl) - handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false, processor) + handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false) ps := handler.peerSet require.Equal(t, unknownPeer, ps.peerState.peerStatus(testSetID, peer1)) @@ -229,13 +225,14 @@ func TestReAllocAfterBanned(t *testing.T) { checkPeerStateSetNumIn(t, ps.peerState, testSetID, 1) // We ban a node by setting its reputation under the threshold. - processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: peer1}) rep := newReputationChange(BannedThresholdValue-1, "") handler.ReportPeer(rep, peer1) + time.Sleep(time.Millisecond * 100) + checkMessageStatus(t, <-ps.resultMsgCh, Drop) + // banning a incoming peer should decrease the numIn count by 1 checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) - checkNodePeerMembershipState(t, ps.peerState, peer1, testSetID, notConnected) n, exists := getNodePeer(t, ps.peerState, peer1) @@ -245,8 +242,8 @@ func TestReAllocAfterBanned(t *testing.T) { lastTimeConnected := n.lastConnected[testSetID] // Check that an incoming connection from that node gets refused. - processor.EXPECT().Process(Message{Status: Reject, setID: testSetID, PeerID: peer1}) handler.Incoming(testSetID, peer1) + checkMessageStatus(t, <-ps.resultMsgCh, Reject) // when calling Incoming method the peer1 is with status notConnectedPeer // so we update its lastConnected field to time.Now() again @@ -257,58 +254,59 @@ func TestReAllocAfterBanned(t *testing.T) { require.True(t, lastTimeConnected.Before(currentLastTimeConnected)) // wait a bit for the node's reputation to go above the threshold. - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: peer1}) <-time.After(allocTimeDuration + time.Second) checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) + checkMessageStatus(t, <-ps.resultMsgCh, Connect) } func TestRemovePeer(t *testing.T) { t.Parallel() - - ctrl := gomock.NewController(t) - processor := NewMockMessageProcessor(ctrl) - - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: "testDiscovered1"}) - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: "testDiscovered2"}) handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, - nil, false, processor) + nil, false) ps := handler.peerSet + require.Equal(t, 2, len(ps.resultMsgCh)) + for len(ps.resultMsgCh) != 0 { + checkMessageStatus(t, <-ps.resultMsgCh, Connect) + } + require.Equal(t, 2, len(ps.peerState.nodes)) checkPeerStateSetNumOut(t, ps.peerState, testSetID, 2) - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered1"}) - processor.EXPECT().Process(Message{Status: Drop, setID: 0, PeerID: "testDiscovered2"}) handler.RemovePeer(testSetID, discovered1, discovered2) + time.Sleep(200 * time.Millisecond) + + require.Equal(t, 2, len(ps.resultMsgCh)) + for len(ps.resultMsgCh) != 0 { + checkMessageStatus(t, <-ps.resultMsgCh, Drop) + } - require.Equal(t, 0, len(ps.peerState.nodes)) + checkPeerStateNodeCount(t, ps.peerState, 0) checkPeerStateSetNumOut(t, ps.peerState, testSetID, 0) } func TestSetReservePeer(t *testing.T) { t.Parallel() - - ctrl := gomock.NewController(t) - processor := NewMockMessageProcessor(ctrl) - - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: reservedPeer}) - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: reservedPeer2}) handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, - true, processor) + true) ps := handler.peerSet + require.Equal(t, 2, len(ps.resultMsgCh)) + for len(ps.resultMsgCh) != 0 { + checkMessageStatus(t, <-ps.resultMsgCh, Connect) + } + require.Equal(t, 2, len(ps.reservedNode)) newRsrPeerSet := peer.IDSlice{reservedPeer, peer.ID("newRsrPeer")} // add newRsrPeer but remove reservedPeer2 - processor.EXPECT().Process(Message{Status: Connect, setID: testSetID, PeerID: "newRsrPeer"}) - processor.EXPECT().Process(Message{Status: Drop, setID: testSetID, PeerID: reservedPeer2}) handler.SetReservedPeer(testSetID, newRsrPeerSet...) + time.Sleep(200 * time.Millisecond) - require.Equal(t, 2, len(ps.reservedNode)) + checkPeerSetReservedNodeCount(t, ps, 2) for _, p := range newRsrPeerSet { - require.Contains(t, ps.reservedNode, p) + checkReservedNodePeerExists(t, ps, p) } } @@ -384,3 +382,17 @@ func checkPeerStateSetNumIn(t *testing.T, ps *PeersState, setID int, expectedNum gotNumIn := ps.sets[testSetID].numIn require.Equal(t, expectedNumIn, gotNumIn) } + +func checkPeerStateNodeCount(t *testing.T, ps *PeersState, expectedCount int) { + ps.RLock() + defer ps.RUnlock() + + require.Equal(t, expectedCount, len(ps.nodes)) +} + +func checkPeerSetReservedNodeCount(t *testing.T, ps *PeerSet, expectedCount int) { + ps.reservedLock.RLock() + defer ps.reservedLock.RUnlock() + + require.Equal(t, expectedCount, len(ps.reservedNode)) +} diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index 7fe8f6cdb3..36b88a368f 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -29,7 +29,7 @@ const allocTimeDuration = 2 * time.Second //go:generate mockgen -destination=mock_message_processor_test.go -package $GOPACKAGE . MessageProcessor func newTestPeerSet(t *testing.T, maxIn, maxOut uint32, bootNodes, - reservedPeers []peer.ID, reservedOnly bool, processor MessageProcessor) *Handler { + reservedPeers []peer.ID, reservedOnly bool) *Handler { t.Helper() con := &ConfigSet{ @@ -46,7 +46,6 @@ func newTestPeerSet(t *testing.T, maxIn, maxOut uint32, bootNodes, handler, err := NewPeerSetHandler(con) require.NoError(t, err) - handler.SetMessageProcessor(processor) handler.Start(context.Background()) handler.AddPeer(0, bootNodes...) @@ -68,3 +67,10 @@ func newTestPeerState(t *testing.T, maxIn, maxOut uint32) *PeersState { return state } + +func checkMessageStatus(t *testing.T, m interface{}, expectedStatus Status) { + t.Helper() + msg, ok := m.(Message) + require.True(t, ok) + require.Equal(t, expectedStatus, msg.Status) +} From 266d7a29617bd51dd06a01b7ef7fd1791afb8506 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 30 Mar 2022 12:37:34 -0400 Subject: [PATCH 38/50] chore: remove nolintlint warns --- dot/peerset/peerset_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index c55eb17719..b3ba092d22 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -365,7 +365,6 @@ func checkPeerIsInNoSlotsNode(t *testing.T, ps *PeersState, pid peer.ID, setID i require.True(t, exists) } -//nolint:unparam func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNumOut uint32) { ps.RLock() defer ps.RUnlock() @@ -374,7 +373,6 @@ func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNu require.Equal(t, expectedNumOut, gotNumOut) } -//nolint:unparam func checkPeerStateSetNumIn(t *testing.T, ps *PeersState, setID int, expectedNumIn uint32) { ps.RLock() defer ps.RUnlock() From 0dbad30294ac6dc4896345734f38d457b1f52ca6 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 25 Apr 2022 10:11:20 +0200 Subject: [PATCH 39/50] chore: fixing the newtork service Stop method --- dot/network/service.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dot/network/service.go b/dot/network/service.go index 9f6ba2779f..e4ed7be4e0 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -490,6 +490,10 @@ func (s *Service) Stop() error { // check if closeCh is closed, if not, close it. for { select { + case _, hasMore := <-s.closeCh: + if !hasMore { + return nil + } case <-s.closeCh: return nil default: From 2207aad669c99a6a227da951f1ef930121a92f1e Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 25 Apr 2022 10:12:20 +0200 Subject: [PATCH 40/50] chore: reintroduce the `msgChanSize` comment --- dot/peerset/peerset.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index d5f3b49032..93735e5ac1 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -26,6 +26,7 @@ const ( // from a node and the moment we remove it from the list. forgetAfterTime = time.Second * 3600 // one hour + // default channel size for peerSet. msgChanSize = 100 ) From 04221fa2c49e69ff022ac14dede35bf1aff3d96b Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 25 Apr 2022 10:19:48 +0200 Subject: [PATCH 41/50] chore: close the `resultMsgCh` once ctx ends --- dot/peerset/peerset.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 93735e5ac1..5057750bdc 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -610,13 +610,13 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { if ps.isReservedOnly { _, has := ps.reservedNode[pid] if !has { - message := Message{ + rejectMessage := Message{ Status: Reject, setID: uint64(setID), PeerID: pid, } - ps.resultMsgCh <- message + ps.resultMsgCh <- rejectMessage continue } } @@ -642,26 +642,26 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { } state.RUnlock() - message := Message{ + incomingMessage := Message{ setID: uint64(setID), PeerID: pid, } switch { case nodeReputation < BannedThresholdValue: - message.Status = Reject + incomingMessage.Status = Reject default: err := state.tryAcceptIncoming(setID, pid) if err != nil { logger.Errorf("cannot accept incomming peer %pid: %w", pid, err) - message.Status = Reject + incomingMessage.Status = Reject } else { logger.Debugf("incoming connection accepted from peer %s", pid) - message.Status = Accept + incomingMessage.Status = Accept } } - ps.resultMsgCh <- message + ps.resultMsgCh <- incomingMessage } return nil @@ -776,7 +776,11 @@ func (ps *PeerSet) listenAction(ctx context.Context) { func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { ticker := time.NewTicker(ps.nextPeriodicAllocSlots) - defer ticker.Stop() + + defer func() { + ticker.Stop() + close(ps.resultMsgCh) + }() for { select { From 0dffe8aa2ccb0b66192cd4c0112dddc8b685099d Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 25 Apr 2022 10:23:55 +0200 Subject: [PATCH 42/50] chore: use variables on eclesio/refactor-peerstate --- dot/peerset/peerset.go | 3 +-- dot/peerset/peerset_test.go | 14 +++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 5057750bdc..8e08650e3b 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -788,8 +788,7 @@ func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { // TODO: log context error? return case <-ticker.C: - setLen := ps.peerState.getSetLength() - for i := 0; i < setLen; i++ { + for i := 0; i < ps.peerState.getSetLength(); i++ { if err := ps.allocSlots(i); err != nil { logger.Warnf("failed to do action on peerSet: %s", err) } diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index b3ba092d22..33b58076e6 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -13,7 +13,7 @@ import ( const testSetID = 0 -func Test_Ban_Reject_Accept_Peer(t *testing.T) { +func TestBanRejectAcceptPeer(t *testing.T) { t.Parallel() handler := newTestPeerSet(t, 25, 25, nil, nil, false) @@ -21,7 +21,8 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { ps := handler.peerSet checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) - require.Equal(t, unknownPeer, ps.peerState.peerStatus(testSetID, peer1)) + peer1Status := ps.peerState.peerStatus(testSetID, peer1) + require.Equal(t, unknownPeer, peer1Status) ps.peerState.discover(testSetID, peer1) // adding peer1 with incoming slot. @@ -29,7 +30,8 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { require.NoError(t, err) checkPeerStateSetNumIn(t, ps.peerState, testSetID, 1) - require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, peer1)) + peer1Status = ps.peerState.peerStatus(testSetID, peer1) + require.Equal(t, connectedPeer, peer1Status) // we ban a node by setting its reputation under the threshold. rpc := newReputationChange(BannedThresholdValue-1, "") @@ -42,7 +44,8 @@ func Test_Ban_Reject_Accept_Peer(t *testing.T) { checkMessageStatus(t, <-ps.resultMsgCh, Drop) checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) - require.Equal(t, notConnectedPeer, ps.peerState.peerStatus(testSetID, peer1)) + peer1Status = ps.peerState.peerStatus(testSetID, peer1) + require.Equal(t, notConnectedPeer, peer1Status) lastDisconectedAt := ps.peerState.nodes[peer1].lastConnected[testSetID] // check that an incoming connection from that node gets refused. @@ -214,7 +217,8 @@ func TestReAllocAfterBanned(t *testing.T) { handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false) ps := handler.peerSet - require.Equal(t, unknownPeer, ps.peerState.peerStatus(testSetID, peer1)) + peer1Status := ps.peerState.peerStatus(testSetID, peer1) + require.Equal(t, unknownPeer, peer1Status) ps.peerState.discover(testSetID, peer1) err := ps.peerState.tryAcceptIncoming(testSetID, peer1) From 77784e2befa3bde04e9d9c5d9311195df4a987b2 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 25 Apr 2022 10:28:02 +0200 Subject: [PATCH 43/50] chore: adding comments --- dot/peerset/peerset.go | 58 ++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 8e08650e3b..28f265755d 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -191,7 +191,8 @@ type PeerSet struct { // next time to do a periodic call to allocSlots with all Set. This is done once two // second, to match the period of the Reputation updates. nextPeriodicAllocSlots time.Duration - actionQueue <-chan action + // chan for receiving action request. + actionQueue <-chan action } // config is configuration of a single set. @@ -358,14 +359,12 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, i, err) } - dropMessage := Message{ + ps.resultMsgCh <- Message{ Status: Drop, setID: uint64(i), PeerID: pid, } - ps.resultMsgCh <- dropMessage - if err = ps.allocSlots(i); err != nil { return fmt.Errorf("could not allocate slots: %w", err) } @@ -406,13 +405,11 @@ func (ps *PeerSet) allocSlots(setIdx int) error { return fmt.Errorf("cannot set peer %s from set %d as outgoing: %w", reservePeer, setIdx, err) } - connectMessage := Message{ + ps.resultMsgCh <- Message{ Status: Connect, setID: uint64(setIdx), PeerID: reservePeer, } - - ps.resultMsgCh <- connectMessage } // nothing more to do if we're in reserved mode. @@ -437,14 +434,12 @@ func (ps *PeerSet) allocSlots(setIdx int) error { break } - connectMessage := Message{ + ps.resultMsgCh <- Message{ Status: Connect, setID: uint64(setIdx), PeerID: peerID, } - ps.resultMsgCh <- connectMessage - logger.Debugf("Sent connect message to peer %s", peerID) } return nil @@ -502,13 +497,12 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { return fmt.Errorf("cannot disconnect peer %s at set %d: %w", peerID, setID, err) } - dropMessage := Message{ + ps.resultMsgCh <- Message{ Status: Drop, setID: uint64(setID), PeerID: peerID, } - ps.resultMsgCh <- dropMessage } } @@ -571,14 +565,12 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { } if status := ps.peerState.peerStatus(setID, pid); status == connectedPeer { - dropMessage := Message{ + ps.resultMsgCh <- Message{ Status: Drop, setID: uint64(setID), PeerID: pid, } - ps.resultMsgCh <- dropMessage - // disconnect and forget err := ps.peerState.disconnect(setID, pid) if err != nil { @@ -610,13 +602,11 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { if ps.isReservedOnly { _, has := ps.reservedNode[pid] if !has { - rejectMessage := Message{ + ps.resultMsgCh <- Message{ Status: Reject, setID: uint64(setID), PeerID: pid, } - - ps.resultMsgCh <- rejectMessage continue } } @@ -642,26 +632,34 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { } state.RUnlock() - incomingMessage := Message{ - setID: uint64(setID), - PeerID: pid, - } - switch { case nodeReputation < BannedThresholdValue: - incomingMessage.Status = Reject + ps.resultMsgCh <- Message{ + Status: Reject, + setID: uint64(setID), + PeerID: pid, + } + default: err := state.tryAcceptIncoming(setID, pid) if err != nil { logger.Errorf("cannot accept incomming peer %pid: %w", pid, err) - incomingMessage.Status = Reject + + ps.resultMsgCh <- Message{ + Status: Reject, + setID: uint64(setID), + PeerID: pid, + } } else { logger.Debugf("incoming connection accepted from peer %s", pid) - incomingMessage.Status = Accept + + ps.resultMsgCh <- Message{ + Status: Accept, + setID: uint64(setID), + PeerID: pid, + } } } - - ps.resultMsgCh <- incomingMessage } return nil @@ -702,14 +700,12 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, setIdx, err) } - dropMessage := Message{ + ps.resultMsgCh <- Message{ Status: Drop, setID: uint64(setIdx), PeerID: pid, } - ps.resultMsgCh <- dropMessage - // TODO: figure out the condition of connection refuse. if reason == RefusedDrop { if err = ps.removePeer(setIdx, pid); err != nil { From d563386fbfda83b3d03f7fd48a710ab53d1a1f78 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 25 Apr 2022 10:33:00 +0200 Subject: [PATCH 44/50] chore: logging ctx error as debug --- dot/peerset/peerset.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 28f265755d..044c7858d2 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -781,7 +781,10 @@ func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { for { select { case <-ctx.Done(): - // TODO: log context error? + err := ctx.Err() + if err != nil { + logger.Debugf("peerset context finished with error: %w", err) + } return case <-ticker.C: for i := 0; i < ps.peerState.getSetLength(); i++ { From df15a5df9913f52dbcb249d5f92e97a4b8e79eaa Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 25 Apr 2022 10:37:14 +0200 Subject: [PATCH 45/50] chore: fixing the tests helpers --- dot/peerset/peerset_test.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index 33b58076e6..e285796201 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -239,7 +239,7 @@ func TestReAllocAfterBanned(t *testing.T) { checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) checkNodePeerMembershipState(t, ps.peerState, peer1, testSetID, notConnected) - n, exists := getNodePeer(t, ps.peerState, peer1) + n, exists := getNodePeer(ps.peerState, peer1) require.True(t, exists) // when the peer1 was banned we updated its lastConnected field to time.Now() @@ -251,14 +251,14 @@ func TestReAllocAfterBanned(t *testing.T) { // when calling Incoming method the peer1 is with status notConnectedPeer // so we update its lastConnected field to time.Now() again - n, exists = getNodePeer(t, ps.peerState, peer1) + n, exists = getNodePeer(ps.peerState, peer1) require.True(t, exists) currentLastTimeConnected := n.lastConnected[testSetID] require.True(t, lastTimeConnected.Before(currentLastTimeConnected)) // wait a bit for the node's reputation to go above the threshold. - <-time.After(allocTimeDuration + time.Second) + time.Sleep(allocTimeDuration + time.Second) checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) checkMessageStatus(t, <-ps.resultMsgCh, Connect) @@ -270,18 +270,18 @@ func TestRemovePeer(t *testing.T) { nil, false) ps := handler.peerSet - require.Equal(t, 2, len(ps.resultMsgCh)) + require.Len(t, ps.resultMsgCh, 2) for len(ps.resultMsgCh) != 0 { checkMessageStatus(t, <-ps.resultMsgCh, Connect) } - require.Equal(t, 2, len(ps.peerState.nodes)) + require.Len(t, ps.peerState.nodes, 2) checkPeerStateSetNumOut(t, ps.peerState, testSetID, 2) handler.RemovePeer(testSetID, discovered1, discovered2) time.Sleep(200 * time.Millisecond) - require.Equal(t, 2, len(ps.resultMsgCh)) + require.Len(t, ps.resultMsgCh, 2) for len(ps.resultMsgCh) != 0 { checkMessageStatus(t, <-ps.resultMsgCh, Drop) } @@ -296,12 +296,13 @@ func TestSetReservePeer(t *testing.T) { true) ps := handler.peerSet - require.Equal(t, 2, len(ps.resultMsgCh)) + require.Len(t, ps.resultMsgCh, 2) + for len(ps.resultMsgCh) != 0 { checkMessageStatus(t, <-ps.resultMsgCh, Connect) } - require.Equal(t, 2, len(ps.reservedNode)) + require.Len(t, ps.reservedNode, 2) newRsrPeerSet := peer.IDSlice{reservedPeer, peer.ID("newRsrPeer")} // add newRsrPeer but remove reservedPeer2 @@ -314,9 +315,7 @@ func TestSetReservePeer(t *testing.T) { } } -func getNodePeer(t *testing.T, ps *PeersState, pid peer.ID) (node, bool) { - t.Helper() - +func getNodePeer(ps *PeersState, pid peer.ID) (node, bool) { ps.RLock() defer ps.RUnlock() @@ -362,6 +361,8 @@ func checkReservedNodePeerExists(t *testing.T, ps *PeerSet, pid peer.ID) { } func checkPeerIsInNoSlotsNode(t *testing.T, ps *PeersState, pid peer.ID, setID int) { + t.Helper() + ps.RLock() defer ps.RUnlock() @@ -370,6 +371,8 @@ func checkPeerIsInNoSlotsNode(t *testing.T, ps *PeersState, pid peer.ID, setID i } func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNumOut uint32) { + t.Helper() + ps.RLock() defer ps.RUnlock() @@ -378,6 +381,8 @@ func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNu } func checkPeerStateSetNumIn(t *testing.T, ps *PeersState, setID int, expectedNumIn uint32) { + t.Helper() + ps.RLock() defer ps.RUnlock() @@ -386,6 +391,8 @@ func checkPeerStateSetNumIn(t *testing.T, ps *PeersState, setID int, expectedNum } func checkPeerStateNodeCount(t *testing.T, ps *PeersState, expectedCount int) { + t.Helper() + ps.RLock() defer ps.RUnlock() @@ -393,6 +400,8 @@ func checkPeerStateNodeCount(t *testing.T, ps *PeersState, expectedCount int) { } func checkPeerSetReservedNodeCount(t *testing.T, ps *PeerSet, expectedCount int) { + t.Helper() + ps.reservedLock.RLock() defer ps.reservedLock.RUnlock() From 3480177d1f9272e9396ec01869e377fa4e33a457 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 5 May 2022 08:38:54 -0400 Subject: [PATCH 46/50] chore: address comment, asserting log format types --- dot/network/service.go | 7 ++++--- dot/peerset/peerset.go | 6 +++--- dot/peerset/peerstate.go | 1 - dot/peerset/test_helpers.go | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dot/network/service.go b/dot/network/service.go index e4ed7be4e0..6374bdb9ea 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -488,18 +488,19 @@ func (s *Service) Stop() error { } // check if closeCh is closed, if not, close it. +mainloop: for { select { case _, hasMore := <-s.closeCh: if !hasMore { - return nil + break mainloop } - case <-s.closeCh: - return nil default: close(s.closeCh) } } + + return nil } // RegisterNotificationsProtocol registers a protocol with the network service with the given handler diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 044c7858d2..5562416326 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -159,7 +159,7 @@ type ReputationChange struct { Reason string } -func (r *ReputationChange) String() string { +func (r ReputationChange) String() string { return fmt.Sprintf("value: %d, reason: %s", r.Value, r.Reason) } @@ -340,7 +340,7 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { for _, pid := range peers { rep, err := ps.peerState.addReputation(pid, change) if err != nil { - return fmt.Errorf("cannot add reputation (%s) to peer %s: %w", pid, change.String(), err) + return fmt.Errorf("cannot add reputation (%s) to peer %s: %w", change, pid, err) } if rep >= BannedThresholdValue { @@ -643,7 +643,7 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { default: err := state.tryAcceptIncoming(setID, pid) if err != nil { - logger.Errorf("cannot accept incomming peer %pid: %w", pid, err) + logger.Errorf("cannot accept incomming peer %s: %s", pid, err) ps.resultMsgCh <- Message{ Status: Reject, diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index 4b60da1eb5..1e08f6bc9d 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -302,7 +302,6 @@ func (ps *PeersState) addNoSlotNode(idx int, peerID peer.ID) error { node, has := ps.nodes[peerID] if !has { - fmt.Println("error here") return fmt.Errorf("%w: for peer id %s", ErrPeerDoesNotExist, peerID) } diff --git a/dot/peerset/test_helpers.go b/dot/peerset/test_helpers.go index 36b88a368f..1100b00ecb 100644 --- a/dot/peerset/test_helpers.go +++ b/dot/peerset/test_helpers.go @@ -28,6 +28,7 @@ const ( const allocTimeDuration = 2 * time.Second //go:generate mockgen -destination=mock_message_processor_test.go -package $GOPACKAGE . MessageProcessor + func newTestPeerSet(t *testing.T, maxIn, maxOut uint32, bootNodes, reservedPeers []peer.ID, reservedOnly bool) *Handler { t.Helper() From 233d2ebed45b28fd895050ca1a62de42b901491c Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 5 May 2022 08:40:22 -0400 Subject: [PATCH 47/50] chore: removing diffs --- dot/peerset/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dot/peerset/handler.go b/dot/peerset/handler.go index 24dfee1847..4e0fd35f7a 100644 --- a/dot/peerset/handler.go +++ b/dot/peerset/handler.go @@ -12,10 +12,10 @@ import ( // Handler manages peerSet. type Handler struct { actionQueue chan<- action + peerSet *PeerSet closeCh chan struct{} - cancelCtx context.CancelFunc - peerSet *PeerSet + cancelCtx context.CancelFunc } // NewPeerSetHandler creates a new *peerset.Handler. From 9315906e9c480ec77b8a6f24756ce99c974560d2 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 5 May 2022 09:09:42 -0400 Subject: [PATCH 48/50] chore: removing diffs, fix logger format --- dot/peerset/peerset.go | 48 ++++++++++++++-------------------- dot/peerset/peerset_test.go | 52 ++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 55 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 5562416326..5a1b3f944b 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -502,7 +502,6 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { setID: uint64(setID), PeerID: peerID, } - } } @@ -574,15 +573,15 @@ func (ps *PeerSet) removePeer(setID int, peers ...peer.ID) error { // disconnect and forget err := ps.peerState.disconnect(setID, pid) if err != nil { - return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, setID, err) + return fmt.Errorf("cannot disconnect: %w", err) } if err = ps.peerState.forgetPeer(setID, pid); err != nil { - return fmt.Errorf("cannot forget peer %s from set %d: %w", pid, setID, err) + return fmt.Errorf("cannot forget peer: %w", err) } } else if status == notConnectedPeer { if err := ps.peerState.forgetPeer(setID, pid); err != nil { - return fmt.Errorf("cannot forget peer %s from set %d: %w", pid, setID, err) + return fmt.Errorf("cannot forget peer: %w", err) } } } @@ -632,34 +631,26 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { } state.RUnlock() + message := Message{ + setID: uint64(setID), + PeerID: pid, + } + switch { case nodeReputation < BannedThresholdValue: - ps.resultMsgCh <- Message{ - Status: Reject, - setID: uint64(setID), - PeerID: pid, - } - + message.Status = Reject default: err := state.tryAcceptIncoming(setID, pid) if err != nil { logger.Errorf("cannot accept incomming peer %s: %s", pid, err) - - ps.resultMsgCh <- Message{ - Status: Reject, - setID: uint64(setID), - PeerID: pid, - } + message.Status = Reject } else { logger.Debugf("incoming connection accepted from peer %s", pid) - - ps.resultMsgCh <- Message{ - Status: Accept, - setID: uint64(setID), - PeerID: pid, - } + message.Status = Accept } } + + ps.resultMsgCh <- message } return nil @@ -697,7 +688,7 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e state.nodes[pid] = n if err = state.disconnect(setIdx, pid); err != nil { - return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, setIdx, err) + return fmt.Errorf("cannot disconnect: %w", err) } ps.resultMsgCh <- Message{ @@ -709,7 +700,7 @@ func (ps *PeerSet) disconnect(setIdx int, reason DropReason, peers ...peer.ID) e // TODO: figure out the condition of connection refuse. if reason == RefusedDrop { if err = ps.removePeer(setIdx, pid); err != nil { - return fmt.Errorf("cannot remove peer %s at set %d: %w", pid, setIdx, err) + return fmt.Errorf("cannot remove peer: %w", err) } } } @@ -732,7 +723,6 @@ func (ps *PeerSet) listenAction(ctx context.Context) { case <-ctx.Done(): return case act, ok := <-ps.actionQueue: - if !ok { return } @@ -783,13 +773,13 @@ func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { case <-ctx.Done(): err := ctx.Err() if err != nil { - logger.Debugf("peerset context finished with error: %w", err) + logger.Debugf("peerset slot allocation exiting: %s", err) } return case <-ticker.C: - for i := 0; i < ps.peerState.getSetLength(); i++ { - if err := ps.allocSlots(i); err != nil { - logger.Warnf("failed to do action on peerSet: %s", err) + for setID := 0; setID < ps.peerState.getSetLength(); setID++ { + if err := ps.allocSlots(setID); err != nil { + logger.Warnf("failed to allocate slots: %s", err) } } } diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index e285796201..b7b5942814 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -11,9 +11,9 @@ import ( "github.com/stretchr/testify/require" ) -const testSetID = 0 - func TestBanRejectAcceptPeer(t *testing.T) { + const testSetID = 0 + t.Parallel() handler := newTestPeerSet(t, 25, 25, nil, nil, false) @@ -65,6 +65,8 @@ func TestBanRejectAcceptPeer(t *testing.T) { } func TestAddReservedPeers(t *testing.T) { + const testSetID = 0 + t.Parallel() handler := newTestPeerSet(t, 0, 2, []peer.ID{bootNode}, []peer.ID{}, false) ps := handler.peerSet @@ -75,26 +77,17 @@ func TestAddReservedPeers(t *testing.T) { checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) checkPeerStateSetNumOut(t, ps.peerState, testSetID, 1) - reservedPeers := []struct { - peerID peer.ID - }{ - { - peerID: reservedPeer, - }, - { - peerID: reservedPeer2, - }, - } + reservedPeers := peer.IDSlice{reservedPeer, reservedPeer2} - for _, tt := range reservedPeers { - handler.AddReservedPeer(testSetID, tt.peerID) + for _, peerID := range reservedPeers { + handler.AddReservedPeer(testSetID, peerID) time.Sleep(time.Millisecond * 100) - checkReservedNodePeerExists(t, ps, tt.peerID) - checkPeerIsInNoSlotsNode(t, ps.peerState, tt.peerID, testSetID) + checkReservedNodePeerExists(t, ps, peerID) + checkPeerIsInNoSlotsNode(t, ps.peerState, peerID, testSetID) - require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, tt.peerID)) - checkNodePeerMembershipState(t, ps.peerState, tt.peerID, testSetID, outgoing) + require.Equal(t, connectedPeer, ps.peerState.peerStatus(testSetID, peerID)) + checkNodePeerMembershipState(t, ps.peerState, peerID, testSetID, outgoing) // peers in noSlotNodes maps should not increase the // numIn and numOut count @@ -120,9 +113,10 @@ func TestAddReservedPeers(t *testing.T) { } func TestPeerSetIncoming(t *testing.T) { + const testSetID = 0 + t.Parallel() - handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, - []peer.ID{}, false) + handler := newTestPeerSet(t, 2, 1, []peer.ID{bootNode}, []peer.ID{}, false) ps := handler.peerSet checkMessageStatus(t, <-ps.resultMsgCh, Connect) @@ -182,6 +176,8 @@ func TestPeerSetIncoming(t *testing.T) { } func TestPeerSetDiscovered(t *testing.T) { + const testSetID = 0 + t.Parallel() handler := newTestPeerSet(t, 0, 2, []peer.ID{}, []peer.ID{reservedPeer}, false) @@ -213,6 +209,8 @@ func TestPeerSetDiscovered(t *testing.T) { } func TestReAllocAfterBanned(t *testing.T) { + const testSetID = 0 + t.Parallel() handler := newTestPeerSet(t, 25, 25, []peer.ID{}, []peer.ID{}, false) @@ -265,9 +263,10 @@ func TestReAllocAfterBanned(t *testing.T) { } func TestRemovePeer(t *testing.T) { + const testSetID = 0 + t.Parallel() - handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, - nil, false) + handler := newTestPeerSet(t, 0, 2, []peer.ID{discovered1, discovered2}, nil, false) ps := handler.peerSet require.Len(t, ps.resultMsgCh, 2) @@ -291,9 +290,10 @@ func TestRemovePeer(t *testing.T) { } func TestSetReservePeer(t *testing.T) { + const testSetID = 0 + t.Parallel() - handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, - true) + handler := newTestPeerSet(t, 0, 2, nil, []peer.ID{reservedPeer, reservedPeer2}, true) ps := handler.peerSet require.Len(t, ps.resultMsgCh, 2) @@ -376,7 +376,7 @@ func checkPeerStateSetNumOut(t *testing.T, ps *PeersState, setID int, expectedNu ps.RLock() defer ps.RUnlock() - gotNumOut := ps.sets[testSetID].numOut + gotNumOut := ps.sets[setID].numOut require.Equal(t, expectedNumOut, gotNumOut) } @@ -386,7 +386,7 @@ func checkPeerStateSetNumIn(t *testing.T, ps *PeersState, setID int, expectedNum ps.RLock() defer ps.RUnlock() - gotNumIn := ps.sets[testSetID].numIn + gotNumIn := ps.sets[setID].numIn require.Equal(t, expectedNumIn, gotNumIn) } From cb763ab30a549658f0a0efff4f2044e4bbcdc911 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 5 May 2022 10:35:53 -0400 Subject: [PATCH 49/50] chore: remove unneeded time.Sleep --- dot/peerset/peerset_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dot/peerset/peerset_test.go b/dot/peerset/peerset_test.go index b7b5942814..d0bb8a52dd 100644 --- a/dot/peerset/peerset_test.go +++ b/dot/peerset/peerset_test.go @@ -40,7 +40,6 @@ func TestBanRejectAcceptPeer(t *testing.T) { // report peer will disconnect the peer and set the `lastConnected` to time.Now handler.ReportPeer(rpc, peer1) - time.Sleep(100 * time.Millisecond) checkMessageStatus(t, <-ps.resultMsgCh, Drop) checkPeerStateSetNumIn(t, ps.peerState, testSetID, 0) From 53987c1289c557941f9c0e97cc9431f7adc94423 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 5 May 2022 10:55:22 -0400 Subject: [PATCH 50/50] chore: address error wraping --- dot/peerset/peerset.go | 22 +++++++++------------- dot/peerset/peerstate.go | 4 +--- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/dot/peerset/peerset.go b/dot/peerset/peerset.go index 5a1b3f944b..df19ec2d51 100644 --- a/dot/peerset/peerset.go +++ b/dot/peerset/peerset.go @@ -318,7 +318,7 @@ func (ps *PeerSet) updateTime() error { // forget peer removes the peer from the list of members of the set. err = ps.peerState.forgetPeer(set, peerID) if err != nil { - return fmt.Errorf("cannot forget peer %s from set %d: %w", peerID, set, err) + return fmt.Errorf("cannot forget peer: %w", err) } } } @@ -340,7 +340,7 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { for _, pid := range peers { rep, err := ps.peerState.addReputation(pid, change) if err != nil { - return fmt.Errorf("cannot add reputation (%s) to peer %s: %w", change, pid, err) + return fmt.Errorf("cannot add reputation: %w", err) } if rep >= BannedThresholdValue { @@ -356,7 +356,7 @@ func (ps *PeerSet) reportPeer(change ReputationChange, peers ...peer.ID) error { // disconnect peer err = ps.peerState.disconnect(i, pid) if err != nil { - return fmt.Errorf("cannot disconnect peer %s at set %d: %w", pid, i, err) + return fmt.Errorf("cannot disconnect: %w", err) } ps.resultMsgCh <- Message{ @@ -392,7 +392,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { node, err := ps.peerState.getNode(reservePeer) if err != nil { - return fmt.Errorf("cannot get node using peer id %s: %w", reservePeer, err) + return fmt.Errorf("cannot get node: %w", err) } if node.reputation < BannedThresholdValue { @@ -402,7 +402,7 @@ func (ps *PeerSet) allocSlots(setIdx int) error { } if err = peerState.tryOutgoing(setIdx, reservePeer); err != nil { - return fmt.Errorf("cannot set peer %s from set %d as outgoing: %w", reservePeer, setIdx, err) + return fmt.Errorf("cannot set as outgoing: %w", err) } ps.resultMsgCh <- Message{ @@ -494,7 +494,7 @@ func (ps *PeerSet) removeReservedPeers(setID int, peers ...peer.ID) error { if ps.peerState.peerStatus(setID, peerID) == connectedPeer { err := ps.peerState.disconnect(setID, peerID) if err != nil { - return fmt.Errorf("cannot disconnect peer %s at set %d: %w", peerID, setID, err) + return fmt.Errorf("cannot disconnect: %w", err) } ps.resultMsgCh <- Message{ @@ -636,10 +636,9 @@ func (ps *PeerSet) incoming(setID int, peers ...peer.ID) error { PeerID: pid, } - switch { - case nodeReputation < BannedThresholdValue: + if nodeReputation < BannedThresholdValue { message.Status = Reject - default: + } else { err := state.tryAcceptIncoming(setID, pid) if err != nil { logger.Errorf("cannot accept incomming peer %s: %s", pid, err) @@ -771,10 +770,7 @@ func (ps *PeerSet) periodicallyAllocateSlots(ctx context.Context) { for { select { case <-ctx.Done(): - err := ctx.Err() - if err != nil { - logger.Debugf("peerset slot allocation exiting: %s", err) - } + logger.Debugf("peerset slot allocation exiting: %s", ctx.Err()) return case <-ticker.C: for setID := 0; setID < ps.peerState.getSetLength(); setID++ { diff --git a/dot/peerset/peerstate.go b/dot/peerset/peerstate.go index 1e08f6bc9d..a8f627d801 100644 --- a/dot/peerset/peerstate.go +++ b/dot/peerset/peerstate.go @@ -254,13 +254,11 @@ func (ps *PeersState) addReputation(peerID peer.ID, change ReputationChange) ( } // highestNotConnectedPeer returns the peer with the highest Reputation and that we are not connected to. -func (ps *PeersState) highestNotConnectedPeer(set int) peer.ID { +func (ps *PeersState) highestNotConnectedPeer(set int) (highestPeerID peer.ID) { ps.RLock() defer ps.RUnlock() maxRep := math.MinInt32 - var highestPeerID peer.ID - for peerID, node := range ps.nodes { if node.state[set] != notConnected { continue