From 83409c3aff383eb5b08afdc3523574a41805957f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 10 Jan 2024 11:11:39 -0800 Subject: [PATCH 01/57] refactors duplicate string tracker --- network/p2p/inspector/validation/utils.go | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/network/p2p/inspector/validation/utils.go b/network/p2p/inspector/validation/utils.go index 0e57a9dd345..84792b9a70a 100644 --- a/network/p2p/inspector/validation/utils.go +++ b/network/p2p/inspector/validation/utils.go @@ -1,12 +1,25 @@ package validation -type duplicateStrTracker map[string]struct{} +// duplicateStrTracker is a map of strings to the number of times they have been tracked. +// It is a non-concurrent map, so it should only be used in a single goroutine. +// It is used to track duplicate strings. +type duplicateStrTracker map[string]uint -func (d duplicateStrTracker) set(s string) { - d[s] = struct{}{} -} +// track stores the string and returns the number of times it has been tracked and whether it is a duplicate. +// If the string has not been tracked before, it is stored with a count of 1. +// If the string has been tracked before, the count is incremented. +// Args: +// +// s: the string to track +// +// Returns: +// The number of times this string has been tracked, e.g., 1 if it is the first time, 2 if it is the second time, etc. +func (d duplicateStrTracker) track(s string) uint { + if _, ok := d[s]; ok { + d[s]++ + } else { + d[s] = 1 + } -func (d duplicateStrTracker) isDuplicate(s string) bool { - _, ok := d[s] - return ok + return d[s] } From ff71dee47b68a8713095fc3028fd7f578e99183f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 10 Jan 2024 11:27:26 -0800 Subject: [PATCH 02/57] refactors duplicate string tracker --- network/p2p/inspector/validation/utils_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/network/p2p/inspector/validation/utils_test.go b/network/p2p/inspector/validation/utils_test.go index 14015dcc621..d12fbada848 100644 --- a/network/p2p/inspector/validation/utils_test.go +++ b/network/p2p/inspector/validation/utils_test.go @@ -6,11 +6,17 @@ import ( "github.com/stretchr/testify/require" ) -// TestDuplicateStrTracker ensures duplicateStrTracker performs simple set and returns expected result for duplicate str. -func TestDuplicateStrTracker(t *testing.T) { +// TestDuplicateStringTracker tests the duplicateStrTracker.track function. +func TestDuplicateStringTracker(t *testing.T) { tracker := make(duplicateStrTracker) - s := "hello world" - require.False(t, tracker.isDuplicate(s)) - tracker.set(s) - require.True(t, tracker.isDuplicate(s)) + require.Equal(t, uint(1), tracker.track("test1")) + require.Equal(t, uint(2), tracker.track("test1")) + + // tracking a new string, 3 times + require.Equal(t, uint(1), tracker.track("test2")) + require.Equal(t, uint(2), tracker.track("test2")) + require.Equal(t, uint(3), tracker.track("test2")) + + // tracking an empty string + require.Equal(t, uint(1), tracker.track("")) } From 9de01c8b1ffa86fe08eec27381169a96dad58132 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 10 Jan 2024 12:34:53 -0800 Subject: [PATCH 03/57] refactors errors to reflect the number of duplicate messages --- network/p2p/inspector/validation/errors.go | 33 ++++++++++++++++------ 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/network/p2p/inspector/validation/errors.go b/network/p2p/inspector/validation/errors.go index b08af405a23..9db439309f3 100644 --- a/network/p2p/inspector/validation/errors.go +++ b/network/p2p/inspector/validation/errors.go @@ -54,8 +54,9 @@ func IsIWantCacheMissThresholdErr(err error) bool { // DuplicateTopicErr error that indicates a duplicate has been detected. This can be duplicate topic or message ID tracking. type DuplicateTopicErr struct { - topic string - msgType p2pmsg.ControlMessageType + topic string // the topic that is duplicated + count uint // the number of times the topic has been duplicated + msgType p2pmsg.ControlMessageType // the control message type that the topic was found in } func (e DuplicateTopicErr) Error() string { @@ -63,8 +64,18 @@ func (e DuplicateTopicErr) Error() string { } // NewDuplicateTopicErr returns a new DuplicateTopicErr. -func NewDuplicateTopicErr(topic string, msgType p2pmsg.ControlMessageType) DuplicateTopicErr { - return DuplicateTopicErr{topic, msgType} +// Args: +// +// topic: the topic that is duplicated +// count: the number of times the topic has been duplicated +// msgType: the control message type that the topic was found in +// +// Returns: +// +// A new DuplicateTopicErr. +func NewDuplicateTopicErr(topic string, count uint, msgType p2pmsg.ControlMessageType) DuplicateTopicErr { + + return DuplicateTopicErr{topic, count, msgType} } // IsDuplicateTopicErr returns true if an error is DuplicateTopicErr. @@ -75,8 +86,9 @@ func IsDuplicateTopicErr(err error) bool { // DuplicateMessageIDErr error that indicates a duplicate message ID has been detected in a IHAVE or IWANT control message. type DuplicateMessageIDErr struct { - id string - msgType p2pmsg.ControlMessageType + id string // id of the message that is duplicated + count uint // the number of times the message ID has been duplicated + msgType p2pmsg.ControlMessageType // the control message type that the message ID was found in } func (e DuplicateMessageIDErr) Error() string { @@ -84,8 +96,13 @@ func (e DuplicateMessageIDErr) Error() string { } // NewDuplicateMessageIDErr returns a new DuplicateMessageIDErr. -func NewDuplicateMessageIDErr(id string, msgType p2pmsg.ControlMessageType) DuplicateMessageIDErr { - return DuplicateMessageIDErr{id, msgType} +// Args: +// +// id: id of the message that is duplicated +// count: the number of times the message ID has been duplicated +// msgType: the control message type that the message ID was found in. +func NewDuplicateMessageIDErr(id string, count uint, msgType p2pmsg.ControlMessageType) DuplicateMessageIDErr { + return DuplicateMessageIDErr{id, count, msgType} } // IsDuplicateMessageIDErr returns true if an error is DuplicateMessageIDErr. From fa9f0babeb4b428e5fc5f6a52e51accf44f97bd8 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 10 Jan 2024 12:38:16 -0800 Subject: [PATCH 04/57] extends gossipsub rpc inspector parameters --- .../p2p/config/gossipsub_rpc_inspectors.go | 86 +++++++++++++++---- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index 829cb33ca61..f44a9b3ee4f 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -17,31 +17,59 @@ type RpcInspectorParameters struct { // RpcValidationInspectorParameters keys. const ( - ClusterPrefixedMessageConfigKey = "cluster-prefixed-messages" - IWantConfigKey = "iwant" - IHaveConfigKey = "ihave" - QueueSizeKey = "queue-size" - GraftPruneMessageMaxSampleSizeKey = "graft-and-prune-message-max-sample-size" - MessageMaxSampleSizeKey = "message-max-sample-size" - MessageErrorThresholdKey = "error-threshold" + ClusterPrefixedMessageConfigKey = "cluster-prefixed-messages" + IWantConfigKey = "iwant" + IHaveConfigKey = "ihave" + GraftPruneKey = "graft-and-prune" + PublishMessagesConfigKey = "publish-messages" + InspectionQueueConfigKey = "inspection-queue" ) // RpcValidationInspector validation limits used for gossipsub RPC control message inspection. type RpcValidationInspector struct { ClusterPrefixedMessage ClusterPrefixedMessageInspectionParameters `mapstructure:"cluster-prefixed-messages"` - IWant IWantRPCInspectionParameters `mapstructure:"iwant"` + IWant IWantRpcInspectionParameters `mapstructure:"iwant"` IHave IHaveRpcInspectionParameters `mapstructure:"ihave"` + GraftPrune GraftPruneRpcInspectionParameters `mapstructure:"graft-and-prune"` + PublishMessages PublishMessageInspectionParameters `mapstructure:"publish-messages"` + InspectionQueue InspectionQueueParameters `mapstructure:"inspection-queue"` +} + +const ( + QueueSizeKey = "queue-size" +) + +type InspectionQueueParameters struct { // NumberOfWorkers number of worker pool workers. NumberOfWorkers int `validate:"gte=1" mapstructure:"workers"` - // QueueSize size of the queue used by worker pool for the control message validation inspector. - QueueSize uint32 `validate:"gte=100" mapstructure:"queue-size"` - // GraftPruneMessageMaxSampleSize the max sample size used for control message validation of GRAFT and PRUNE. If the total number of control messages (GRAFT or PRUNE) + // Size size of the queue used by worker pool for the control message validation inspector. + Size uint32 `validate:"gte=100" mapstructure:"queue-size"` +} + +const ( + MessageErrorThresholdKey = "error-threshold" +) + +type PublishMessageInspectionParameters struct { + // MaxSampleSize the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated. + MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` + // ErrorThreshold the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value. + ErrorThreshold int `validate:"gte=500" mapstructure:"error-threshold"` +} + +const ( + MaxTotalDuplicateTopicIdThreshold = "max-total-duplicate-topic-id-threshold" +) + +type GraftPruneRpcInspectionParameters struct { + // MaxSampleSize the max sample size used for control message validation of GRAFT and PRUNE. If the total number of control messages (GRAFT or PRUNE) // exceeds this max sample size then the respective message will be truncated to this value before being processed. - GraftPruneMessageMaxSampleSize int `validate:"gte=1000" mapstructure:"graft-and-prune-message-max-sample-size"` - // RPCMessageMaxSampleSize the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated. - MessageMaxSampleSize int `validate:"gte=1000" mapstructure:"message-max-sample-size"` - // RPCMessageErrorThreshold the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value. - MessageErrorThreshold int `validate:"gte=500" mapstructure:"error-threshold"` + MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` + + // MaxTotalDuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in a single GRAFT or PRUNE message under inspection. + // Ideally, a GRAFT or PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. + // When the total number of duplicate topic ids in a single GRAFT or PRUNE message exceeds this threshold, the inspection of message will fail. + MaxTotalDuplicateTopicIdThreshold uint `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` } const ( @@ -52,8 +80,8 @@ const ( DuplicateMsgIDThresholdKey = "duplicate-message-id-threshold" ) -// IWantRPCInspectionParameters contains the "numerical values" for the iwant rpc control message inspection. -type IWantRPCInspectionParameters struct { +// IWantRpcInspectionParameters contains the "numerical values" for the iwant rpc control message inspection. +type IWantRpcInspectionParameters struct { // MaxSampleSize max inspection sample size to use. If the total number of iWant control messages // exceeds this max sample size then the respective message will be truncated before being processed. MaxSampleSize uint `validate:"gt=0" mapstructure:"max-sample-size"` @@ -70,6 +98,12 @@ type IWantRPCInspectionParameters struct { DuplicateMsgIDThreshold float64 `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` } +const ( + DuplicateTopicIdThresholdKey = "duplicate-topic-id-threshold" + MaxTotalDuplicateTopicIdThresholdKey = "max-total-duplicate-topic-id-threshold" + MaxTotalDuplicateMessageIdThresholdKey = "max-total-duplicate-message-id-threshold" +) + // IHaveRpcInspectionParameters contains the "numerical values" for ihave rpc control inspection. type IHaveRpcInspectionParameters struct { // MaxSampleSize max inspection sample size to use. If the number of ihave messages exceeds this configured value @@ -78,6 +112,22 @@ type IHaveRpcInspectionParameters struct { // MaxMessageIDSampleSize max inspection sample size to use for iHave message ids. Each ihave message includes a list of message ids // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` + + // DuplicateTopicIdThreshold is the threshold for considering the repeated topic IDs in a single iHave message as a duplicate. + // For example, if the threshold is 2, a maximum of two duplicate topic ids will be allowed in a single iHave message. + // This is to allow GossipSub protocol send iHave messages in batches without consolidating the topic IDs. + DuplicateTopicIdThreshold uint `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` + + // MaxTotalDuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in an iHave message under inspection. + // When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. + // Note that a topic ID is counted as a duplicate only if it is repeated more than DuplicateTopicIdThreshold times. + MaxTotalDuplicateTopicIdThreshold uint `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` + + // MaxTotalDuplicateMessageIdThreshold is the threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. + // When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + // Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once + // within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + MaxTotalDuplicateMessageIdThreshold uint `validate:"gte=0" mapstructure:"max-total-duplicate-message-id-threshold"` } const ( From 5a2c26381b27dc8dc7b8e404aaf1746ab686039b Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 10 Jan 2024 12:38:52 -0800 Subject: [PATCH 05/57] refactors tracker logic in rpc tracker --- .../control_message_validation_inspector.go | 93 ++++++++++++------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 5556edf685d..cc471bb2141 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -116,10 +116,10 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid return nil, fmt.Errorf("failed to create cluster prefix topics received tracker") } - if params.Config.MessageMaxSampleSize < params.Config.MessageErrorThreshold { + if params.Config.PublishMessages.MaxSampleSize < params.Config.PublishMessages.ErrorThreshold { return nil, fmt.Errorf("rpc message max sample size must be greater than or equal to rpc message error threshold, got %d and %d respectively", - params.Config.MessageMaxSampleSize, - params.Config.MessageErrorThreshold) + params.Config.PublishMessages.MaxSampleSize, + params.Config.PublishMessages.ErrorThreshold) } c := &ControlMsgValidationInspector{ @@ -135,7 +135,7 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid topicOracle: params.TopicOracle, } - store := queue.NewHeroStore(params.Config.QueueSize, params.Logger, inspectMsgQueueCacheCollector) + store := queue.NewHeroStore(params.Config.InspectionQueue.Size, params.Logger, inspectMsgQueueCacheCollector) pool := worker.NewWorkerPoolBuilder[*InspectRPCRequest](lg, store, c.processInspectRPCReq).Build() @@ -158,7 +158,7 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid <-c.distributor.Done() c.logger.Debug().Msg("rpc inspector distributor shutdown complete") }) - for i := 0; i < c.config.NumberOfWorkers; i++ { + for i := 0; i < c.config.InspectionQueue.NumberOfWorkers; i++ { builder.AddWorker(pool.WorkerLogic()) } c.Component = builder.Build() @@ -326,13 +326,18 @@ func (c *ControlMsgValidationInspector) checkPubsubMessageSender(message *pubsub // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { - tracker := make(duplicateStrTracker) + duplicateTopicTracker := make(duplicateStrTracker) + totalDuplicateTopicIds := uint(0) for _, graft := range grafts { topic := channels.Topic(graft.GetTopicID()) - if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType + if duplicateTopicTracker.track(topic.String()) > 1 { + // ideally, a GRAFT message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. + totalDuplicateTopicIds++ + // check if the total number of duplicates exceeds the configured threshold. + if totalDuplicateTopicIds > c.config.GraftPrune.MaxTotalDuplicateTopicIdThreshold { + return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType + } } - tracker.set(topic.String()) err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { return err, ctrlMsgType @@ -353,12 +358,17 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { tracker := make(duplicateStrTracker) + totalDuplicateTopicIds := uint(0) for _, prune := range prunes { topic := channels.Topic(prune.GetTopicID()) - if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgPrune), p2p.CtrlMsgNonClusterTopicType + if tracker.track(topic.String()) > 1 { + // ideally, a PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. + totalDuplicateTopicIds++ + // check if the total number of duplicates exceeds the configured threshold. + if totalDuplicateTopicIds > c.config.GraftPrune.MaxTotalDuplicateTopicIdThreshold { + return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgPrune), p2p.CtrlMsgNonClusterTopicType + } } - tracker.set(topic.String()) err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { return err, ctrlMsgType @@ -388,28 +398,42 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave Logger() duplicateTopicTracker := make(duplicateStrTracker) duplicateMessageIDTracker := make(duplicateStrTracker) + totalDuplicateTopicIds := uint(0) + totalDuplicateMessageIds := uint(0) totalMessageIds := 0 for _, ihave := range ihaves { messageIds := ihave.GetMessageIDs() topic := ihave.GetTopicID() - if duplicateTopicTracker.isDuplicate(topic) { - return NewDuplicateTopicErr(topic, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType - } - duplicateTopicTracker.set(topic) + + // first check if the topic is valid, fail fast if it is not err, ctrlMsgType := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) if err != nil { return err, ctrlMsgType } + // then track the topic ensuring it is not beyond a duplicate threshold. + if dupCnt := duplicateTopicTracker.track(topic); dupCnt > c.config.IHave.DuplicateTopicIdThreshold { + totalDuplicateTopicIds++ + // the topic is duplicated, check if the total number of duplicates exceeds the configured threshold + if totalDuplicateTopicIds > c.config.IHave.MaxTotalDuplicateTopicIdThreshold { + return NewDuplicateTopicErr(topic, dupCnt, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType + } + } + for _, messageID := range messageIds { - if duplicateMessageIDTracker.isDuplicate(messageID) { - return NewDuplicateTopicErr(messageID, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType + if dupCnt := duplicateMessageIDTracker.track(messageID); dupCnt > 1 { + totalDuplicateMessageIds++ + // the message is duplicated, check if the total number of duplicates exceeds the configured threshold + if totalDuplicateMessageIds > c.config.IHave.MaxTotalDuplicateMessageIdThreshold { + return NewDuplicateMessageIDErr(messageID, totalDuplicateMessageIds, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType + } } - duplicateMessageIDTracker.set(messageID) } } lg.Debug(). Int("total_message_ids", totalMessageIds). + Uint("total_duplicate_topic_ids", totalDuplicateTopicIds). + Uint("total_duplicate_message_ids", totalDuplicateMessageIds). Msg("ihave control message validation complete") return nil, p2p.CtrlMsgNonClusterTopicType } @@ -436,10 +460,12 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant Int64("last_highest_ihave_rpc_size", lastHighest). Logger() sampleSize := uint(len(iWants)) - tracker := make(duplicateStrTracker) + duplicateMsgIdTracker := make(duplicateStrTracker) cacheMisses := 0 + // keeps track of the total duplicate message ids found in the sample; a message id is considered duplicate if it appears + // more than a configurable threshold count in the sample. + totalDuplicateMessageIds := 0 allowedCacheMissesThreshold := float64(sampleSize) * c.config.IWant.CacheMissThreshold - duplicates := 0 allowedDuplicatesThreshold := float64(sampleSize) * c.config.IWant.DuplicateMsgIDThreshold checkCacheMisses := len(iWants) >= c.config.IWant.CacheMissCheckSize lg = lg.With(). @@ -455,10 +481,11 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant messageIDCount := uint(len(messageIds)) for _, messageID := range messageIds { // check duplicate allowed threshold - if tracker.isDuplicate(messageID) { - duplicates++ - if float64(duplicates) > allowedDuplicatesThreshold { - return NewIWantDuplicateMsgIDThresholdErr(duplicates, messageIDCount, c.config.IWant.DuplicateMsgIDThreshold) + if duplicateMsgIdTracker.track(messageID) > 1 { + // ideally, an iWant message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once. + totalDuplicateMessageIds++ + if float64(totalDuplicateMessageIds) > allowedDuplicatesThreshold { + return NewIWantDuplicateMsgIDThresholdErr(totalDuplicateMessageIds, messageIDCount, c.config.IWant.DuplicateMsgIDThreshold) } } // check cache miss threshold @@ -470,7 +497,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant } } } - tracker.set(messageID) + duplicateMsgIdTracker.track(messageID) totalMessageIds++ } } @@ -478,7 +505,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant lg.Debug(). Int("total_message_ids", totalMessageIds). Int("cache_misses", cacheMisses). - Int("duplicates", duplicates). + Int("total_duplicate_message_ids", totalDuplicateMessageIds). Msg("iwant control message validation complete") return nil @@ -501,7 +528,7 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, if totalMessages == 0 { return nil, 0 } - sampleSize := c.config.MessageMaxSampleSize + sampleSize := c.config.PublishMessages.MaxSampleSize if sampleSize > totalMessages { sampleSize = totalMessages } @@ -544,7 +571,7 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, } // return an error when we exceed the error threshold - if errs != nil && errs.Len() > c.config.MessageErrorThreshold { + if errs != nil && errs.Len() > c.config.PublishMessages.ErrorThreshold { return NewInvalidRpcPublishMessagesErr(errs.ErrorOrNil(), errs.Len()), uint64(errs.Len()) } @@ -580,12 +607,12 @@ func (c *ControlMsgValidationInspector) truncateRPC(from peer.ID, rpc *pubsub.RP func (c *ControlMsgValidationInspector) truncateGraftMessages(rpc *pubsub.RPC) { grafts := rpc.GetControl().GetGraft() originalGraftSize := len(grafts) - if originalGraftSize <= c.config.GraftPruneMessageMaxSampleSize { + if originalGraftSize <= c.config.GraftPrune.MaxSampleSize { return // nothing to truncate } // truncate grafts and update metrics - sampleSize := c.config.GraftPruneMessageMaxSampleSize + sampleSize := c.config.GraftPrune.MaxSampleSize c.performSample(p2pmsg.CtrlMsgGraft, uint(originalGraftSize), uint(sampleSize), func(i, j uint) { grafts[i], grafts[j] = grafts[j], grafts[i] }) @@ -600,11 +627,11 @@ func (c *ControlMsgValidationInspector) truncateGraftMessages(rpc *pubsub.RPC) { func (c *ControlMsgValidationInspector) truncatePruneMessages(rpc *pubsub.RPC) { prunes := rpc.GetControl().GetPrune() originalPruneSize := len(prunes) - if originalPruneSize <= c.config.GraftPruneMessageMaxSampleSize { + if originalPruneSize <= c.config.GraftPrune.MaxSampleSize { return // nothing to truncate } - sampleSize := c.config.GraftPruneMessageMaxSampleSize + sampleSize := c.config.GraftPrune.MaxSampleSize c.performSample(p2pmsg.CtrlMsgPrune, uint(originalPruneSize), uint(sampleSize), func(i, j uint) { prunes[i], prunes[j] = prunes[j], prunes[i] }) From 6bf67e8e7763c6a8b026b0969b79c83bd90ff6a1 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 10 Jan 2024 12:51:13 -0800 Subject: [PATCH 06/57] fixes default config --- config/default-config.yml | 48 +++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index 85a1ba18c02..69770645e9e 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -136,23 +136,45 @@ network-config: # The size of the queue for notifications about invalid RPC messages notification-cache-size: 10_000 validation: # RPC control message validation inspector configs - # Rpc validation inspector number of pool workers - workers: 5 - # The size of the queue used by worker pool for the control message validation inspector - queue-size: 100 - # The max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated - message-max-sample-size: 1000 - # Max number of control messages in a sample to be inspected when inspecting GRAFT and PRUNE message types. If the total number of control messages (GRAFT or PRUNE) - # exceeds this max sample size then the respective message will be truncated before being processed. - graft-and-prune-message-max-sample-size: 1000 - # The threshold at which an error will be returned if the number of invalid RPC messages exceeds this value - error-threshold: 500 + inspection-queue: + # Rpc validation inspector number of pool workers + workers: 5 + # The size of the queue used by worker pool for the control message validation inspector + queue-size: 100 + publish-messages: + # The max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated + max-sample-size: 1000 + # The threshold at which an error will be returned if the number of invalid RPC messages exceeds this value + error-threshold: 500 + graft-and-prune: # Max number of control messages in a sample to be inspected when inspecting GRAFT and PRUNE message types. If the total number of control messages (GRAFT or PRUNE) + # exceeds this max sample size then the respective message will be truncated before being processed. + message-max-sample-size: 1000 + # Maximum number of total duplicate topic ids in a single GRAFT or PRUNE message, ideally this should be 0 but we allow for some tolerance + # to avoid penalizing peers that are not malicious but are misbehaving due to bugs or other issues. + # A topic id is considered duplicate if it appears more than once in a single GRAFT or PRUNE message. + max-total-duplicate-topic-id-threshold: 10 ihave: # Max number of ihave messages in a sample to be inspected. If the number of ihave messages exceeds this configured value # the control message ihaves will be truncated to the max sample size. This sample is randomly selected. max-sample-size: 1000 # Max number of ihave message ids in a sample to be inspected per ihave. Each ihave message includes a list of message ids # each. If the size of the message ids list for a single ihave message exceeds the configured max message id sample size the list of message ids will be truncated. max-message-id-sample-size: 1000 + # the threshold for considering the repeated topic IDs in a single iHave message as a duplicate. + # For example, if the threshold is 2, a maximum of two duplicate topic ids will be allowed in a single iHave message. + # This is to allow GossipSub protocol send iHave messages in batches without consolidating the topic IDs. + # Recommended value is to be somewhere between 10 (moderate) to 100 (aggressive) depending on the message traffic. + duplicate-topic-id-threshold: 10 + # The tolerance threshold for having duplicate topics in an iHave message under inspection. + # When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. + # Note that a topic ID is counted as a duplicate only if it is repeated more than (duplicate-topic-id-threshold) times. + # We currently have `duplicate-topic-id-threshold = 10` meaning that a topic id is considered duplicate if it is repeated more than 10 times in a single iHave message. + # Setting `max-total-duplicate-topic-id-threshold = 10` means that if we have an overall 100 (10 * 10) repeated topic ids in a single iHave message, the message will fail inspection. + max-total-duplicate-topic-id-threshold: 10 + # Threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. + # When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + # Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once + # within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + max-total-duplicate-message-id-threshold: 10 iwant: # Max number of iwant messages in a sample to be inspected. If the total number of iWant control messages # exceeds this max sample size then the respective message will be truncated before being processed. @@ -165,8 +187,10 @@ network-config: cache-miss-threshold: .5 # The iWants size at which message id cache misses will be checked. cache-miss-check-size: 1000 - # The max allowed duplicate message IDs in a single iWant control message. If the duplicate message threshold is exceeded an invalid control message + # The max allowed FRACTION of duplicate message IDs in a single iWant control message. If the duplicate message threshold is exceeded an invalid control message # notification is disseminated and the sender will be penalized. + # note that ideally there should be no duplicate message ids in a single iwant message but we allow for some tolerance to avoid penalizing peers that are not malicious + # but are misbehaving due to bugs or other issues. Hence, the recommended value is a small FRACTION of the total message ids in a single iwant message. duplicate-message-id-threshold: .15 cluster-prefixed-messages: # Cluster prefixed control message validation configs From 1c8d7d0eee739ace218ec6bcb86180777b643728 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 10:51:36 -0800 Subject: [PATCH 07/57] fixes flag names --- network/netconf/flags.go | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/network/netconf/flags.go b/network/netconf/flags.go index f782e4e6eaf..510394c368f 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -83,22 +83,22 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerQueueCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerNumOfWorkersKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.NumberOfWorkersKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.QueueSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.NumberOfWorkersKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.QueueSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.TrackerCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.TrackerCacheDecayKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.HardThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.NotificationCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneMessageMaxSampleSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissCheckSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageMaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageErrorThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MaxSampleSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.CacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.AppSpecificScoreRegistryKey, p2pconfig.ScoreUpdateWorkerNumKey), @@ -184,11 +184,11 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerNumOfWorkersKey), config.GossipSub.RpcTracer.RpcSentTrackerNumOfWorkers, "number of workers for the rpc sent tracker worker pool.") // gossipsub RPC control message validation limits used for validation configuration and rate limiting - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.NumberOfWorkersKey), - config.GossipSub.RpcInspector.Validation.NumberOfWorkers, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.NumberOfWorkersKey), + config.GossipSub.RpcInspector.Validation.InspectionQueue.NumberOfWorkers, "number of gossipsub RPC control message validation inspector component workers") - flags.Uint32(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.QueueSizeKey), - config.GossipSub.RpcInspector.Validation.QueueSize, + flags.Uint32(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.QueueSizeKey), + config.GossipSub.RpcInspector.Validation.InspectionQueue.Size, "queue size for gossipsub RPC validation inspector events worker pool queue.") flags.Uint32(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.TrackerCacheSizeKey), config.GossipSub.RpcInspector.Validation.ClusterPrefixedMessage.ControlMsgsReceivedCacheSize, @@ -234,8 +234,8 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), config.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize, "max number of message ids to sample when performing validation per ihave") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneMessageMaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.GraftPruneMessageMaxSampleSize, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), + config.GossipSub.RpcInspector.Validation.GraftPrune.MaxSampleSize, "max number of control messages to sample when performing validation on GRAFT and PRUNE message types") flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), config.GossipSub.RpcInspector.Validation.IWant.MaxSampleSize, @@ -252,12 +252,15 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), config.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold, "max allowed duplicate message IDs in a single iWant control message") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageMaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.MessageMaxSampleSize, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MaxSampleSizeKey), + config.GossipSub.RpcInspector.Validation.PublishMessages.MaxSampleSize, "the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageErrorThresholdKey), - config.GossipSub.RpcInspector.Validation.MessageErrorThreshold, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), + config.GossipSub.RpcInspector.Validation.PublishMessages.ErrorThreshold, "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") + flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold, + "the max allowed duplicate topic IDs in a single iHave control message") flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), config.GossipSub.SubscriptionProvider.UpdateInterval, "interval for updating the list of subscribed topics for all peers in the gossipsub, recommended value is a few minutes") From bee18ea700eb7958f9bf81ceae99e753b0ea31a2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 10:51:45 -0800 Subject: [PATCH 08/57] fixes config name mismatch --- config/default-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default-config.yml b/config/default-config.yml index 69770645e9e..e34753ef1d8 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -148,7 +148,7 @@ network-config: error-threshold: 500 graft-and-prune: # Max number of control messages in a sample to be inspected when inspecting GRAFT and PRUNE message types. If the total number of control messages (GRAFT or PRUNE) # exceeds this max sample size then the respective message will be truncated before being processed. - message-max-sample-size: 1000 + max-sample-size: 1000 # Maximum number of total duplicate topic ids in a single GRAFT or PRUNE message, ideally this should be 0 but we allow for some tolerance # to avoid penalizing peers that are not malicious but are misbehaving due to bugs or other issues. # A topic id is considered duplicate if it appears more than once in a single GRAFT or PRUNE message. From 4c6b0d8424b8d08118112d63398bc797029277f6 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 11:03:11 -0800 Subject: [PATCH 09/57] fixes flag names --- network/netconf/flags.go | 15 ++++++++++++++- network/p2p/config/gossipsub_rpc_inspectors.go | 4 ---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 510394c368f..3b575367f1e 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -91,6 +91,10 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.NotificationCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateMessageIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), @@ -259,8 +263,17 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { config.GossipSub.RpcInspector.Validation.PublishMessages.ErrorThreshold, "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateTopicIdThreshold, + "the max allowed duplicate topic IDs across all iHave control messages in a single RPC message") + flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateMessageIdThresholdKey), config.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold, - "the max allowed duplicate topic IDs in a single iHave control message") + "the max allowed duplicate message ids in a single iHave control message") + flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, + "the number of times a topic ID can be repeated in iHave control messages across the same RPC message before it is considered a duplicate") + flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold, + "the max allowed duplicate topic IDs across all graft/prune control messages in a single RPC message") flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), config.GossipSub.SubscriptionProvider.UpdateInterval, "interval for updating the list of subscribed topics for all peers in the gossipsub, recommended value is a few minutes") diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index f44a9b3ee4f..2060d5fe243 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -57,10 +57,6 @@ type PublishMessageInspectionParameters struct { ErrorThreshold int `validate:"gte=500" mapstructure:"error-threshold"` } -const ( - MaxTotalDuplicateTopicIdThreshold = "max-total-duplicate-topic-id-threshold" -) - type GraftPruneRpcInspectionParameters struct { // MaxSampleSize the max sample size used for control message validation of GRAFT and PRUNE. If the total number of control messages (GRAFT or PRUNE) // exceeds this max sample size then the respective message will be truncated to this value before being processed. From 13e6b878fa91e875f53a733b50d881b3d543c563 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 11:08:40 -0800 Subject: [PATCH 10/57] lint fix --- .../control_message_validation_inspector_test.go | 10 +++++----- network/p2p/inspector/validation/errors_test.go | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index f9268224dee..7af19dbf9c9 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -86,7 +86,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("truncateGraftMessages should truncate graft messages as expected", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.GraftPruneMessageMaxSampleSize = graftPruneMessageMaxSampleSize + params.Config.GraftPrune.MaxSampleSize = graftPruneMessageMaxSampleSize }) // topic validation is ignored set any topic oracle distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() @@ -115,7 +115,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("truncatePruneMessages should truncate prune messages as expected", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.GraftPruneMessageMaxSampleSize = graftPruneMessageMaxSampleSize + params.Config.GraftPrune.MaxSampleSize = graftPruneMessageMaxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -553,7 +553,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured RpcMessageErrorThreshold", func(t *testing.T) { errThreshold := 500 inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.MessageErrorThreshold = errThreshold + params.Config.PublishMessages.ErrorThreshold = errThreshold }) // create unknown topic unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() @@ -593,7 +593,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when subscription missing for topic", func(t *testing.T) { errThreshold := 500 inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.MessageErrorThreshold = errThreshold + params.Config.PublishMessages.ErrorThreshold = errThreshold }) pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) from := unittest.PeerIdFixture(t) @@ -611,7 +611,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { errThreshold := 500 inspector, signalerCtx, cancel, distributor, _, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { // 5 invalid pubsub messages will force notification dissemination - params.Config.MessageErrorThreshold = errThreshold + params.Config.PublishMessages.ErrorThreshold = errThreshold }) pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, "") rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) diff --git a/network/p2p/inspector/validation/errors_test.go b/network/p2p/inspector/validation/errors_test.go index b042dd98bcf..9374c578534 100644 --- a/network/p2p/inspector/validation/errors_test.go +++ b/network/p2p/inspector/validation/errors_test.go @@ -30,7 +30,7 @@ func TestErrActiveClusterIDsNotSetRoundTrip(t *testing.T) { // TestErrDuplicateTopicRoundTrip ensures correct error formatting for DuplicateTopicErr. func TestDuplicateTopicErrRoundTrip(t *testing.T) { expectedErrorMsg := fmt.Sprintf("duplicate topic found in %s control message type: %s", p2pmsg.CtrlMsgGraft, channels.TestNetworkChannel) - err := NewDuplicateTopicErr(channels.TestNetworkChannel.String(), p2pmsg.CtrlMsgGraft) + err := NewDuplicateTopicErr(channels.TestNetworkChannel.String(), uint(8), p2pmsg.CtrlMsgGraft) assert.Equal(t, expectedErrorMsg, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateTopicErr(err), "IsDuplicateTopicErr should return true for DuplicateTopicErr error") @@ -44,11 +44,11 @@ func TestDuplicateMessageIDErrRoundTrip(t *testing.T) { msgID := "flow-1804flkjnafo" expectedErrMsg1 := fmt.Sprintf("duplicate message ID foud in %s control message type: %s", p2pmsg.CtrlMsgIHave, msgID) expectedErrMsg2 := fmt.Sprintf("duplicate message ID foud in %s control message type: %s", p2pmsg.CtrlMsgIWant, msgID) - err := NewDuplicateMessageIDErr(msgID, p2pmsg.CtrlMsgIHave) + err := NewDuplicateMessageIDErr(msgID, uint(8), p2pmsg.CtrlMsgIHave) assert.Equal(t, expectedErrMsg1, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateMessageIDErr(err), "IsDuplicateMessageIDErr should return true for DuplicateMessageIDErr error") - err = NewDuplicateMessageIDErr(msgID, p2pmsg.CtrlMsgIWant) + err = NewDuplicateMessageIDErr(msgID, uint(8), p2pmsg.CtrlMsgIWant) assert.Equal(t, expectedErrMsg2, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateMessageIDErr(err), "IsDuplicateMessageIDErr should return true for DuplicateMessageIDErr error") From 23a8f988e4bdad8b17e85e3a2ec4c56f709bb749 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 11:11:13 -0800 Subject: [PATCH 11/57] lint fix --- network/p2p/inspector/validation/utils.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/network/p2p/inspector/validation/utils.go b/network/p2p/inspector/validation/utils.go index 84792b9a70a..42fb4ed377e 100644 --- a/network/p2p/inspector/validation/utils.go +++ b/network/p2p/inspector/validation/utils.go @@ -15,11 +15,10 @@ type duplicateStrTracker map[string]uint // Returns: // The number of times this string has been tracked, e.g., 1 if it is the first time, 2 if it is the second time, etc. func (d duplicateStrTracker) track(s string) uint { - if _, ok := d[s]; ok { - d[s]++ - } else { - d[s] = 1 + if _, ok := d[s]; !ok { + d[s] = 0 } + d[s]++ return d[s] } From 6a6955563a5261ac8ea7e31d0beb6d164dc92389 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 12:30:54 -0800 Subject: [PATCH 12/57] switches uints to ints --- network/netconf/flags.go | 8 ++++---- network/p2p/config/gossipsub_rpc_inspectors.go | 8 ++++---- .../control_message_validation_inspector.go | 14 +++++++------- network/p2p/inspector/validation/errors.go | 8 ++++---- network/p2p/inspector/validation/errors_test.go | 6 +++--- network/p2p/inspector/validation/utils.go | 4 ++-- network/p2p/inspector/validation/utils_test.go | 12 ++++++------ 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 3b575367f1e..877857ccb85 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -262,16 +262,16 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), config.GossipSub.RpcInspector.Validation.PublishMessages.ErrorThreshold, "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") - flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), config.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateTopicIdThreshold, "the max allowed duplicate topic IDs across all iHave control messages in a single RPC message") - flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateMessageIdThresholdKey), + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateMessageIdThresholdKey), config.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold, "the max allowed duplicate message ids in a single iHave control message") - flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, "the number of times a topic ID can be repeated in iHave control messages across the same RPC message before it is considered a duplicate") - flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), config.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold, "the max allowed duplicate topic IDs across all graft/prune control messages in a single RPC message") flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index 2060d5fe243..59e2b2ca7ac 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -65,7 +65,7 @@ type GraftPruneRpcInspectionParameters struct { // MaxTotalDuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in a single GRAFT or PRUNE message under inspection. // Ideally, a GRAFT or PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. // When the total number of duplicate topic ids in a single GRAFT or PRUNE message exceeds this threshold, the inspection of message will fail. - MaxTotalDuplicateTopicIdThreshold uint `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` + MaxTotalDuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` } const ( @@ -112,18 +112,18 @@ type IHaveRpcInspectionParameters struct { // DuplicateTopicIdThreshold is the threshold for considering the repeated topic IDs in a single iHave message as a duplicate. // For example, if the threshold is 2, a maximum of two duplicate topic ids will be allowed in a single iHave message. // This is to allow GossipSub protocol send iHave messages in batches without consolidating the topic IDs. - DuplicateTopicIdThreshold uint `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` + DuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` // MaxTotalDuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in an iHave message under inspection. // When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. // Note that a topic ID is counted as a duplicate only if it is repeated more than DuplicateTopicIdThreshold times. - MaxTotalDuplicateTopicIdThreshold uint `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` + MaxTotalDuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` // MaxTotalDuplicateMessageIdThreshold is the threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. // When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. // Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once // within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. - MaxTotalDuplicateMessageIdThreshold uint `validate:"gte=0" mapstructure:"max-total-duplicate-message-id-threshold"` + MaxTotalDuplicateMessageIdThreshold int `validate:"gte=0" mapstructure:"max-total-duplicate-message-id-threshold"` } const ( diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index cc471bb2141..115dd8d5844 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -327,7 +327,7 @@ func (c *ControlMsgValidationInspector) checkPubsubMessageSender(message *pubsub // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { duplicateTopicTracker := make(duplicateStrTracker) - totalDuplicateTopicIds := uint(0) + totalDuplicateTopicIds := 0 for _, graft := range grafts { topic := channels.Topic(graft.GetTopicID()) if duplicateTopicTracker.track(topic.String()) > 1 { @@ -358,7 +358,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { tracker := make(duplicateStrTracker) - totalDuplicateTopicIds := uint(0) + totalDuplicateTopicIds := 0 for _, prune := range prunes { topic := channels.Topic(prune.GetTopicID()) if tracker.track(topic.String()) > 1 { @@ -398,9 +398,9 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave Logger() duplicateTopicTracker := make(duplicateStrTracker) duplicateMessageIDTracker := make(duplicateStrTracker) - totalDuplicateTopicIds := uint(0) - totalDuplicateMessageIds := uint(0) totalMessageIds := 0 + totalDuplicateTopicIds := 0 + totalDuplicateMessageIds := 0 for _, ihave := range ihaves { messageIds := ihave.GetMessageIDs() topic := ihave.GetTopicID() @@ -421,7 +421,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave } for _, messageID := range messageIds { - if dupCnt := duplicateMessageIDTracker.track(messageID); dupCnt > 1 { + if duplicateMessageIDTracker.track(messageID) > 1 { totalDuplicateMessageIds++ // the message is duplicated, check if the total number of duplicates exceeds the configured threshold if totalDuplicateMessageIds > c.config.IHave.MaxTotalDuplicateMessageIdThreshold { @@ -432,8 +432,8 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave } lg.Debug(). Int("total_message_ids", totalMessageIds). - Uint("total_duplicate_topic_ids", totalDuplicateTopicIds). - Uint("total_duplicate_message_ids", totalDuplicateMessageIds). + Int("total_duplicate_topic_ids", totalDuplicateTopicIds). + Int("total_duplicate_message_ids", totalDuplicateMessageIds). Msg("ihave control message validation complete") return nil, p2p.CtrlMsgNonClusterTopicType } diff --git a/network/p2p/inspector/validation/errors.go b/network/p2p/inspector/validation/errors.go index 9db439309f3..0215e18be27 100644 --- a/network/p2p/inspector/validation/errors.go +++ b/network/p2p/inspector/validation/errors.go @@ -55,7 +55,7 @@ func IsIWantCacheMissThresholdErr(err error) bool { // DuplicateTopicErr error that indicates a duplicate has been detected. This can be duplicate topic or message ID tracking. type DuplicateTopicErr struct { topic string // the topic that is duplicated - count uint // the number of times the topic has been duplicated + count int // the number of times the topic has been duplicated msgType p2pmsg.ControlMessageType // the control message type that the topic was found in } @@ -73,7 +73,7 @@ func (e DuplicateTopicErr) Error() string { // Returns: // // A new DuplicateTopicErr. -func NewDuplicateTopicErr(topic string, count uint, msgType p2pmsg.ControlMessageType) DuplicateTopicErr { +func NewDuplicateTopicErr(topic string, count int, msgType p2pmsg.ControlMessageType) DuplicateTopicErr { return DuplicateTopicErr{topic, count, msgType} } @@ -87,7 +87,7 @@ func IsDuplicateTopicErr(err error) bool { // DuplicateMessageIDErr error that indicates a duplicate message ID has been detected in a IHAVE or IWANT control message. type DuplicateMessageIDErr struct { id string // id of the message that is duplicated - count uint // the number of times the message ID has been duplicated + count int // the number of times the message ID has been duplicated msgType p2pmsg.ControlMessageType // the control message type that the message ID was found in } @@ -101,7 +101,7 @@ func (e DuplicateMessageIDErr) Error() string { // id: id of the message that is duplicated // count: the number of times the message ID has been duplicated // msgType: the control message type that the message ID was found in. -func NewDuplicateMessageIDErr(id string, count uint, msgType p2pmsg.ControlMessageType) DuplicateMessageIDErr { +func NewDuplicateMessageIDErr(id string, count int, msgType p2pmsg.ControlMessageType) DuplicateMessageIDErr { return DuplicateMessageIDErr{id, count, msgType} } diff --git a/network/p2p/inspector/validation/errors_test.go b/network/p2p/inspector/validation/errors_test.go index 9374c578534..a180badf28f 100644 --- a/network/p2p/inspector/validation/errors_test.go +++ b/network/p2p/inspector/validation/errors_test.go @@ -30,7 +30,7 @@ func TestErrActiveClusterIDsNotSetRoundTrip(t *testing.T) { // TestErrDuplicateTopicRoundTrip ensures correct error formatting for DuplicateTopicErr. func TestDuplicateTopicErrRoundTrip(t *testing.T) { expectedErrorMsg := fmt.Sprintf("duplicate topic found in %s control message type: %s", p2pmsg.CtrlMsgGraft, channels.TestNetworkChannel) - err := NewDuplicateTopicErr(channels.TestNetworkChannel.String(), uint(8), p2pmsg.CtrlMsgGraft) + err := NewDuplicateTopicErr(channels.TestNetworkChannel.String(), 1, p2pmsg.CtrlMsgGraft) assert.Equal(t, expectedErrorMsg, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateTopicErr(err), "IsDuplicateTopicErr should return true for DuplicateTopicErr error") @@ -44,11 +44,11 @@ func TestDuplicateMessageIDErrRoundTrip(t *testing.T) { msgID := "flow-1804flkjnafo" expectedErrMsg1 := fmt.Sprintf("duplicate message ID foud in %s control message type: %s", p2pmsg.CtrlMsgIHave, msgID) expectedErrMsg2 := fmt.Sprintf("duplicate message ID foud in %s control message type: %s", p2pmsg.CtrlMsgIWant, msgID) - err := NewDuplicateMessageIDErr(msgID, uint(8), p2pmsg.CtrlMsgIHave) + err := NewDuplicateMessageIDErr(msgID, 1, p2pmsg.CtrlMsgIHave) assert.Equal(t, expectedErrMsg1, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateMessageIDErr(err), "IsDuplicateMessageIDErr should return true for DuplicateMessageIDErr error") - err = NewDuplicateMessageIDErr(msgID, uint(8), p2pmsg.CtrlMsgIWant) + err = NewDuplicateMessageIDErr(msgID, 1, p2pmsg.CtrlMsgIWant) assert.Equal(t, expectedErrMsg2, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateMessageIDErr(err), "IsDuplicateMessageIDErr should return true for DuplicateMessageIDErr error") diff --git a/network/p2p/inspector/validation/utils.go b/network/p2p/inspector/validation/utils.go index 42fb4ed377e..d84bdd0cbf2 100644 --- a/network/p2p/inspector/validation/utils.go +++ b/network/p2p/inspector/validation/utils.go @@ -3,7 +3,7 @@ package validation // duplicateStrTracker is a map of strings to the number of times they have been tracked. // It is a non-concurrent map, so it should only be used in a single goroutine. // It is used to track duplicate strings. -type duplicateStrTracker map[string]uint +type duplicateStrTracker map[string]int // track stores the string and returns the number of times it has been tracked and whether it is a duplicate. // If the string has not been tracked before, it is stored with a count of 1. @@ -14,7 +14,7 @@ type duplicateStrTracker map[string]uint // // Returns: // The number of times this string has been tracked, e.g., 1 if it is the first time, 2 if it is the second time, etc. -func (d duplicateStrTracker) track(s string) uint { +func (d duplicateStrTracker) track(s string) int { if _, ok := d[s]; !ok { d[s] = 0 } diff --git a/network/p2p/inspector/validation/utils_test.go b/network/p2p/inspector/validation/utils_test.go index d12fbada848..81f0ffca936 100644 --- a/network/p2p/inspector/validation/utils_test.go +++ b/network/p2p/inspector/validation/utils_test.go @@ -9,14 +9,14 @@ import ( // TestDuplicateStringTracker tests the duplicateStrTracker.track function. func TestDuplicateStringTracker(t *testing.T) { tracker := make(duplicateStrTracker) - require.Equal(t, uint(1), tracker.track("test1")) - require.Equal(t, uint(2), tracker.track("test1")) + require.Equal(t, 1, tracker.track("test1")) + require.Equal(t, 2, tracker.track("test1")) // tracking a new string, 3 times - require.Equal(t, uint(1), tracker.track("test2")) - require.Equal(t, uint(2), tracker.track("test2")) - require.Equal(t, uint(3), tracker.track("test2")) + require.Equal(t, 1, tracker.track("test2")) + require.Equal(t, 2, tracker.track("test2")) + require.Equal(t, 3, tracker.track("test2")) // tracking an empty string - require.Equal(t, uint(1), tracker.track("")) + require.Equal(t, 1, tracker.track("")) } From b5b34da0b7de48e0da60baa8a623ad9af67b872a Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 12:37:31 -0800 Subject: [PATCH 13/57] adds tests for ihave message id duplications --- ...ntrol_message_validation_inspector_test.go | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 7af19dbf9c9..dce4c0170d0 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -435,22 +435,61 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with duplicate message ids as expected", func(t *testing.T) { + t.Run("inspectIHaveMessages should NOT disseminate invalid control message notification for iHave messages when duplicate message ids are below the threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) // avoid unknown topics errors topicProviderOracle.UpdateTopics([]string{validTopic}) duplicateMsgID := unittest.IdentifierFixture() - msgIds := flow.IdentifierList{duplicateMsgID, duplicateMsgID, duplicateMsgID} + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + msgIds := flow.IdentifierList{} + // includes as many duplicates as allowed by the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold; i++ { + msgIds = append(msgIds, duplicateMsgID) + } + duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) + from := unittest.PeerIdFixture(t) + + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + stopInspector(t, cancel, inspector) + }) + + t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages when duplicate message ids are above the threshold", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) + duplicateMsgID := unittest.IdentifierFixture() + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + msgIds := flow.IdentifierList{} + // includes as many duplicates as beyond the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold+2; i++ { + msgIds = append(msgIds, duplicateMsgID) + } duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr, p2p.CtrlMsgNonClusterTopicType) + + // one notification should be disseminated for invalid messages when the number of duplicates exceeds the threshold + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateMessageIDErr, p2p.CtrlMsgNonClusterTopicType) distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) - // sleep for 1 second to ensure rpc's is processed + // TODO: this sleeps should be replaced with a queue size checker. time.Sleep(time.Second) stopInspector(t, cancel, inspector) }) From 807dae500d827490345fb8e2e0e0f54812c28f31 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 13:33:33 -0800 Subject: [PATCH 14/57] adds tests for graft and prune duplicate topic ids below threshold --- ...ntrol_message_validation_inspector_test.go | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index dce4c0170d0..3daa9a97998 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -320,34 +320,65 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("processInspectRPCReq should disseminate invalid control message notification for control messages with duplicate topics", func(t *testing.T) { + // duplicate graft topic ids beyond threshold should trigger invalid control message notification + t.Run("duplicate graft topic ids beyond threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) // avoid unknown topics errors topicProviderOracle.UpdateTopics([]string{duplicateTopic}) - // create control messages with duplicate topic - grafts := []*pubsub_pb.ControlGraft{unittest.P2PRPCGraftFixture(&duplicateTopic), unittest.P2PRPCGraftFixture(&duplicateTopic)} - prunes := []*pubsub_pb.ControlPrune{unittest.P2PRPCPruneFixture(&duplicateTopic), unittest.P2PRPCPruneFixture(&duplicateTopic)} - ihaves := []*pubsub_pb.ControlIHave{unittest.P2PRPCIHaveFixture(&duplicateTopic, unittest.IdentifierListFixture(20).Strings()...), - unittest.P2PRPCIHaveFixture(&duplicateTopic, unittest.IdentifierListFixture(20).Strings()...)} + var grafts []*pubsub_pb.ControlGraft + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold+2; i++ { + grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) + } from := unittest.PeerIdFixture(t) - duplicateTopicGraftsRpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) - duplicateTopicPrunesRpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) - duplicateTopicIHavesRpc := unittest.P2PRPCFixture(unittest.WithIHaves(ihaves...)) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(func(args mock.Arguments) { + rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") require.Equal(t, from, notification.PeerID) - require.Contains(t, []p2pmsg.ControlMessageType{p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave}, notification.MsgType) + require.Equal(t, p2pmsg.CtrlMsgGraft, notification.MsgType) require.True(t, validation.IsDuplicateTopicErr(notification.Error)) }) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + stopInspector(t, cancel, inspector) + }) - require.NoError(t, inspector.Inspect(from, duplicateTopicGraftsRpc)) - require.NoError(t, inspector.Inspect(from, duplicateTopicPrunesRpc)) - require.NoError(t, inspector.Inspect(from, duplicateTopicIHavesRpc)) + // duplicate prune topic ids beyond threshold should trigger invalid control message notification + t.Run("duplicate prune topic ids beyond threshold", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var prunes []*pubsub_pb.ControlPrune + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold+2; i++ { + prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, from, notification.PeerID) + require.Equal(t, p2pmsg.CtrlMsgPrune, notification.MsgType) + require.True(t, validation.IsDuplicateTopicErr(notification.Error)) + }) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) stopInspector(t, cancel, inspector) From e9d2e536f7fa93c5ae98f9dd203fe881d94bdac8 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 13:43:22 -0800 Subject: [PATCH 15/57] adds test for duplicate prune topic id above the threshold --- ...ntrol_message_validation_inspector_test.go | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 3daa9a97998..3bc4b689851 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -361,6 +361,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { var prunes []*pubsub_pb.ControlPrune cfg, err := config.DefaultConfig() require.NoError(t, err) + // we need threshold + 1 to trigger the invalid control message notification; as the first duplicate topic id is not counted for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold+2; i++ { prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) } @@ -384,6 +385,32 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) + // duplicate prune topic ids below threshold should NOT trigger invalid control message notification + t.Run("duplicate prune topic ids below threshold", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var prunes []*pubsub_pb.ControlPrune + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold; i++ { + prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + stopInspector(t, cancel, inspector) + }) + t.Run("inspectGraftMessages should disseminate invalid control message notification for invalid graft messages as expected", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) // create unknown topic From 8e3a70cc03aac74919306e259cf8ebdc3f0f85b4 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 13:48:14 -0800 Subject: [PATCH 16/57] adds test for duplicate graft topic id below the threshold --- ...ntrol_message_validation_inspector_test.go | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 3bc4b689851..27366fe3840 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -352,6 +352,32 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) + // duplicate graft topic ids below threshold should NOT trigger invalid control message notification + t.Run("duplicate graft topic ids below threshold", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var grafts []*pubsub_pb.ControlGraft + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold; i++ { + grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + stopInspector(t, cancel, inspector) + }) + // duplicate prune topic ids beyond threshold should trigger invalid control message notification t.Run("duplicate prune topic ids beyond threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) From ebf4b75b1248ccaa0f982c6ddc958a0dd3b3c723 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 13:57:11 -0800 Subject: [PATCH 17/57] removes threshold for individual topic ids --- config/default-config.yml | 18 +++++---------- network/netconf/flags.go | 18 +++++++-------- .../p2p/config/gossipsub_rpc_inspectors.go | 22 +++++++------------ .../control_message_validation_inspector.go | 12 +++++----- ...ntrol_message_validation_inspector_test.go | 12 +++++----- 5 files changed, 35 insertions(+), 47 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index e34753ef1d8..88fa48fd155 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -152,29 +152,23 @@ network-config: # Maximum number of total duplicate topic ids in a single GRAFT or PRUNE message, ideally this should be 0 but we allow for some tolerance # to avoid penalizing peers that are not malicious but are misbehaving due to bugs or other issues. # A topic id is considered duplicate if it appears more than once in a single GRAFT or PRUNE message. - max-total-duplicate-topic-id-threshold: 10 - ihave: # Max number of ihave messages in a sample to be inspected. If the number of ihave messages exceeds this configured value + duplicate-topic-id-threshold: 10 + ihave: + # Max number of ihave messages in a sample to be inspected. If the number of ihave messages exceeds this configured value # the control message ihaves will be truncated to the max sample size. This sample is randomly selected. max-sample-size: 1000 # Max number of ihave message ids in a sample to be inspected per ihave. Each ihave message includes a list of message ids # each. If the size of the message ids list for a single ihave message exceeds the configured max message id sample size the list of message ids will be truncated. max-message-id-sample-size: 1000 - # the threshold for considering the repeated topic IDs in a single iHave message as a duplicate. - # For example, if the threshold is 2, a maximum of two duplicate topic ids will be allowed in a single iHave message. - # This is to allow GossipSub protocol send iHave messages in batches without consolidating the topic IDs. - # Recommended value is to be somewhere between 10 (moderate) to 100 (aggressive) depending on the message traffic. - duplicate-topic-id-threshold: 10 # The tolerance threshold for having duplicate topics in an iHave message under inspection. # When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. - # Note that a topic ID is counted as a duplicate only if it is repeated more than (duplicate-topic-id-threshold) times. - # We currently have `duplicate-topic-id-threshold = 10` meaning that a topic id is considered duplicate if it is repeated more than 10 times in a single iHave message. - # Setting `max-total-duplicate-topic-id-threshold = 10` means that if we have an overall 100 (10 * 10) repeated topic ids in a single iHave message, the message will fail inspection. - max-total-duplicate-topic-id-threshold: 10 + # Note that a topic ID is counted as a duplicate only if it is repeated more than once. + duplicate-topic-id-threshold: 10 # Threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. # When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. # Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once # within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. - max-total-duplicate-message-id-threshold: 10 + duplicate-message-id-threshold: 10 iwant: # Max number of iwant messages in a sample to be inspected. If the total number of iWant control messages # exceeds this max sample size then the respective message will be truncated before being processed. diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 877857ccb85..106469d723a 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -91,10 +91,10 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.NotificationCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateMessageIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), @@ -262,17 +262,17 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), config.GossipSub.RpcInspector.Validation.PublishMessages.ErrorThreshold, "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateTopicIdThreshold, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, "the max allowed duplicate topic IDs across all iHave control messages in a single RPC message") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxTotalDuplicateMessageIdThresholdKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold, "the max allowed duplicate message ids in a single iHave control message") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, "the number of times a topic ID can be repeated in iHave control messages across the same RPC message before it is considered a duplicate") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxTotalDuplicateTopicIdThresholdKey), - config.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold, "the max allowed duplicate topic IDs across all graft/prune control messages in a single RPC message") flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), config.GossipSub.SubscriptionProvider.UpdateInterval, diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index 59e2b2ca7ac..b3a1b7024e4 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -62,10 +62,10 @@ type GraftPruneRpcInspectionParameters struct { // exceeds this max sample size then the respective message will be truncated to this value before being processed. MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` - // MaxTotalDuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in a single GRAFT or PRUNE message under inspection. + // DuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in a single GRAFT or PRUNE message under inspection. // Ideally, a GRAFT or PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. // When the total number of duplicate topic ids in a single GRAFT or PRUNE message exceeds this threshold, the inspection of message will fail. - MaxTotalDuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` + DuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` } const ( @@ -95,9 +95,8 @@ type IWantRpcInspectionParameters struct { } const ( - DuplicateTopicIdThresholdKey = "duplicate-topic-id-threshold" - MaxTotalDuplicateTopicIdThresholdKey = "max-total-duplicate-topic-id-threshold" - MaxTotalDuplicateMessageIdThresholdKey = "max-total-duplicate-message-id-threshold" + DuplicateTopicIdThresholdKey = "duplicate-topic-id-threshold" + DuplicateMessageIdThresholdKey = "duplicate-message-id-threshold" ) // IHaveRpcInspectionParameters contains the "numerical values" for ihave rpc control inspection. @@ -109,21 +108,16 @@ type IHaveRpcInspectionParameters struct { // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` - // DuplicateTopicIdThreshold is the threshold for considering the repeated topic IDs in a single iHave message as a duplicate. - // For example, if the threshold is 2, a maximum of two duplicate topic ids will be allowed in a single iHave message. - // This is to allow GossipSub protocol send iHave messages in batches without consolidating the topic IDs. - DuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` - - // MaxTotalDuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in an iHave message under inspection. + // DuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in an iHave message under inspection. // When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. // Note that a topic ID is counted as a duplicate only if it is repeated more than DuplicateTopicIdThreshold times. - MaxTotalDuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"max-total-duplicate-topic-id-threshold"` + DuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` - // MaxTotalDuplicateMessageIdThreshold is the threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. + // DuplicateMessageIdThreshold is the threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. // When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. // Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once // within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. - MaxTotalDuplicateMessageIdThreshold int `validate:"gte=0" mapstructure:"max-total-duplicate-message-id-threshold"` + DuplicateMessageIdThreshold int `validate:"gte=0" mapstructure:"duplicate-message-id-threshold"` } const ( diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 115dd8d5844..e7ef1e4292c 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -334,7 +334,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft // ideally, a GRAFT message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. totalDuplicateTopicIds++ // check if the total number of duplicates exceeds the configured threshold. - if totalDuplicateTopicIds > c.config.GraftPrune.MaxTotalDuplicateTopicIdThreshold { + if totalDuplicateTopicIds > c.config.GraftPrune.DuplicateTopicIdThreshold { return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType } } @@ -365,7 +365,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune // ideally, a PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. totalDuplicateTopicIds++ // check if the total number of duplicates exceeds the configured threshold. - if totalDuplicateTopicIds > c.config.GraftPrune.MaxTotalDuplicateTopicIdThreshold { + if totalDuplicateTopicIds > c.config.GraftPrune.DuplicateTopicIdThreshold { return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgPrune), p2p.CtrlMsgNonClusterTopicType } } @@ -412,11 +412,11 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave } // then track the topic ensuring it is not beyond a duplicate threshold. - if dupCnt := duplicateTopicTracker.track(topic); dupCnt > c.config.IHave.DuplicateTopicIdThreshold { + if duplicateTopicTracker.track(topic) > 1 { totalDuplicateTopicIds++ // the topic is duplicated, check if the total number of duplicates exceeds the configured threshold - if totalDuplicateTopicIds > c.config.IHave.MaxTotalDuplicateTopicIdThreshold { - return NewDuplicateTopicErr(topic, dupCnt, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType + if totalDuplicateTopicIds > c.config.IHave.DuplicateTopicIdThreshold { + return NewDuplicateTopicErr(topic, totalDuplicateTopicIds, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType } } @@ -424,7 +424,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave if duplicateMessageIDTracker.track(messageID) > 1 { totalDuplicateMessageIds++ // the message is duplicated, check if the total number of duplicates exceeds the configured threshold - if totalDuplicateMessageIds > c.config.IHave.MaxTotalDuplicateMessageIdThreshold { + if totalDuplicateMessageIds > c.config.IHave.DuplicateMessageIdThreshold { return NewDuplicateMessageIDErr(messageID, totalDuplicateMessageIds, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType } } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 27366fe3840..547f30c5695 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -329,7 +329,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { var grafts []*pubsub_pb.ControlGraft cfg, err := config.DefaultConfig() require.NoError(t, err) - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold+2; i++ { + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) } from := unittest.PeerIdFixture(t) @@ -361,7 +361,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { var grafts []*pubsub_pb.ControlGraft cfg, err := config.DefaultConfig() require.NoError(t, err) - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold; i++ { + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) } from := unittest.PeerIdFixture(t) @@ -388,7 +388,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) // we need threshold + 1 to trigger the invalid control message notification; as the first duplicate topic id is not counted - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold+2; i++ { + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) } from := unittest.PeerIdFixture(t) @@ -420,7 +420,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { var prunes []*pubsub_pb.ControlPrune cfg, err := config.DefaultConfig() require.NoError(t, err) - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.MaxTotalDuplicateTopicIdThreshold; i++ { + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) } from := unittest.PeerIdFixture(t) @@ -530,7 +530,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, err) msgIds := flow.IdentifierList{} // includes as many duplicates as allowed by the threshold - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold; i++ { + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold; i++ { msgIds = append(msgIds, duplicateMsgID) } duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) @@ -559,7 +559,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, err) msgIds := flow.IdentifierList{} // includes as many duplicates as beyond the threshold - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxTotalDuplicateMessageIdThreshold+2; i++ { + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold+2; i++ { msgIds = append(msgIds, duplicateMsgID) } duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) From 0b4043504e9c1e1a905c924a1e477243bf31e13c Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 14:16:46 -0800 Subject: [PATCH 18/57] simplifies test messages --- .../control_message_validation_inspector_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 547f30c5695..4f308cb7fdc 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -83,7 +83,7 @@ func TestNewControlMsgValidationInspector(t *testing.T) { // Message truncation for each control message type occurs when the count of control // messages exceeds the configured maximum sample size for that control message type. func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { - t.Run("truncateGraftMessages should truncate graft messages as expected", func(t *testing.T) { + t.Run("graft truncation", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { params.Config.GraftPrune.MaxSampleSize = graftPruneMessageMaxSampleSize @@ -112,7 +112,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("truncatePruneMessages should truncate prune messages as expected", func(t *testing.T) { + t.Run("prune truncation", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { params.Config.GraftPrune.MaxSampleSize = graftPruneMessageMaxSampleSize @@ -142,7 +142,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("truncateIHaveMessages should truncate iHave messages as expected", func(t *testing.T) { + t.Run("ihave message id truncation", func(t *testing.T) { maxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { params.Config.IHave.MaxSampleSize = maxSampleSize @@ -172,7 +172,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("truncateIHaveMessageIds should truncate iHave message ids as expected", func(t *testing.T) { + t.Run("ihave message ids truncation", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { params.Config.IHave.MaxMessageIDSampleSize = maxMessageIDSampleSize @@ -208,7 +208,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("truncateIWantMessages should truncate iWant messages as expected", func(t *testing.T) { + t.Run("iwant message truncation", func(t *testing.T) { maxSampleSize := uint(100) inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { params.Config.IWant.MaxSampleSize = maxSampleSize @@ -236,7 +236,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("truncateIWantMessageIds should truncate iWant message ids as expected", func(t *testing.T) { + t.Run("iwant message id truncation", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { params.Config.IWant.MaxMessageIDSampleSize = maxMessageIDSampleSize From 799109520f23fc80eae8d1edda6854c63eb577d1 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 11 Jan 2024 14:17:06 -0800 Subject: [PATCH 19/57] fixes flags --- network/netconf/flags.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 106469d723a..b7eaa9f9282 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -93,7 +93,6 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), @@ -238,6 +237,12 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), config.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize, "max number of message ids to sample when performing validation per ihave") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, + "the max allowed duplicate topic IDs across all iHave control messages in a single RPC message") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold, + "the max allowed duplicate message ids in a single iHave control message") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), config.GossipSub.RpcInspector.Validation.GraftPrune.MaxSampleSize, "max number of control messages to sample when performing validation on GRAFT and PRUNE message types") @@ -262,15 +267,6 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), config.GossipSub.RpcInspector.Validation.PublishMessages.ErrorThreshold, "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), - config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, - "the max allowed duplicate topic IDs across all iHave control messages in a single RPC message") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), - config.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold, - "the max allowed duplicate message ids in a single iHave control message") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), - config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, - "the number of times a topic ID can be repeated in iHave control messages across the same RPC message before it is considered a duplicate") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), config.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold, "the max allowed duplicate topic IDs across all graft/prune control messages in a single RPC message") From 833dfa41ac6b9a7088c13f7fd430b8fa8c3eadb3 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Fri, 12 Jan 2024 09:43:43 -0800 Subject: [PATCH 20/57] fixes lint issues with insecure --- .../validation_inspector_test.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index b66b6259535..b9771f6f173 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -47,7 +47,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation messageCount := 100 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 controlMessageCount := int64(1) count := atomic.NewUint64(0) @@ -180,7 +180,7 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 messageCount := 10 controlMessageCount := int64(1) @@ -290,7 +290,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 count := atomic.NewInt64(0) done := make(chan struct{}) @@ -394,7 +394,7 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { // set hard threshold to 0 so that in the case of invalid cluster ID // we force the inspector to return an error inspectorConfig.ClusterPrefixedMessage.HardThreshold = 0 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. @@ -503,7 +503,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 controlMessageCount := int64(10) count := atomic.NewInt64(0) @@ -591,7 +591,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 controlMessageCount := int64(10) count := atomic.NewInt64(0) @@ -677,7 +677,7 @@ func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { // set hard threshold to 0 so that in the case of invalid cluster ID // we force the inspector to return an error inspectorConfig.ClusterPrefixedMessage.HardThreshold = 0 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. @@ -774,7 +774,7 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation // force all cache miss checks inspectorConfig.IWant.CacheMissCheckSize = 1 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 inspectorConfig.IWant.CacheMissThreshold = .5 // set cache miss threshold to 50% messageCount := 1 controlMessageCount := int64(1) @@ -876,7 +876,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { flowConfig, err := config.DefaultConfig() require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 idProvider := mock.NewIdentityProvider(t) spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) @@ -969,7 +969,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { topicProvider.UpdateTopics(topics) // after 7 errors encountered disseminate a notification - inspectorConfig.MessageErrorThreshold = 6 + inspectorConfig.PublishMessages.ErrorThreshold = 6 require.NoError(t, err) corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) From ad1d8419192dbe7c404319f23090cc65ba4f3d3d Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Fri, 12 Jan 2024 11:32:37 -0800 Subject: [PATCH 21/57] revises config names and supports documents --- config/default-config.yml | 60 +++++++++------- .../test/gossipsub/scoring/ihave_spam_test.go | 8 +-- network/netconf/flags.go | 56 +++++++-------- .../p2p/config/gossipsub_rpc_inspectors.go | 72 +++++++++++++------ .../control_message_validation_inspector.go | 38 +++++----- ...ntrol_message_validation_inspector_test.go | 49 +++++++------ 6 files changed, 166 insertions(+), 117 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index 88fa48fd155..9e688554c1e 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -142,24 +142,32 @@ network-config: # The size of the queue used by worker pool for the control message validation inspector queue-size: 100 publish-messages: - # The max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated + # The maximum number of messages in a single RPC message that are randomly sampled for async inspection. + # When the size of a single RPC message exceeds this threshold, a random sample is taken for inspection, but the RPC message is not truncated. max-sample-size: 1000 # The threshold at which an error will be returned if the number of invalid RPC messages exceeds this value error-threshold: 500 - graft-and-prune: # Max number of control messages in a sample to be inspected when inspecting GRAFT and PRUNE message types. If the total number of control messages (GRAFT or PRUNE) - # exceeds this max sample size then the respective message will be truncated before being processed. - max-sample-size: 1000 + graft-and-prune: + # The maximum number of GRAFT or PRUNE messages in a single RPC message. + # When the total number of GRAFT or PRUNE messages in a single RPC message exceeds this threshold, + # a random sample of GRAFT or PRUNE messages will be taken and the RPC message will be truncated to this sample size. + message-count-threshold: 1000 # Maximum number of total duplicate topic ids in a single GRAFT or PRUNE message, ideally this should be 0 but we allow for some tolerance # to avoid penalizing peers that are not malicious but are misbehaving due to bugs or other issues. # A topic id is considered duplicate if it appears more than once in a single GRAFT or PRUNE message. duplicate-topic-id-threshold: 10 ihave: - # Max number of ihave messages in a sample to be inspected. If the number of ihave messages exceeds this configured value - # the control message ihaves will be truncated to the max sample size. This sample is randomly selected. - max-sample-size: 1000 - # Max number of ihave message ids in a sample to be inspected per ihave. Each ihave message includes a list of message ids - # each. If the size of the message ids list for a single ihave message exceeds the configured max message id sample size the list of message ids will be truncated. - max-message-id-sample-size: 1000 + # The maximum allowed number of iHave messages in a single RPC message. + # Each iHave message represents the list of message ids. When the total number of iHave messages + # in a single RPC message exceeds this threshold, a random sample of iHave messages will be taken and the RPC message will be truncated to this sample size. + # The sample size is equal to the configured message-count-threshold. + message-count-threshold: 1000 + # MessageIdCountThreshold is the maximum allowed number of message ids in a single iHave message. + # Each iHave message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + # that can be included in a single iHave message. When the total number of message ids in a single iHave message exceeds this threshold, + # a random sample of message ids will be taken and the iHave message will be truncated to this sample size. + # The sample size is equal to the configured message-id-count-threshold. + message-id-count-threshold: 1000 # The tolerance threshold for having duplicate topics in an iHave message under inspection. # When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. # Note that a topic ID is counted as a duplicate only if it is repeated more than once. @@ -170,22 +178,26 @@ network-config: # within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. duplicate-message-id-threshold: 10 iwant: - # Max number of iwant messages in a sample to be inspected. If the total number of iWant control messages - # exceeds this max sample size then the respective message will be truncated before being processed. - max-sample-size: 1000 - # Max number of iwant message ids in a sample to be inspected per iwant. Each iwant message includes a list of message ids - # each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - max-message-id-sample-size: 1000 - # The allowed threshold of iWant messages received without a corresponding tracked iHave message that was sent. If the cache miss threshold is exceeded an - # invalid control message notification is disseminated and the sender will be penalized. - cache-miss-threshold: .5 + # The maximum allowed number of iWant messages in a single RPC message. + # Each iWant message represents the list of message ids. When the total number of iWant messages + # in a single RPC message exceeds this threshold, a random sample of iWant messages will be taken and the RPC message will be truncated to this sample size. + # The sample size is equal to the configured message-count-threshold. + message-count-threshold: 1000 + # The maximum allowed number of message ids in a single iWant message. + # Each iWant message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + # that can be included in a single iWant message. When the total number of message ids in a single iWant message exceeds this threshold, + # a random sample of message ids will be taken and the iWant message will be truncated to this sample size. + # The sample size is equal to the configured message-id-count-threshold. + message-id-count-threshold: 1000 + # The allowed threshold of iWant messages received without a corresponding tracked iHave message that was sent. + # If the cache miss threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. + cache-miss-threshold: 500 # The iWants size at which message id cache misses will be checked. cache-miss-check-size: 1000 - # The max allowed FRACTION of duplicate message IDs in a single iWant control message. If the duplicate message threshold is exceeded an invalid control message - # notification is disseminated and the sender will be penalized. - # note that ideally there should be no duplicate message ids in a single iwant message but we allow for some tolerance to avoid penalizing peers that are not malicious - # but are misbehaving due to bugs or other issues. Hence, the recommended value is a small FRACTION of the total message ids in a single iwant message. - duplicate-message-id-threshold: .15 + # The max allowed number of duplicate message ids in a single iwant message. + # Note that ideally there should be no duplicate message ids in a single iwant message but + # we allow for some tolerance to avoid penalizing peers that are not malicious + duplicate-message-id-threshold: 10 cluster-prefixed-messages: # Cluster prefixed control message validation configs # The size of the cache used to track the amount of cluster prefixed topics received by peers diff --git a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go index e11951cc7dd..14456335d67 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go @@ -199,10 +199,10 @@ func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { conf, err := config.DefaultConfig() require.NoError(t, err) // overcompensate for RPC truncation - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize = 10000 - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize = 10000 - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize = 10000 - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageCountThreshold = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageIdCountThreshold = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageCountThreshold = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageIdCountThreshold = 10000 // we override the decay interval to 1 second so that the score is updated within 1 second intervals. conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second // score tracer interval is set to 500 milliseconds to speed up the test, it should be shorter than the heartbeat interval (1 second) of gossipsub to catch the score updates in time. diff --git a/network/netconf/flags.go b/network/netconf/flags.go index b7eaa9f9282..10eff5b468b 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -89,14 +89,14 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.TrackerCacheDecayKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.HardThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.NotificationCacheSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageCountThreshold), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageIdCountThreshold), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MessageCountThreshold), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageCountThreshold), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageIdCountThreshold), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissCheckSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), @@ -231,45 +231,45 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.SpamRecordCacheKey, p2pconfig.PenaltyDecayEvaluationPeriodKey), config.GossipSub.ScoringParameters.SpamRecordCache.PenaltyDecayEvaluationPeriod, "defines the period at which the decay for a spam record is okay to be adjusted.") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize, - "max number of ihaves to sample when performing validation") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize, - "max number of message ids to sample when performing validation per ihave") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageCountThreshold), + config.GossipSub.RpcInspector.Validation.IHave.MessageCountThreshold, + "threshold for the number of ihave control messages to accept on a single RPC message, if exceeded the RPC message will be sampled and truncated") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageIdCountThreshold), + config.GossipSub.RpcInspector.Validation.IHave.MessageIdCountThreshold, + "threshold for the number of message ids on a single ihave control message to accept, if exceeded the RPC message ids will be sampled and truncated") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, - "the max allowed duplicate topic IDs across all iHave control messages in a single RPC message") + "the max allowed duplicate topic IDs across all ihave control messages in a single RPC message, if exceeded a misbehavior report will be created") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), config.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold, - "the max allowed duplicate message ids in a single iHave control message") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.GraftPrune.MaxSampleSize, - "max number of control messages to sample when performing validation on GRAFT and PRUNE message types") - flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.MaxSampleSize, - "max number of iwants to sample when performing validation") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.MaxMessageIDSampleSize, - "max number of message ids to sample when performing validation per iwant") + "the max allowed duplicate message IDs in a single ihave control message, if exceeded a misbehavior report will be created") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MessageCountThreshold), + config.GossipSub.RpcInspector.Validation.GraftPrune.MessageCountThreshold, + "threshold for the number of graft or prune control messages to accept on a single RPC message, if exceeded the RPC message will be sampled and truncated") + flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageCountThreshold), + config.GossipSub.RpcInspector.Validation.IWant.MessageCountThreshold, + "threshold for the number of iwant control messages to accept on a single RPC message, if exceeded the RPC message will be sampled and truncated") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageIdCountThreshold), + config.GossipSub.RpcInspector.Validation.IWant.MessageIdCountThreshold, + "threshold for the number of message ids on a single iwant control message to accept, if exceeded the RPC message ids will be sampled and truncated") flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), config.GossipSub.RpcInspector.Validation.IWant.CacheMissThreshold, - "max number of iwants to sample when performing validation") + "max number of cache misses (untracked) allowed in a single iWant control message, if exceeded a misbehavior report will be created") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissCheckSizeKey), config.GossipSub.RpcInspector.Validation.IWant.CacheMissCheckSize, - "the iWants size at which message id cache misses will be checked") + "threshold for the size of iwant control message that triggers cache miss check") flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), config.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold, - "max allowed duplicate message IDs in a single iWant control message") + "max allowed duplicate message IDs in a single iWant control message, if exceeded a misbehavior report will be created") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MaxSampleSizeKey), config.GossipSub.RpcInspector.Validation.PublishMessages.MaxSampleSize, - "the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated") + "the max sample size for async validation of publish messages, if exceeded the message will be sampled for inspection, but is not truncated") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), config.GossipSub.RpcInspector.Validation.PublishMessages.ErrorThreshold, - "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") + "the max number of errors allowed in a (sampled) set of publish messages on a single rpc, if exceeded a misbehavior report will be created") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), config.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold, - "the max allowed duplicate topic IDs across all graft/prune control messages in a single RPC message") + "the max allowed duplicate topic IDs across all graft or prune control messages in a single RPC message, if exceeded a misbehavior report will be created") flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), config.GossipSub.SubscriptionProvider.UpdateInterval, "interval for updating the list of subscribed topics for all peers in the gossipsub, recommended value is a few minutes") diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index b3a1b7024e4..6e51090e716 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -39,6 +39,10 @@ const ( QueueSizeKey = "queue-size" ) +// InspectionQueueParameters contains the "numerical values" for the control message validation inspector. +// Incoming GossipSub RPCs are queued for async inspection by a worker pool. This worker pool is configured +// by the parameters in this struct. +// Each RPC has a number of "publish messages" accompanied by control messages. type InspectionQueueParameters struct { // NumberOfWorkers number of worker pool workers. NumberOfWorkers int `validate:"gte=1" mapstructure:"workers"` @@ -47,20 +51,29 @@ type InspectionQueueParameters struct { } const ( + MaxSampleSizeKey = "max-sample-size" MessageErrorThresholdKey = "error-threshold" ) +// PublishMessageInspectionParameters contains the "numerical values" for the publish control message inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of these publish messages. type PublishMessageInspectionParameters struct { - // MaxSampleSize the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated. - MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` + // MaxSampleSize is the maximum number of messages in a single RPC message that are randomly sampled for async inspection. + // When the size of a single RPC message exceeds this threshold, a random sample is taken for inspection, but the RPC message is not truncated. + MaxSampleSize int `validate:"gte=0" mapstructure:"max-sample-size"` // ErrorThreshold the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value. - ErrorThreshold int `validate:"gte=500" mapstructure:"error-threshold"` + ErrorThreshold int `validate:"gte=0" mapstructure:"error-threshold"` } +// GraftPruneRpcInspectionParameters contains the "numerical values" for the graft and prune control message inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of these graft and prune control messages. type GraftPruneRpcInspectionParameters struct { - // MaxSampleSize the max sample size used for control message validation of GRAFT and PRUNE. If the total number of control messages (GRAFT or PRUNE) - // exceeds this max sample size then the respective message will be truncated to this value before being processed. - MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` + // MessageCountThreshold is the maximum number of GRAFT or PRUNE messages in a single RPC message. + // When the total number of GRAFT or PRUNE messages in a single RPC message exceeds this threshold, + // a random sample of GRAFT or PRUNE messages will be taken and the RPC message will be truncated to this sample size. + MessageCountThreshold int `validate:"gte=0" mapstructure:"message-count-threshold"` // DuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in a single GRAFT or PRUNE message under inspection. // Ideally, a GRAFT or PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. @@ -69,21 +82,28 @@ type GraftPruneRpcInspectionParameters struct { } const ( - MaxSampleSizeKey = "max-sample-size" - MaxMessageIDSampleSizeKey = "max-message-id-sample-size" + MessageCountThreshold = "message-count-threshold" + MessageIdCountThreshold = "message-id-count-threshold" CacheMissThresholdKey = "cache-miss-threshold" CacheMissCheckSizeKey = "cache-miss-check-size" DuplicateMsgIDThresholdKey = "duplicate-message-id-threshold" ) -// IWantRpcInspectionParameters contains the "numerical values" for the iwant rpc control message inspection. +// IWantRpcInspectionParameters contains the "numerical values" for iwant rpc control inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of the iwant control messages. type IWantRpcInspectionParameters struct { - // MaxSampleSize max inspection sample size to use. If the total number of iWant control messages - // exceeds this max sample size then the respective message will be truncated before being processed. - MaxSampleSize uint `validate:"gt=0" mapstructure:"max-sample-size"` - // MaxMessageIDSampleSize max inspection sample size to use for iWant message ids. Each iWant message includes a list of message ids - // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` + // MessageCountThreshold is the maximum allowed number of iWant messages in a single RPC message. + // Each iWant message represents the list of message ids. When the total number of iWant messages + // in a single RPC message exceeds this threshold, a random sample of iWant messages will be taken and the RPC message will be truncated to this sample size. + // The sample size is equal to the configured MessageCountThreshold. + MessageCountThreshold uint `validate:"gt=0" mapstructure:"message-count-threshold"` + // MessageIdCountThreshold is the maximum allowed number of message ids in a single iWant message. + // Each iWant message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + // that can be included in a single iWant message. When the total number of message ids in a single iWant message exceeds this threshold, + // a random sample of message ids will be taken and the iWant message will be truncated to this sample size. + // The sample size is equal to the configured MessageIdCountThreshold. + MessageIdCountThreshold int `validate:"gte=0" mapstructure:"message-id-count-threshold"` // CacheMissThreshold the threshold of missing corresponding iHave messages for iWant messages received before an invalid control message notification is disseminated. // If the cache miss threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. CacheMissThreshold float64 `validate:"gt=0" mapstructure:"cache-miss-threshold"` @@ -100,13 +120,20 @@ const ( ) // IHaveRpcInspectionParameters contains the "numerical values" for ihave rpc control inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of the ihave control messages. type IHaveRpcInspectionParameters struct { - // MaxSampleSize max inspection sample size to use. If the number of ihave messages exceeds this configured value - // the control message ihaves will be truncated to the max sample size. This sample is randomly selected. - MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` - // MaxMessageIDSampleSize max inspection sample size to use for iHave message ids. Each ihave message includes a list of message ids - // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` + // MessageCountThreshold is the maximum allowed number of iHave messages in a single RPC message. + // Each iHave message represents the list of message ids for a specific topic. When the total number of iHave messages + // in a single RPC message exceeds this threshold, a random sample of iHave messages will be taken and the RPC message will be truncated to this sample size. + // The sample size is equal to the configured MessageCountThreshold. + MessageCountThreshold int `validate:"gte=0" mapstructure:"message-count-threshold"` + // MessageIdCountThreshold is the maximum allowed number of message ids in a single iHave message. + // Each iHave message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + // that can be included in a single iHave message. When the total number of message ids in a single iHave message exceeds this threshold, + // a random sample of message ids will be taken and the iHave message will be truncated to this sample size. + // The sample size is equal to the configured MessageIdCountThreshold. + MessageIdCountThreshold int `validate:"gte=0" mapstructure:"message-id-count-threshold"` // DuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in an iHave message under inspection. // When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. @@ -127,6 +154,9 @@ const ( ) // ClusterPrefixedMessageInspectionParameters contains the "numerical values" for cluster prefixed control message inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits for the inspection +// of messages (publish messages and control messages) that belongs to cluster prefixed topics. +// Cluster-prefixed topics are topics that are prefixed with the cluster ID of the node that published the message. type ClusterPrefixedMessageInspectionParameters struct { // HardThreshold the upper bound on the amount of cluster prefixed control messages that will be processed // before a node starts to get penalized. This allows LN nodes to process some cluster prefixed control messages during startup diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index e7ef1e4292c..d938bf75e08 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -394,7 +394,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). Int("sample_size", len(ihaves)). - Int("max_sample_size", c.config.IHave.MaxSampleSize). + Int("max_sample_size", c.config.IHave.MessageCountThreshold). Logger() duplicateTopicTracker := make(duplicateStrTracker) duplicateMessageIDTracker := make(duplicateStrTracker) @@ -456,7 +456,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant lastHighest := c.rpcTracker.LastHighestIHaveRPCSize() lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). - Uint("max_sample_size", c.config.IWant.MaxSampleSize). + Uint("max_sample_size", c.config.IWant.MessageCountThreshold). Int64("last_highest_ihave_rpc_size", lastHighest). Logger() sampleSize := uint(len(iWants)) @@ -607,12 +607,12 @@ func (c *ControlMsgValidationInspector) truncateRPC(from peer.ID, rpc *pubsub.RP func (c *ControlMsgValidationInspector) truncateGraftMessages(rpc *pubsub.RPC) { grafts := rpc.GetControl().GetGraft() originalGraftSize := len(grafts) - if originalGraftSize <= c.config.GraftPrune.MaxSampleSize { + if originalGraftSize <= c.config.GraftPrune.MessageCountThreshold { return // nothing to truncate } // truncate grafts and update metrics - sampleSize := c.config.GraftPrune.MaxSampleSize + sampleSize := c.config.GraftPrune.MessageCountThreshold c.performSample(p2pmsg.CtrlMsgGraft, uint(originalGraftSize), uint(sampleSize), func(i, j uint) { grafts[i], grafts[j] = grafts[j], grafts[i] }) @@ -627,11 +627,11 @@ func (c *ControlMsgValidationInspector) truncateGraftMessages(rpc *pubsub.RPC) { func (c *ControlMsgValidationInspector) truncatePruneMessages(rpc *pubsub.RPC) { prunes := rpc.GetControl().GetPrune() originalPruneSize := len(prunes) - if originalPruneSize <= c.config.GraftPrune.MaxSampleSize { + if originalPruneSize <= c.config.GraftPrune.MessageCountThreshold { return // nothing to truncate } - sampleSize := c.config.GraftPrune.MaxSampleSize + sampleSize := c.config.GraftPrune.MessageCountThreshold c.performSample(p2pmsg.CtrlMsgPrune, uint(originalPruneSize), uint(sampleSize), func(i, j uint) { prunes[i], prunes[j] = prunes[j], prunes[i] }) @@ -640,7 +640,7 @@ func (c *ControlMsgValidationInspector) truncatePruneMessages(rpc *pubsub.RPC) { } // truncateIHaveMessages truncates the iHaves control messages in the RPC. If the total number of iHaves in the RPC exceeds the configured -// MaxSampleSize the list of iHaves will be truncated. +// MessageCountThreshold the list of iHaves will be truncated. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIHaveMessages(rpc *pubsub.RPC) { @@ -650,9 +650,9 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessages(rpc *pubsub.RPC) { return } - if originalIHaveCount > c.config.IHave.MaxSampleSize { + if originalIHaveCount > c.config.IHave.MessageCountThreshold { // truncate ihaves and update metrics - sampleSize := c.config.IHave.MaxSampleSize + sampleSize := c.config.IHave.MessageCountThreshold if sampleSize > originalIHaveCount { sampleSize = originalIHaveCount } @@ -666,7 +666,7 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessages(rpc *pubsub.RPC) { } // truncateIHaveMessageIds truncates the message ids for each iHave control message in the RPC. If the total number of message ids in a single iHave exceeds the configured -// MaxMessageIDSampleSize the list of message ids will be truncated. Before message ids are truncated the iHave control messages should have been truncated themselves. +// MessageIdCountThreshold the list of message ids will be truncated. Before message ids are truncated the iHave control messages should have been truncated themselves. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIHaveMessageIds(rpc *pubsub.RPC) { @@ -677,8 +677,8 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessageIds(rpc *pubsub.RPC) continue // nothing to truncate; skip } - if originalMessageIdCount > c.config.IHave.MaxMessageIDSampleSize { - sampleSize := c.config.IHave.MaxMessageIDSampleSize + if originalMessageIdCount > c.config.IHave.MessageIdCountThreshold { + sampleSize := c.config.IHave.MessageIdCountThreshold if sampleSize > originalMessageIdCount { sampleSize = originalMessageIdCount } @@ -693,7 +693,7 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessageIds(rpc *pubsub.RPC) } // truncateIWantMessages truncates the iWant control messages in the RPC. If the total number of iWants in the RPC exceeds the configured -// MaxSampleSize the list of iWants will be truncated. +// MessageCountThreshold the list of iWants will be truncated. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIWantMessages(from peer.ID, rpc *pubsub.RPC) { @@ -703,9 +703,9 @@ func (c *ControlMsgValidationInspector) truncateIWantMessages(from peer.ID, rpc return } - if originalIWantCount > c.config.IWant.MaxSampleSize { + if originalIWantCount > c.config.IWant.MessageCountThreshold { // truncate iWants and update metrics - sampleSize := c.config.IWant.MaxSampleSize + sampleSize := c.config.IWant.MessageCountThreshold if sampleSize > originalIWantCount { sampleSize = originalIWantCount } @@ -719,22 +719,22 @@ func (c *ControlMsgValidationInspector) truncateIWantMessages(from peer.ID, rpc } // truncateIWantMessageIds truncates the message ids for each iWant control message in the RPC. If the total number of message ids in a single iWant exceeds the configured -// MaxMessageIDSampleSize the list of message ids will be truncated. Before message ids are truncated the iWant control messages should have been truncated themselves. +// MessageIdCountThreshold the list of message ids will be truncated. Before message ids are truncated the iWant control messages should have been truncated themselves. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIWantMessageIds(from peer.ID, rpc *pubsub.RPC) { lastHighest := c.rpcTracker.LastHighestIHaveRPCSize() lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). - Uint("max_sample_size", c.config.IWant.MaxSampleSize). + Uint("max_sample_size", c.config.IWant.MessageCountThreshold). Int64("last_highest_ihave_rpc_size", lastHighest). Logger() sampleSize := int(10 * lastHighest) - if sampleSize == 0 || sampleSize > c.config.IWant.MaxMessageIDSampleSize { + if sampleSize == 0 || sampleSize > c.config.IWant.MessageIdCountThreshold { // invalid or 0 sample size is suspicious lg.Warn().Str(logging.KeySuspicious, "true").Msg("zero or invalid sample size, using default max sample size") - sampleSize = c.config.IWant.MaxMessageIDSampleSize + sampleSize = c.config.IWant.MessageIdCountThreshold } for _, iWant := range rpc.GetControl().GetIwant() { messageIDs := iWant.GetMessageIDs() diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 4f308cb7fdc..4c9bf53019f 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -86,7 +86,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("graft truncation", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.GraftPrune.MaxSampleSize = graftPruneMessageMaxSampleSize + params.Config.GraftPrune.MessageCountThreshold = graftPruneMessageMaxSampleSize }) // topic validation is ignored set any topic oracle distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() @@ -115,7 +115,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("prune truncation", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.GraftPrune.MaxSampleSize = graftPruneMessageMaxSampleSize + params.Config.GraftPrune.MessageCountThreshold = graftPruneMessageMaxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -145,7 +145,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("ihave message id truncation", func(t *testing.T) { maxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IHave.MaxSampleSize = maxSampleSize + params.Config.IHave.MessageCountThreshold = maxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -163,9 +163,9 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iHavesGreaterThanMaxSampleSize)) require.NoError(t, inspector.Inspect(from, iHavesLessThanMaxSampleSize)) require.Eventually(t, func() bool { - // rpc with iHaves greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iHaves greater than configured max sample size should be truncated to MessageCountThreshold shouldBeTruncated := len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()) == maxSampleSize - // rpc with iHaves less than MaxSampleSize should not be truncated + // rpc with iHaves less than MessageCountThreshold should not be truncated shouldNotBeTruncated := len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) @@ -175,7 +175,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("ihave message ids truncation", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IHave.MaxMessageIDSampleSize = maxMessageIDSampleSize + params.Config.IHave.MessageIdCountThreshold = maxMessageIDSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -192,13 +192,13 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iHavesLessThanMaxSampleSize)) require.Eventually(t, func() bool { for _, iHave := range iHavesGreaterThanMaxSampleSize.GetControl().GetIhave() { - // rpc with iHaves message ids greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iHaves message ids greater than configured max sample size should be truncated to MessageCountThreshold if len(iHave.GetMessageIDs()) != maxMessageIDSampleSize { return false } } for _, iHave := range iHavesLessThanMaxSampleSize.GetControl().GetIhave() { - // rpc with iHaves message ids less than MaxSampleSize should not be truncated + // rpc with iHaves message ids less than MessageCountThreshold should not be truncated if len(iHave.GetMessageIDs()) != 50 { return false } @@ -211,7 +211,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("iwant message truncation", func(t *testing.T) { maxSampleSize := uint(100) inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWant.MaxSampleSize = maxSampleSize + params.Config.IWant.MessageCountThreshold = maxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -227,9 +227,9 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iWantsGreaterThanMaxSampleSize)) require.NoError(t, inspector.Inspect(from, iWantsLessThanMaxSampleSize)) require.Eventually(t, func() bool { - // rpc with iWants greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iWants greater than configured max sample size should be truncated to MessageCountThreshold shouldBeTruncated := len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant()) == int(maxSampleSize) - // rpc with iWants less than MaxSampleSize should not be truncated + // rpc with iWants less than MessageCountThreshold should not be truncated shouldNotBeTruncated := len(iWantsLessThanMaxSampleSize.GetControl().GetIwant()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) @@ -239,7 +239,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("iwant message id truncation", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWant.MaxMessageIDSampleSize = maxMessageIDSampleSize + params.Config.IWant.MessageIdCountThreshold = maxMessageIDSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -254,13 +254,13 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iWantsLessThanMaxSampleSize)) require.Eventually(t, func() bool { for _, iWant := range iWantsGreaterThanMaxSampleSize.GetControl().GetIwant() { - // rpc with iWants message ids greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iWants message ids greater than configured max sample size should be truncated to MessageCountThreshold if len(iWant.GetMessageIDs()) != maxMessageIDSampleSize { return false } } for _, iWant := range iWantsLessThanMaxSampleSize.GetControl().GetIwant() { - // rpc with iWants less than MaxSampleSize should not be truncated + // rpc with iWants less than MessageCountThreshold should not be truncated if len(iWant.GetMessageIDs()) != 50 { return false } @@ -275,7 +275,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { // It ensures that valid RPC control messages do not trigger erroneous invalid control message notifications, // while all types of invalid control messages trigger expected notifications. func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { - t.Run("processInspectRPCReq should not disseminate any invalid notification errors for valid RPC's", func(t *testing.T) { + // valid rpc should not trigger invalid control message notification + t.Run("valid rpc", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, _, topicProviderOracle := inspectorFixture(t) defer distributor.AssertNotCalled(t, "Distribute") @@ -437,7 +438,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectGraftMessages should disseminate invalid control message notification for invalid graft messages as expected", func(t *testing.T) { + // invalid graft topic ids should trigger invalid control message notification + t.Run("invalid graft topic", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) // create unknown topic unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) @@ -465,7 +467,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectPruneMessages should disseminate invalid control message notification for invalid prune messages as expected", func(t *testing.T) { + // invalid prune topic ids should trigger invalid control message notification + t.Run("invalid prune topic", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) // create unknown topic unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) @@ -492,7 +495,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with invalid topics as expected", func(t *testing.T) { + // invalid ihave topic ids should trigger invalid control message notification + t.Run("invalid ihave topic", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) // create unknown topic unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) @@ -519,7 +523,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectIHaveMessages should NOT disseminate invalid control message notification for iHave messages when duplicate message ids are below the threshold", func(t *testing.T) { + // ihave duplicate topic ids below threshold should NOT trigger invalid control message notification. + t.Run("ihave duplicate topic ids below threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) // avoid unknown topics errors @@ -548,7 +553,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages when duplicate message ids are above the threshold", func(t *testing.T) { + // ihave duplicate topic ids beyond threshold should trigger invalid control message notification. + t.Run("ihave duplicate message ids beyond threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) // avoid unknown topics errors @@ -578,7 +584,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when duplicate message ids exceeds the allowed threshold", func(t *testing.T) { + // ihave duplicate message ids beyond threshold should trigger invalid control message notification. + t.Run("iwant duplicate message ids above threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) // oracle must be set even though iWant messages do not have topic IDs duplicateMsgID := unittest.IdentifierFixture() From ccb8a74a724c5c2a2fdb05cf16a7b89c8ef413f7 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 09:13:09 -0800 Subject: [PATCH 22/57] adds metrics for iwant and ihave inspections --- module/metrics.go | 30 +++++ .../gossipsub_rpc_validation_inspector.go | 113 ++++++++++++++++++ .../control_message_validation_inspector.go | 34 +++--- 3 files changed, 163 insertions(+), 14 deletions(-) diff --git a/module/metrics.go b/module/metrics.go index 9713e022430..d3bf02e13ab 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -276,6 +276,36 @@ type GossipSubRpcValidationInspectorMetrics interface { // messageType: the type of the control message that was truncated // diff: the number of control messages truncated. OnControlMessagesTruncated(messageType p2pmsg.ControlMessageType, diff int) + + // OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on an iWant message at the end of the async inspection iWants + // across one RPC. + // Args: + // duplicateCount: the number of duplicate message ids received by the node on an iWant message at the end of the async inspection iWants. + // cacheMissCount: the number of cache miss message ids received by the node on an iWant message at the end of the async inspection iWants. + OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) + + // OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of duplicate message ids + // received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. + OnIWantDuplicateMessageIdsExceedThreshold() + + // OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of cache miss message ids + // received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. + OnIWantCacheMissMessageIdsExceedThreshold() + + // OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages, regardless of the result of the inspection. + // It tracks the number of duplicate topic ids and duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. + // Args: + // duplicateTopicIds: the number of duplicate topic ids received by the node on an iHave message at the end of the async inspection iHaves. + // duplicateMessageIds: the number of duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. + OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) + + // OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate topic ids + // received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. + OnIHaveDuplicateTopicIdsExceedThreshold() + + // OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate message ids + // received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. + OnIHaveDuplicateMessageIdsExceedThreshold() } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index 69706fa0083..0c6824a05e0 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -29,6 +29,18 @@ type GossipSubRpcValidationInspectorMetrics struct { receivedGraftCount prometheus.Counter receivedPublishMessageCount prometheus.Counter incomingRpcCount prometheus.Counter + + // iHave inspection + iHaveDuplicateMessageIdHistogram prometheus.Histogram + iHaveDuplicateTopicIdHistogram prometheus.Histogram + iHaveDuplicateMessageIdExceedThresholdCount prometheus.Counter + iHaveDuplicateTopicIdExceedThresholdCount prometheus.Counter + + // iWant inspection + iWantDuplicateMessageIdHistogram prometheus.Histogram + iWantCacheMissHistogram prometheus.Histogram + iWantDuplicateMessageIdExceedThresholdCount prometheus.Counter + iWantCacheMissMessageIdExceedThresholdCount prometheus.Counter } var _ module.GossipSubRpcValidationInspectorMetrics = (*GossipSubRpcValidationInspectorMetrics)(nil) @@ -118,6 +130,62 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Help: "number of incoming rpc messages from gossipsub protocol", }) + gc.iHaveDuplicateMessageIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Buckets: []float64{1, 100, 1000}, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids", + Help: "number of duplicate message ids received from gossipsub protocol during the async inspection", + }) + + gc.iHaveDuplicateTopicIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Buckets: []float64{1, 100, 1000}, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids", + Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection", + }) + + gc.iHaveDuplicateMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids_exceed_threshold_total", + }) + + gc.iHaveDuplicateTopicIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids_exceed_threshold_total", + }) + + gc.iWantDuplicateMessageIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids", + Buckets: []float64{1, 100, 1000}, + Help: "number of duplicate message ids received from gossipsub protocol during the async inspection", + }) + + gc.iWantCacheMissHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids", + Buckets: []float64{1, 100, 1000}, + Help: "number of cache miss message ids received from gossipsub protocol during the async inspection", + }) + + gc.iWantDuplicateMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids_exceed_threshold_total", + }) + + gc.iWantCacheMissMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids_exceed_threshold_total", + }) + return gc } @@ -188,3 +256,48 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnIncomingRpcReceived(iHaveCoun c.receivedIWantMsgCount.Add(float64(iWantCount)) c.receivedIHaveMsgCount.Add(float64(iHaveCount)) } + +// OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on an iWant message at the end of the async inspection iWants +// across one RPC. +// +// duplicateCount: the number of duplicate message ids received by the node on an iWant message at the end of the async inspection iWants. +// cacheMissCount: the number of cache miss message ids received by the node on an iWant message at the end of the async inspection iWants. +func (c *GossipSubRpcValidationInspectorMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + c.iWantDuplicateMessageIdHistogram.Observe(float64(duplicateCount)) + c.iWantCacheMissHistogram.Observe(float64(cacheMissCount)) +} + +// OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of duplicate message ids +// received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + c.iWantDuplicateMessageIdExceedThresholdCount.Inc() +} + +// OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of cache miss message ids +// received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + c.iWantCacheMissMessageIdExceedThresholdCount.Inc() +} + +// OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages, regardless of the result of the inspection. +// It tracks the number of duplicate topic ids and duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. +// Args: +// +// duplicateTopicIds: the number of duplicate topic ids received by the node on an iHave message at the end of the async inspection iHaves. +// duplicateMessageIds: the number of duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. +func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + c.iHaveDuplicateTopicIdHistogram.Observe(float64(duplicateTopicIds)) + c.iHaveDuplicateMessageIdHistogram.Observe(float64(duplicateMessageIds)) +} + +// OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate topic ids +// received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + c.iHaveDuplicateTopicIdExceedThresholdCount.Inc() +} + +// OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate message ids +// received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + c.iHaveDuplicateMessageIdExceedThresholdCount.Inc() +} diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index d938bf75e08..a1eeb2ab46f 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -401,6 +401,9 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave totalMessageIds := 0 totalDuplicateTopicIds := 0 totalDuplicateMessageIds := 0 + defer func() { + c.metrics.OnIHaveMessagesInspected(totalDuplicateTopicIds, totalDuplicateMessageIds) + }() for _, ihave := range ihaves { messageIds := ihave.GetMessageIDs() topic := ihave.GetTopicID() @@ -416,6 +419,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave totalDuplicateTopicIds++ // the topic is duplicated, check if the total number of duplicates exceeds the configured threshold if totalDuplicateTopicIds > c.config.IHave.DuplicateTopicIdThreshold { + c.metrics.OnIHaveDuplicateTopicIdsExceedThreshold() return NewDuplicateTopicErr(topic, totalDuplicateTopicIds, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType } } @@ -425,6 +429,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave totalDuplicateMessageIds++ // the message is duplicated, check if the total number of duplicates exceeds the configured threshold if totalDuplicateMessageIds > c.config.IHave.DuplicateMessageIdThreshold { + c.metrics.OnIHaveDuplicateMessageIdsExceedThreshold() return NewDuplicateMessageIDErr(messageID, totalDuplicateMessageIds, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType } } @@ -459,19 +464,18 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant Uint("max_sample_size", c.config.IWant.MessageCountThreshold). Int64("last_highest_ihave_rpc_size", lastHighest). Logger() - sampleSize := uint(len(iWants)) duplicateMsgIdTracker := make(duplicateStrTracker) cacheMisses := 0 - // keeps track of the total duplicate message ids found in the sample; a message id is considered duplicate if it appears - // more than a configurable threshold count in the sample. - totalDuplicateMessageIds := 0 - allowedCacheMissesThreshold := float64(sampleSize) * c.config.IWant.CacheMissThreshold - allowedDuplicatesThreshold := float64(sampleSize) * c.config.IWant.DuplicateMsgIDThreshold + duplicateMessageIds := 0 + defer func() { + c.metrics.OnIWantMessagesInspected(duplicateMessageIds, cacheMisses) + }() + checkCacheMisses := len(iWants) >= c.config.IWant.CacheMissCheckSize lg = lg.With(). - Uint("iwant_sample_size", sampleSize). - Float64("allowed_cache_misses_threshold", allowedCacheMissesThreshold). - Float64("allowed_duplicates_threshold", allowedDuplicatesThreshold).Logger() + Int("iwant_msg_count", len(iWants)). + Float64("cache_misses_threshold", c.config.IWant.CacheMissThreshold). + Float64("duplicates_threshold", c.config.IWant.DuplicateMsgIDThreshold).Logger() lg.Trace().Msg("validating sample of message ids from iwant control message") @@ -483,16 +487,18 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant // check duplicate allowed threshold if duplicateMsgIdTracker.track(messageID) > 1 { // ideally, an iWant message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once. - totalDuplicateMessageIds++ - if float64(totalDuplicateMessageIds) > allowedDuplicatesThreshold { - return NewIWantDuplicateMsgIDThresholdErr(totalDuplicateMessageIds, messageIDCount, c.config.IWant.DuplicateMsgIDThreshold) + duplicateMessageIds++ + if float64(duplicateMessageIds) > c.config.IWant.DuplicateMsgIDThreshold { + c.metrics.OnIWantDuplicateMessageIdsExceedThreshold() + return NewIWantDuplicateMsgIDThresholdErr(duplicateMessageIds, messageIDCount, c.config.IWant.DuplicateMsgIDThreshold) } } // check cache miss threshold if !c.rpcTracker.WasIHaveRPCSent(messageID) { cacheMisses++ if checkCacheMisses { - if float64(cacheMisses) > allowedCacheMissesThreshold { + if float64(cacheMisses) > c.config.IWant.CacheMissThreshold { + c.metrics.OnIWantCacheMissMessageIdsExceedThreshold() return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWant.CacheMissThreshold) } } @@ -505,7 +511,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant lg.Debug(). Int("total_message_ids", totalMessageIds). Int("cache_misses", cacheMisses). - Int("total_duplicate_message_ids", totalDuplicateMessageIds). + Int("total_duplicate_message_ids", duplicateMessageIds). Msg("iwant control message validation complete") return nil From 26e096854a05e9d56b244c224acbb42676239838 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 09:26:00 -0800 Subject: [PATCH 23/57] adds metrics for invalid topic ids --- module/metrics.go | 5 +++++ .../gossipsub_rpc_validation_inspector.go | 19 +++++++++++++++++++ .../control_message_validation_inspector.go | 2 ++ 3 files changed, 26 insertions(+) diff --git a/module/metrics.go b/module/metrics.go index d3bf02e13ab..3ec54c0b665 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -306,6 +306,11 @@ type GossipSubRpcValidationInspectorMetrics interface { // OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate message ids // received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. OnIHaveDuplicateMessageIdsExceedThreshold() + + // OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message failed due to an invalid topic id. + // Args: + // - messageType: the type of the control message that was truncated. + OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index 0c6824a05e0..dbd5a323f21 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -21,6 +21,7 @@ type GossipSubRpcValidationInspectorMetrics struct { rpcCtrlMsgInAsyncPreProcessingGauge prometheus.Gauge rpcCtrlMsgAsyncProcessingTimeHistogram prometheus.Histogram rpcCtrlMsgTruncation prometheus.HistogramVec + ctrlMsgInvalidTopicIdCount prometheus.CounterVec receivedIWantMsgCount prometheus.Counter receivedIWantMsgIDsHistogram prometheus.Histogram receivedIHaveMsgCount prometheus.Counter @@ -150,12 +151,14 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids_exceed_threshold_total", + Help: "number of times that the async inspection of iHave messages failed due to the number of duplicate message ids exceeding the threshold", }) gc.iHaveDuplicateTopicIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids_exceed_threshold_total", + Help: "number of times that the async inspection of iHave messages failed due to the number of duplicate topic ids exceeding the threshold", }) gc.iWantDuplicateMessageIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ @@ -178,14 +181,23 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids_exceed_threshold_total", + Help: "number of times that the async inspection of iWant messages failed due to the number of duplicate message ids ", }) gc.iWantCacheMissMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids_exceed_threshold_total", + Help: "number of times that the async inspection of iWant messages failed due to the number of cache miss message ids ", }) + gc.ctrlMsgInvalidTopicIdCount = *promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "control_message_invalid_topic_id_count", + Help: "number of control messages with invalid topic id received from gossipsub protocol during the async inspection", + }, []string{LabelMessage}) + return gc } @@ -301,3 +313,10 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateTopicIdsExceedT func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { c.iHaveDuplicateMessageIdExceedThresholdCount.Inc() } + +// OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message failed due to an invalid topic id. +// Args: +// - messageType: the type of the control message that was truncated. +func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + c.ctrlMsgInvalidTopicIdCount.WithLabelValues(messageType.String()).Inc() +} diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index a1eeb2ab46f..87db21f3180 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -411,6 +411,8 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave // first check if the topic is valid, fail fast if it is not err, ctrlMsgType := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) if err != nil { + // TODO: consider adding a threshold for this error similar to the duplicate topic id threshold. + c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.CtrlMsgIHave) return err, ctrlMsgType } From f2970031acc453693b61ce4c3c78045f3bdd31b9 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 09:29:04 -0800 Subject: [PATCH 24/57] wires all invalid topic id metrics --- .../validation/control_message_validation_inspector.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 87db21f3180..1caca823172 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -340,6 +340,8 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft } err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { + // TODO: consider adding a threshold for this error similar to the duplicate topic id threshold. + c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.CtrlMsgGraft) return err, ctrlMsgType } } @@ -371,6 +373,8 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune } err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { + // TODO: consider adding a threshold for this error similar to the duplicate topic id threshold. + c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.CtrlMsgPrune) return err, ctrlMsgType } } @@ -569,6 +573,7 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, err, _ := c.validateTopic(from, topic, activeClusterIDS) if err != nil { // we can skip checking for subscription of topic that failed validation and continue + c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.RpcPublishMessage) errs = multierror.Append(errs, err) continue } From 8675cf6276e2b3ac371a93528ae75c57849e4604 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 09:39:41 -0800 Subject: [PATCH 25/57] adds metrics for inspection result --- module/metrics.go | 11 +++++ .../gossipsub_rpc_validation_inspector.go | 43 +++++++++++++++++++ .../control_message_validation_inspector.go | 3 ++ 3 files changed, 57 insertions(+) diff --git a/module/metrics.go b/module/metrics.go index 3ec54c0b665..bfd841244fd 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -311,6 +311,17 @@ type GossipSubRpcValidationInspectorMetrics interface { // Args: // - messageType: the type of the control message that was truncated. OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) + + // OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message failed due to active cluster ids not set inspection failure. + // This is not causing a misbehaviour report. + OnActiveClusterIDsNotSetErr() + + // OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message failed due to unstaked peer inspection failure. + // This is not causing a misbehaviour report. + OnUnstakedPeerInspectionFailed() + + // OnInvalidControlMessageSent tracks the number of times that the async inspection of a control message failed and an invalid control message was sent. + OnInvalidControlMessageSent() } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index dbd5a323f21..9a1f8b57ed2 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -42,6 +42,11 @@ type GossipSubRpcValidationInspectorMetrics struct { iWantCacheMissHistogram prometheus.Histogram iWantDuplicateMessageIdExceedThresholdCount prometheus.Counter iWantCacheMissMessageIdExceedThresholdCount prometheus.Counter + + // inspection result + errActiveClusterIdsNotSetCount prometheus.Counter + errUnstakedPeerInspectionFailedCount prometheus.Counter + invalidControlMessageSentCount prometheus.Counter } var _ module.GossipSubRpcValidationInspectorMetrics = (*GossipSubRpcValidationInspectorMetrics)(nil) @@ -198,6 +203,27 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Help: "number of control messages with invalid topic id received from gossipsub protocol during the async inspection", }, []string{LabelMessage}) + gc.errActiveClusterIdsNotSetCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "active_cluster_ids_not_inspection_err_count", + Help: "number of inspection errors due to active cluster ids not set inspection failure", + }) + + gc.errUnstakedPeerInspectionFailedCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "unstaked_peer_inspection_err_count", + Help: "number of inspection errors due to unstaked peer inspection failure", + }) + + gc.invalidControlMessageSentCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "invalid_control_message_sent_count", + Help: "number of invalid control messages sent due to async inspection failure", + }) + return gc } @@ -320,3 +346,20 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateMessageIdsExcee func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { c.ctrlMsgInvalidTopicIdCount.WithLabelValues(messageType.String()).Inc() } + +// OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message failed due to active cluster ids not set inspection failure. +// This is not causing a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnActiveClusterIDsNotSetErr() { + c.errActiveClusterIdsNotSetCount.Inc() +} + +// OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message failed due to unstaked peer inspection failure. +// This is not causing a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnUnstakedPeerInspectionFailed() { + c.errUnstakedPeerInspectionFailedCount.Inc() +} + +// OnInvalidControlMessageSent tracks the number of times that the async inspection of a control message failed and an invalid control message was sent. +func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageSent() { + c.invalidControlMessageSentCount.Inc() +} diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 1caca823172..af46779021a 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -914,10 +914,13 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In switch { case IsErrActiveClusterIDsNotSet(err): + c.metrics.OnActiveClusterIDsNotSetErr() lg.Warn().Msg("active cluster ids not set") case IsErrUnstakedPeer(err): + c.metrics.OnUnstakedPeerInspectionFailed() lg.Warn().Msg("control message received from unstaked peer") default: + c.metrics.OnInvalidControlMessageSent() distErr := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err, count, topicType)) if distErr != nil { lg.Error(). From c0d2f4224c385b3105dc62546f681162c5a18fba Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 09:58:53 -0800 Subject: [PATCH 26/57] adds metrics for publish inspection support --- module/metrics.go | 11 +++++ .../gossipsub_rpc_validation_inspector.go | 43 +++++++++++++++++++ .../control_message_validation_inspector.go | 3 ++ 3 files changed, 57 insertions(+) diff --git a/module/metrics.go b/module/metrics.go index bfd841244fd..9a0fb2c6bc6 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -322,6 +322,17 @@ type GossipSubRpcValidationInspectorMetrics interface { // OnInvalidControlMessageSent tracks the number of times that the async inspection of a control message failed and an invalid control message was sent. OnInvalidControlMessageSent() + + // OnInvalidSenderForPublishMessage tracks the number of times that the async inspection of a publish message detected an invalid sender. + // Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. + OnInvalidSenderForPublishMessage() + + // OnPublishMessagesInspectionErrorExceedsThreshold tracks the number of times that async inspection of publish messages failed due to the number of errors. + OnPublishMessagesInspectionErrorExceedsThreshold() + + // OnPublishMessageInvalidSubscription tracks the number of times that the async inspection of a publish message detected an invalid subscription. + // Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. + OnPublishMessageInvalidSubscription() } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index 9a1f8b57ed2..52ccb82d067 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -47,6 +47,11 @@ type GossipSubRpcValidationInspectorMetrics struct { errActiveClusterIdsNotSetCount prometheus.Counter errUnstakedPeerInspectionFailedCount prometheus.Counter invalidControlMessageSentCount prometheus.Counter + + // publish messages + publishMessageInspectionErrExceedThresholdCount prometheus.Counter + publishMessageInvalidSenderCount prometheus.Counter + publishMessageInvalidSubscriptionsCount prometheus.Counter } var _ module.GossipSubRpcValidationInspectorMetrics = (*GossipSubRpcValidationInspectorMetrics)(nil) @@ -224,6 +229,27 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Help: "number of invalid control messages sent due to async inspection failure", }) + gc.publishMessageInvalidSenderCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "publish_message_invalid_sender_count", + Help: "number of invalid sender detected for publish messages", + }) + + gc.publishMessageInspectionErrExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "publish_message_inspection_err_exceed_threshold_count", + Help: "number of publish messages inspection errors exceeding the threshold", + }) + + gc.publishMessageInvalidSubscriptionsCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "publish_message_invalid_subscriptions_count", + Help: "number of publish messages with invalid subscriptions", + }) + return gc } @@ -363,3 +389,20 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnUnstakedPeerInspectionFailed( func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageSent() { c.invalidControlMessageSentCount.Inc() } + +// OnInvalidSenderForPublishMessage tracks the number of times that the async inspection of a publish message detected an invalid sender. +// Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. +func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidSenderForPublishMessage() { + c.publishMessageInvalidSenderCount.Inc() +} + +// OnPublishMessagesInspectionErrorExceedsThreshold tracks the number of times that async inspection of publish messages failed due to the number of errors. +func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + c.publishMessageInspectionErrExceedThresholdCount.Inc() +} + +// OnPublishMessageInvalidSubscription tracks the number of times that the async inspection of a publish message detected an invalid subscription. +// Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. +func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInvalidSubscription() { + c.publishMessageInvalidSubscriptionsCount.Inc() +} diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index af46779021a..de3c6fc96a4 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -562,6 +562,7 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, if c.networkingType == network.PrivateNetwork { err := c.checkPubsubMessageSender(message) if err != nil { + c.metrics.OnInvalidSenderForPublishMessage() errs = multierror.Append(errs, err) continue } @@ -579,12 +580,14 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, } if !hasSubscription(topic.String()) { + c.metrics.OnPublishMessageInvalidSubscription() errs = multierror.Append(errs, fmt.Errorf("subscription for topic %s not found", topic)) } } // return an error when we exceed the error threshold if errs != nil && errs.Len() > c.config.PublishMessages.ErrorThreshold { + c.metrics.OnPublishMessagesInspectionErrorExceedsThreshold() return NewInvalidRpcPublishMessagesErr(errs.ErrorOrNil(), errs.Len()), uint64(errs.Len()) } From 70940abea46e986e1922bc309515b87ab85518ac Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 10:45:22 -0800 Subject: [PATCH 27/57] adds metrics for graft and prune inspection --- module/metrics.go | 19 ++++++ .../gossipsub_rpc_validation_inspector.go | 65 +++++++++++++++++++ .../control_message_validation_inspector.go | 8 +++ 3 files changed, 92 insertions(+) diff --git a/module/metrics.go b/module/metrics.go index 9a0fb2c6bc6..61a540f5b57 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -333,6 +333,25 @@ type GossipSubRpcValidationInspectorMetrics interface { // OnPublishMessageInvalidSubscription tracks the number of times that the async inspection of a publish message detected an invalid subscription. // Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. OnPublishMessageInvalidSubscription() + + // OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a prune message failed due to the number of duplicate topic ids + // received by the node on prune messages of the same rpc excesses threshold, which results in a misbehaviour report. + // Note that it does cause a misbehaviour report. + OnPruneDuplicateTopicIdsExceedThreshold() + + // OnPruneMessageInspected is called at the end of the async inspection of prune messages, regardless of the result of the inspection. + // Args: + // duplicateTopicIds: the number of duplicate topic ids received by the node on a prune message at the end of the async inspection prunes. + OnPruneMessageInspected(duplicateTopicIds int) + + // OnGraftDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a graft message failed due to the number of duplicate topic ids. + // received by the node on graft messages of the same rpc excesses threshold, which results in a misbehaviour report. + OnGraftDuplicateTopicIdsExceedThreshold() + + // OnGraftMessageInspected is called at the end of the async inspection of graft messages, regardless of the result of the inspection. + // Args: + // duplicateTopicIds: the number of duplicate topic ids received by the node on a graft message at the end of the async inspection grafts. + OnGraftMessageInspected(duplicateTopicIds int) } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index 52ccb82d067..499e3760cd9 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -31,6 +31,14 @@ type GossipSubRpcValidationInspectorMetrics struct { receivedPublishMessageCount prometheus.Counter incomingRpcCount prometheus.Counter + // graft inspection + graftDuplicateTopicIdsHistogram prometheus.Histogram + graftDuplicateTopicIdsExceedThresholdCount prometheus.Counter + + // prune inspection + pruneDuplicateTopicIdsHistogram prometheus.Histogram + pruneDuplicateTopicIdsExceedThresholdCount prometheus.Counter + // iHave inspection iHaveDuplicateMessageIdHistogram prometheus.Histogram iHaveDuplicateTopicIdHistogram prometheus.Histogram @@ -250,6 +258,34 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Help: "number of publish messages with invalid subscriptions", }) + gc.graftDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids", + Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection of graft messages", + }) + + gc.graftDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids_exceed_threshold_total", + Help: "number of times that the async inspection of graft messages failed due to the number of duplicate topic ids exceeding the threshold", + }) + + gc.pruneDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids", + Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection of prune messages", + }) + + gc.pruneDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids_exceed_threshold_total", + Help: "number of times that the async inspection of prune messages failed due to the number of duplicate topic ids exceeding the threshold", + }) + return gc } @@ -406,3 +442,32 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessagesInspectionErro func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInvalidSubscription() { c.publishMessageInvalidSubscriptionsCount.Inc() } + +// OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a prune message failed due to the number of duplicate topic ids +// received by the node on prune messages of the same rpc excesses threshold, which results in a misbehaviour report. +// Note that it does cause a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + c.pruneDuplicateTopicIdsExceedThresholdCount.Inc() +} + +// OnPruneMessageInspected is called at the end of the async inspection of prune messages, regardless of the result of the inspection. +// Args: +// +// duplicateTopicIds: the number of duplicate topic ids received by the node on a prune message at the end of the async inspection prunes. +func (c *GossipSubRpcValidationInspectorMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + c.pruneDuplicateTopicIdsHistogram.Observe(float64(duplicateTopicIds)) +} + +// OnGraftDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a graft message failed due to the number of duplicate topic ids. +// received by the node on graft messages of the same rpc excesses threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + c.graftDuplicateTopicIdsExceedThresholdCount.Inc() +} + +// OnGraftMessageInspected is called at the end of the async inspection of graft messages, regardless of the result of the inspection. +// Args: +// +// duplicateTopicIds: the number of duplicate topic ids received by the node on a graft message at the end of the async inspection grafts. +func (c *GossipSubRpcValidationInspectorMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + c.graftDuplicateTopicIdsHistogram.Observe(float64(duplicateTopicIds)) +} diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index de3c6fc96a4..d082c8f7403 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -328,6 +328,9 @@ func (c *ControlMsgValidationInspector) checkPubsubMessageSender(message *pubsub func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { duplicateTopicTracker := make(duplicateStrTracker) totalDuplicateTopicIds := 0 + defer func() { + c.metrics.OnGraftMessageInspected(totalDuplicateTopicIds) + }() for _, graft := range grafts { topic := channels.Topic(graft.GetTopicID()) if duplicateTopicTracker.track(topic.String()) > 1 { @@ -335,6 +338,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft totalDuplicateTopicIds++ // check if the total number of duplicates exceeds the configured threshold. if totalDuplicateTopicIds > c.config.GraftPrune.DuplicateTopicIdThreshold { + c.metrics.OnGraftDuplicateTopicIdsExceedThreshold() return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType } } @@ -361,6 +365,9 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { tracker := make(duplicateStrTracker) totalDuplicateTopicIds := 0 + defer func() { + c.metrics.OnPruneMessageInspected(totalDuplicateTopicIds) + }() for _, prune := range prunes { topic := channels.Topic(prune.GetTopicID()) if tracker.track(topic.String()) > 1 { @@ -368,6 +375,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune totalDuplicateTopicIds++ // check if the total number of duplicates exceeds the configured threshold. if totalDuplicateTopicIds > c.config.GraftPrune.DuplicateTopicIdThreshold { + c.metrics.OnPruneDuplicateTopicIdsExceedThreshold() return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgPrune), p2p.CtrlMsgNonClusterTopicType } } From efe63e2a585c9bf540be752ee14bd60fa669457d Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 10:56:22 -0800 Subject: [PATCH 28/57] adds metrics for publish inspection --- module/metrics.go | 7 +++++++ .../gossipsub_rpc_validation_inspector.go | 17 +++++++++++++++++ .../control_message_validation_inspector.go | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/module/metrics.go b/module/metrics.go index 61a540f5b57..2c925272dd5 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -352,6 +352,13 @@ type GossipSubRpcValidationInspectorMetrics interface { // Args: // duplicateTopicIds: the number of duplicate topic ids received by the node on a graft message at the end of the async inspection grafts. OnGraftMessageInspected(duplicateTopicIds int) + + // OnPublishMessageInspected tracks the number of errors that occurred during the async inspection of publish messages. + // Note that this function is called on each publish message received by the node regardless of the result of the inspection. + // If the number of errors exceeds the threshold, a misbehaviour report is sent, but this function is still called. + // Args: + // - errCount: the number of errors that occurred during the async inspection of publish messages. + OnPublishMessageInspected(errCount int) } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index 499e3760cd9..b8dd232d36c 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -60,6 +60,7 @@ type GossipSubRpcValidationInspectorMetrics struct { publishMessageInspectionErrExceedThresholdCount prometheus.Counter publishMessageInvalidSenderCount prometheus.Counter publishMessageInvalidSubscriptionsCount prometheus.Counter + publishMessageInspectedErrHistogram prometheus.Histogram } var _ module.GossipSubRpcValidationInspectorMetrics = (*GossipSubRpcValidationInspectorMetrics)(nil) @@ -286,6 +287,13 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Help: "number of times that the async inspection of prune messages failed due to the number of duplicate topic ids exceeding the threshold", }) + gc.publishMessageInspectedErrHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "publish_message_inspected_err_histogram", + Help: "histogram of publish message inspection errors", + }) + return gc } @@ -471,3 +479,12 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnGraftDuplicateTopicIdsExceedT func (c *GossipSubRpcValidationInspectorMetrics) OnGraftMessageInspected(duplicateTopicIds int) { c.graftDuplicateTopicIdsHistogram.Observe(float64(duplicateTopicIds)) } + +// OnPublishMessageInspected tracks the number of errors that occurred during the async inspection of publish messages. +// Note that this function is called on each publish message received by the node regardless of the result of the inspection. +// If the number of errors exceeds the threshold, a misbehaviour report is sent, but this function is still called. +// Args: +// - errCount: the number of errors that occurred during the async inspection of publish messages. +func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(errCount int) { + c.publishMessageInspectedErrHistogram.Observe(float64(errCount)) +} diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index d082c8f7403..b3f7331ea48 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -329,6 +329,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft duplicateTopicTracker := make(duplicateStrTracker) totalDuplicateTopicIds := 0 defer func() { + // regardless of inspection result, update metrics c.metrics.OnGraftMessageInspected(totalDuplicateTopicIds) }() for _, graft := range grafts { @@ -366,6 +367,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune tracker := make(duplicateStrTracker) totalDuplicateTopicIds := 0 defer func() { + // regardless of inspection result, update metrics c.metrics.OnPruneMessageInspected(totalDuplicateTopicIds) }() for _, prune := range prunes { @@ -414,6 +416,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave totalDuplicateTopicIds := 0 totalDuplicateMessageIds := 0 defer func() { + // regardless of inspection result, update metrics c.metrics.OnIHaveMessagesInspected(totalDuplicateTopicIds, totalDuplicateMessageIds) }() for _, ihave := range ihaves { @@ -482,6 +485,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant cacheMisses := 0 duplicateMessageIds := 0 defer func() { + // regardless of inspection result, update metrics c.metrics.OnIWantMessagesInspected(duplicateMessageIds, cacheMisses) }() @@ -566,6 +570,14 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, return false } var errs *multierror.Error + defer func() { + // regardless of inspection result, update metrics + if errs != nil { + c.metrics.OnPublishMessageInspected(errs.Len()) + } else { + c.metrics.OnPublishMessageInspected(0) + } + }() for _, message := range messages[:sampleSize] { if c.networkingType == network.PrivateNetwork { err := c.checkPubsubMessageSender(message) From 7da8f08e33d03898a3fe9edf79297182e669f89a Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 11:06:30 -0800 Subject: [PATCH 29/57] updates mocks --- module/mock/gossip_sub_metrics.go | 90 +++++++++++++++++++ ...ip_sub_rpc_validation_inspector_metrics.go | 90 +++++++++++++++++++ module/mock/lib_p2_p_metrics.go | 90 +++++++++++++++++++ module/mock/network_metrics.go | 90 +++++++++++++++++++ 4 files changed, 360 insertions(+) diff --git a/module/mock/gossip_sub_metrics.go b/module/mock/gossip_sub_metrics.go index f0fbdec5cfd..152b8337fa6 100644 --- a/module/mock/gossip_sub_metrics.go +++ b/module/mock/gossip_sub_metrics.go @@ -26,6 +26,11 @@ func (_m *GossipSubMetrics) AsyncProcessingStarted() { _m.Called() } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *GossipSubMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnAppSpecificScoreUpdated provides a mock function with given fields: _a0 func (_m *GossipSubMetrics) OnAppSpecificScoreUpdated(_a0 float64) { _m.Called(_a0) @@ -46,41 +51,96 @@ func (_m *GossipSubMetrics) OnFirstMessageDeliveredUpdated(_a0 channels.Topic, _ _m.Called(_a0, _a1) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *GossipSubMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *GossipSubMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + // OnIPColocationFactorUpdated provides a mock function with given fields: _a0 func (_m *GossipSubMetrics) OnIPColocationFactorUpdated(_a0 float64) { _m.Called(_a0) } +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *GossipSubMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *GossipSubMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *GossipSubMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageSent provides a mock function with given fields: +func (_m *GossipSubMetrics) OnInvalidControlMessageSent() { + _m.Called() +} + // OnInvalidMessageDeliveredUpdated provides a mock function with given fields: _a0, _a1 func (_m *GossipSubMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _a1 float64) { _m.Called(_a0, _a1) } +// OnInvalidSenderForPublishMessage provides a mock function with given fields: +func (_m *GossipSubMetrics) OnInvalidSenderForPublishMessage() { + _m.Called() +} + +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *GossipSubMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + // OnLocalMeshSizeUpdated provides a mock function with given fields: topic, size func (_m *GossipSubMetrics) OnLocalMeshSizeUpdated(topic string, size int) { _m.Called(topic, size) @@ -156,6 +216,31 @@ func (_m *GossipSubMetrics) OnPeerThrottled() { _m.Called() } +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: errCount +func (_m *GossipSubMetrics) OnPublishMessageInspected(errCount int) { + _m.Called(errCount) +} + +// OnPublishMessageInvalidSubscription provides a mock function with given fields: +func (_m *GossipSubMetrics) OnPublishMessageInvalidSubscription() { + _m.Called() +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + // OnRpcReceived provides a mock function with given fields: msgCount, iHaveCount, iWantCount, graftCount, pruneCount func (_m *GossipSubMetrics) OnRpcReceived(msgCount int, iHaveCount int, iWantCount int, graftCount int, pruneCount int) { _m.Called(msgCount, iHaveCount, iWantCount, graftCount, pruneCount) @@ -176,6 +261,11 @@ func (_m *GossipSubMetrics) OnUndeliveredMessage() { _m.Called() } +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *GossipSubMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + // SetWarningStateCount provides a mock function with given fields: _a0 func (_m *GossipSubMetrics) SetWarningStateCount(_a0 uint) { _m.Called(_a0) diff --git a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go index 0a10b099844..533694ea029 100644 --- a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go +++ b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go @@ -25,36 +25,126 @@ func (_m *GossipSubRpcValidationInspectorMetrics) AsyncProcessingStarted() { _m.Called() } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnControlMessagesTruncated provides a mock function with given fields: messageType, diff func (_m *GossipSubRpcValidationInspectorMetrics) OnControlMessagesTruncated(messageType p2pmsg.ControlMessageType, diff int) { _m.Called(messageType, diff) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubRpcValidationInspectorMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *GossipSubRpcValidationInspectorMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageSent provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageSent() { + _m.Called() +} + +// OnInvalidSenderForPublishMessage provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidSenderForPublishMessage() { + _m.Called() +} + +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubRpcValidationInspectorMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: errCount +func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(errCount int) { + _m.Called(errCount) +} + +// OnPublishMessageInvalidSubscription provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInvalidSubscription() { + _m.Called() +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + type mockConstructorTestingTNewGossipSubRpcValidationInspectorMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/lib_p2_p_metrics.go b/module/mock/lib_p2_p_metrics.go index d555414cd6a..27c3e102cd6 100644 --- a/module/mock/lib_p2_p_metrics.go +++ b/module/mock/lib_p2_p_metrics.go @@ -112,6 +112,11 @@ func (_m *LibP2PMetrics) InboundConnections(connectionCount uint) { _m.Called(connectionCount) } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *LibP2PMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnAppSpecificScoreUpdated provides a mock function with given fields: _a0 func (_m *LibP2PMetrics) OnAppSpecificScoreUpdated(_a0 float64) { _m.Called(_a0) @@ -167,41 +172,96 @@ func (_m *LibP2PMetrics) OnFirstMessageDeliveredUpdated(_a0 channels.Topic, _a1 _m.Called(_a0, _a1) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *LibP2PMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *LibP2PMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *LibP2PMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *LibP2PMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + // OnIPColocationFactorUpdated provides a mock function with given fields: _a0 func (_m *LibP2PMetrics) OnIPColocationFactorUpdated(_a0 float64) { _m.Called(_a0) } +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *LibP2PMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *LibP2PMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *LibP2PMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *LibP2PMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageSent provides a mock function with given fields: +func (_m *LibP2PMetrics) OnInvalidControlMessageSent() { + _m.Called() +} + // OnInvalidMessageDeliveredUpdated provides a mock function with given fields: _a0, _a1 func (_m *LibP2PMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _a1 float64) { _m.Called(_a0, _a1) } +// OnInvalidSenderForPublishMessage provides a mock function with given fields: +func (_m *LibP2PMetrics) OnInvalidSenderForPublishMessage() { + _m.Called() +} + +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *LibP2PMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + // OnLocalMeshSizeUpdated provides a mock function with given fields: topic, size func (_m *LibP2PMetrics) OnLocalMeshSizeUpdated(topic string, size int) { _m.Called(topic, size) @@ -287,6 +347,31 @@ func (_m *LibP2PMetrics) OnPeerThrottled() { _m.Called() } +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *LibP2PMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: errCount +func (_m *LibP2PMetrics) OnPublishMessageInspected(errCount int) { + _m.Called(errCount) +} + +// OnPublishMessageInvalidSubscription provides a mock function with given fields: +func (_m *LibP2PMetrics) OnPublishMessageInvalidSubscription() { + _m.Called() +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + // OnRpcReceived provides a mock function with given fields: msgCount, iHaveCount, iWantCount, graftCount, pruneCount func (_m *LibP2PMetrics) OnRpcReceived(msgCount int, iHaveCount int, iWantCount int, graftCount int, pruneCount int) { _m.Called(msgCount, iHaveCount, iWantCount, graftCount, pruneCount) @@ -332,6 +417,11 @@ func (_m *LibP2PMetrics) OnUndeliveredMessage() { _m.Called() } +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *LibP2PMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + // OutboundConnections provides a mock function with given fields: connectionCount func (_m *LibP2PMetrics) OutboundConnections(connectionCount uint) { _m.Called(connectionCount) diff --git a/module/mock/network_metrics.go b/module/mock/network_metrics.go index cc941222bc8..14799f9f64b 100644 --- a/module/mock/network_metrics.go +++ b/module/mock/network_metrics.go @@ -142,6 +142,11 @@ func (_m *NetworkMetrics) MessageRemoved(priority int) { _m.Called(priority) } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *NetworkMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnAppSpecificScoreUpdated provides a mock function with given fields: _a0 func (_m *NetworkMetrics) OnAppSpecificScoreUpdated(_a0 float64) { _m.Called(_a0) @@ -197,41 +202,96 @@ func (_m *NetworkMetrics) OnFirstMessageDeliveredUpdated(_a0 channels.Topic, _a1 _m.Called(_a0, _a1) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *NetworkMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *NetworkMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *NetworkMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *NetworkMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + // OnIPColocationFactorUpdated provides a mock function with given fields: _a0 func (_m *NetworkMetrics) OnIPColocationFactorUpdated(_a0 float64) { _m.Called(_a0) } +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *NetworkMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *NetworkMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *NetworkMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *NetworkMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageSent provides a mock function with given fields: +func (_m *NetworkMetrics) OnInvalidControlMessageSent() { + _m.Called() +} + // OnInvalidMessageDeliveredUpdated provides a mock function with given fields: _a0, _a1 func (_m *NetworkMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _a1 float64) { _m.Called(_a0, _a1) } +// OnInvalidSenderForPublishMessage provides a mock function with given fields: +func (_m *NetworkMetrics) OnInvalidSenderForPublishMessage() { + _m.Called() +} + +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *NetworkMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + // OnLocalMeshSizeUpdated provides a mock function with given fields: topic, size func (_m *NetworkMetrics) OnLocalMeshSizeUpdated(topic string, size int) { _m.Called(topic, size) @@ -322,6 +382,31 @@ func (_m *NetworkMetrics) OnPeerThrottled() { _m.Called() } +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *NetworkMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: errCount +func (_m *NetworkMetrics) OnPublishMessageInspected(errCount int) { + _m.Called(errCount) +} + +// OnPublishMessageInvalidSubscription provides a mock function with given fields: +func (_m *NetworkMetrics) OnPublishMessageInvalidSubscription() { + _m.Called() +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + // OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason func (_m *NetworkMetrics) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { _m.Called(pid, role, msgType, topic, reason) @@ -377,6 +462,11 @@ func (_m *NetworkMetrics) OnUndeliveredMessage() { _m.Called() } +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *NetworkMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + // OnViolationReportSkipped provides a mock function with given fields: func (_m *NetworkMetrics) OnViolationReportSkipped() { _m.Called() From 84c576a632019bf6e55a5878e56ac12147c5d3ec Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 11:11:09 -0800 Subject: [PATCH 30/57] updates noop collector --- module/metrics/noop.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/module/metrics/noop.go b/module/metrics/noop.go index b2d2f8d08aa..99b03f36a8a 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -323,8 +323,27 @@ func (nc *NoopCollector) OnControlMessagesTruncated(messageType p2pmsg.ControlMe } func (nc *NoopCollector) OnIncomingRpcReceived(iHaveCount, iWantCount, graftCount, pruneCount, msgCount int) { } -func (nc *NoopCollector) AsyncProcessingStarted() {} -func (nc *NoopCollector) AsyncProcessingFinished(time.Duration) {} +func (nc *NoopCollector) AsyncProcessingStarted() {} +func (nc *NoopCollector) AsyncProcessingFinished(time.Duration) {} +func (nc *NoopCollector) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) {} +func (nc *NoopCollector) OnIWantDuplicateMessageIdsExceedThreshold() {} +func (nc *NoopCollector) OnIWantCacheMissMessageIdsExceedThreshold() {} +func (nc *NoopCollector) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) {} +func (nc *NoopCollector) OnIHaveDuplicateTopicIdsExceedThreshold() {} +func (nc *NoopCollector) OnIHaveDuplicateMessageIdsExceedThreshold() {} +func (nc *NoopCollector) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { +} +func (nc *NoopCollector) OnActiveClusterIDsNotSetErr() {} +func (nc *NoopCollector) OnUnstakedPeerInspectionFailed() {} +func (nc *NoopCollector) OnInvalidControlMessageSent() {} +func (nc *NoopCollector) OnInvalidSenderForPublishMessage() {} +func (nc *NoopCollector) OnPublishMessagesInspectionErrorExceedsThreshold() {} +func (nc *NoopCollector) OnPublishMessageInvalidSubscription() {} +func (nc *NoopCollector) OnPruneDuplicateTopicIdsExceedThreshold() {} +func (nc *NoopCollector) OnPruneMessageInspected(duplicateTopicIds int) {} +func (nc *NoopCollector) OnGraftDuplicateTopicIdsExceedThreshold() {} +func (nc *NoopCollector) OnGraftMessageInspected(duplicateTopicIds int) {} +func (nc *NoopCollector) OnPublishMessageInspected(errCount int) {} func (nc *NoopCollector) OnMisbehaviorReported(string, string) {} func (nc *NoopCollector) OnViolationReportSkipped() {} From 94fbb48c2aaea2467c9b5350c78c3cc076be8ee1 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 15 Jan 2024 11:18:53 -0800 Subject: [PATCH 31/57] fixes merge conflicts --- network/netconf/flags.go | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 6eb2098ba04..8c75ca00d49 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -264,18 +264,6 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { config.AlspConfig.SyncEngine.RangeRequestBaseProb, "base probability of creating a misbehavior report for a range request message") flags.Float32(alspSyncEngineSyncRequestProb, config.AlspConfig.SyncEngine.SyncRequestProb, "probability of creating a misbehavior report for a sync request message") - - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.SpamRecordCacheKey, p2pconfig.PenaltyDecaySlowdownThresholdKey), - config.GossipSub.ScoringParameters.SpamRecordCache.PenaltyDecaySlowdownThreshold, - fmt.Sprintf("the penalty level at which the decay rate is reduced by --%s", - BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.SpamRecordCacheKey, p2pconfig.DecayRateReductionFactorKey))) - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.SpamRecordCacheKey, p2pconfig.DecayRateReductionFactorKey), - config.GossipSub.ScoringParameters.SpamRecordCache.DecayRateReductionFactor, - fmt.Sprintf("defines the value by which the decay rate is decreased every time the penalty is below the --%s", - BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.SpamRecordCacheKey, p2pconfig.PenaltyDecaySlowdownThresholdKey))) - flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.SpamRecordCacheKey, p2pconfig.PenaltyDecayEvaluationPeriodKey), - config.GossipSub.ScoringParameters.SpamRecordCache.PenaltyDecayEvaluationPeriod, - "defines the period at which the decay for a spam record is okay to be adjusted.") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageCountThreshold), config.GossipSub.RpcInspector.Validation.IHave.MessageCountThreshold, "threshold for the number of ihave control messages to accept on a single RPC message, if exceeded the RPC message will be sampled and truncated") @@ -297,21 +285,6 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageIdCountThreshold), config.GossipSub.RpcInspector.Validation.IWant.MessageIdCountThreshold, "threshold for the number of message ids on a single iwant control message to accept, if exceeded the RPC message ids will be sampled and truncated") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize, - "max number of ihaves to sample when performing validation") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize, - "max number of message ids to sample when performing validation per ihave") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneMessageMaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.GraftPruneMessageMaxSampleSize, - "max number of control messages to sample when performing validation on GRAFT and PRUNE message types") - flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.MaxSampleSize, - "max number of iwants to sample when performing validation") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.MaxMessageIDSampleSize, - "max number of message ids to sample when performing validation per iwant") flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), config.GossipSub.RpcInspector.Validation.IWant.CacheMissThreshold, "max number of cache misses (untracked) allowed in a single iWant control message, if exceeded a misbehavior report will be created") @@ -419,7 +392,13 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.PeerScoringKey, p2pconfig.ProtocolKey, p2pconfig.AppSpecificKey, p2pconfig.UnknownIdentityKey, p2pconfig.PenaltyKey), config.GossipSub.ScoringParameters.PeerScoring.Protocol.AppSpecificScore.UnknownIdentityPenalty, "the penalty for unknown identity. It is applied to the peer's score when the peer is not in the identity list") - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.PeerScoringKey, p2pconfig.ProtocolKey, p2pconfig.AppSpecificKey, p2pconfig.InvalidSubscriptionKey, p2pconfig.PenaltyKey), + flags.Float64(BuildFlagName(gossipsubKey, + p2pconfig.ScoreParamsKey, + p2pconfig.PeerScoringKey, + p2pconfig.ProtocolKey, + p2pconfig.AppSpecificKey, + p2pconfig.InvalidSubscriptionKey, + p2pconfig.PenaltyKey), config.GossipSub.ScoringParameters.PeerScoring.Protocol.AppSpecificScore.InvalidSubscriptionPenalty, "the penalty for invalid subscription. It is applied to the peer's score when the peer subscribes to a topic that it is not authorized to subscribe to") flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.PeerScoringKey, p2pconfig.ProtocolKey, p2pconfig.AppSpecificKey, p2pconfig.MaxAppSpecificKey, p2pconfig.RewardKey), From 22dc57829100f40a4f6f832cfaa442ebe7115f5f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 16 Jan 2024 09:48:26 -0800 Subject: [PATCH 32/57] fixes iwant cache miss behavior --- .../control_message_validation_inspector.go | 12 +- ...ntrol_message_validation_inspector_test.go | 603 ++++++++++-------- 2 files changed, 349 insertions(+), 266 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index b3f7331ea48..767f0d5fc66 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -489,7 +489,6 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant c.metrics.OnIWantMessagesInspected(duplicateMessageIds, cacheMisses) }() - checkCacheMisses := len(iWants) >= c.config.IWant.CacheMissCheckSize lg = lg.With(). Int("iwant_msg_count", len(iWants)). Float64("cache_misses_threshold", c.config.IWant.CacheMissThreshold). @@ -500,6 +499,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant totalMessageIds := 0 for _, iWant := range iWants { messageIds := iWant.GetMessageIDs() + checkCacheMisses := len(messageIds) >= c.config.IWant.CacheMissCheckSize messageIDCount := uint(len(messageIds)) for _, messageID := range messageIds { // check duplicate allowed threshold @@ -512,13 +512,11 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant } } // check cache miss threshold - if !c.rpcTracker.WasIHaveRPCSent(messageID) { + if checkCacheMisses && !c.rpcTracker.WasIHaveRPCSent(messageID) { cacheMisses++ - if checkCacheMisses { - if float64(cacheMisses) > c.config.IWant.CacheMissThreshold { - c.metrics.OnIWantCacheMissMessageIdsExceedThreshold() - return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWant.CacheMissThreshold) - } + if float64(cacheMisses) > c.config.IWant.CacheMissThreshold { + c.metrics.OnIWantCacheMissMessageIdsExceedThreshold() + return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWant.CacheMissThreshold) } } duplicateMsgIdTracker.track(messageID) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 4c9bf53019f..3b13b0e94f5 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "sync" "testing" "time" @@ -93,6 +94,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + // topic validation not performed so we can use random strings graftsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixtures(unittest.IdentifierListFixture(2000).Strings()...)...)) require.Greater(t, len(graftsGreaterThanMaxSampleSize.GetControl().GetGraft()), graftPruneMessageMaxSampleSize) @@ -109,7 +112,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { shouldNotBeTruncated := len(graftsLessThanMaxSampleSize.GetControl().GetGraft()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("prune truncation", func(t *testing.T) { @@ -123,6 +127,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + // unittest.RequireCloseBefore(t, inspector.Ready(), 100*time.Millisecond, "inspector did not start") // topic validation not performed, so we can use random strings prunesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithPrunes(unittest.P2PRPCPruneFixtures(unittest.IdentifierListFixture(2000).Strings()...)...)) @@ -139,7 +145,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { shouldNotBeTruncated := len(prunesLessThanMaxSampleSize.GetControl().GetPrune()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("ihave message id truncation", func(t *testing.T) { @@ -152,6 +159,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) // topic validation not performed so we can use random strings iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(2000, unittest.IdentifierListFixture(2000).Strings()...)...)) @@ -169,7 +177,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { shouldNotBeTruncated := len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("ihave message ids truncation", func(t *testing.T) { @@ -182,6 +191,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) // topic validation not performed so we can use random strings iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(2000, unittest.IdentifierListFixture(10).Strings()...)...)) @@ -205,7 +215,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { } return true }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("iwant message truncation", func(t *testing.T) { @@ -218,6 +229,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(200, 200)...)) require.Greater(t, uint(len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant())), maxSampleSize) iWantsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(50, 200)...)) @@ -233,7 +246,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { shouldNotBeTruncated := len(iWantsLessThanMaxSampleSize.GetControl().GetIwant()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("iwant message id truncation", func(t *testing.T) { @@ -246,6 +260,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 2000)...)) iWantsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 50)...)) @@ -267,7 +283,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { } return true }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) } @@ -275,52 +292,6 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { // It ensures that valid RPC control messages do not trigger erroneous invalid control message notifications, // while all types of invalid control messages trigger expected notifications. func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { - // valid rpc should not trigger invalid control message notification - t.Run("valid rpc", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, _, topicProviderOracle := inspectorFixture(t) - defer distributor.AssertNotCalled(t, "Distribute") - - topics := []string{ - fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID), - fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID), - fmt.Sprintf("%s/%s", channels.SyncCommittee, sporkID), - fmt.Sprintf("%s/%s", channels.RequestChunks, sporkID), - } - // avoid unknown topics errors - topicProviderOracle.UpdateTopics(topics) - inspector.Start(signalerCtx) - grafts := unittest.P2PRPCGraftFixtures(topics...) - prunes := unittest.P2PRPCPruneFixtures(topics...) - ihaves := unittest.P2PRPCIHaveFixtures(50, topics...) - iwants := unittest.P2PRPCIWantFixtures(2, 5) - pubsubMsgs := unittest.GossipSubMessageFixtures(10, topics[0]) - - // avoid cache misses for iwant messages. - iwants[0].MessageIDs = ihaves[0].MessageIDs[:10] - iwants[1].MessageIDs = ihaves[1].MessageIDs[11:20] - expectedMsgIds := make([]string, 0) - expectedMsgIds = append(expectedMsgIds, ihaves[0].MessageIDs...) - expectedMsgIds = append(expectedMsgIds, ihaves[1].MessageIDs...) - rpc := unittest.P2PRPCFixture( - unittest.WithGrafts(grafts...), - unittest.WithPrunes(prunes...), - unittest.WithIHaves(ihaves...), - unittest.WithIWants(iwants...), - unittest.WithPubsubMessages(pubsubMsgs...)) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - require.Contains(t, expectedMsgIds, id) - }) - - from := unittest.PeerIdFixture(t) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) - // duplicate graft topic ids beyond threshold should trigger invalid control message notification t.Run("duplicate graft topic ids beyond threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) @@ -350,7 +321,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // duplicate graft topic ids below threshold should NOT trigger invalid control message notification @@ -376,7 +348,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // duplicate prune topic ids beyond threshold should trigger invalid control message notification @@ -409,7 +382,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // duplicate prune topic ids below threshold should NOT trigger invalid control message notification @@ -435,7 +409,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // invalid graft topic ids should trigger invalid control message notification @@ -458,13 +433,15 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) require.NoError(t, inspector.Inspect(from, unknownTopicReq)) require.NoError(t, inspector.Inspect(from, malformedTopicReq)) require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // invalid prune topic ids should trigger invalid control message notification @@ -486,13 +463,15 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // invalid ihave topic ids should trigger invalid control message notification @@ -514,17 +493,19 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // ihave duplicate topic ids below threshold should NOT trigger invalid control message notification. - t.Run("ihave duplicate topic ids below threshold", func(t *testing.T) { + t.Run("ihave duplicate message ids below threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) // avoid unknown topics errors @@ -550,11 +531,12 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) // TODO: this sleeps should be replaced with a queue size checker. time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // ihave duplicate topic ids beyond threshold should trigger invalid control message notification. - t.Run("ihave duplicate message ids beyond threshold", func(t *testing.T) { + t.Run("ihave duplicate message ids above threshold", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) // avoid unknown topics errors @@ -581,7 +563,8 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) // TODO: this sleeps should be replaced with a queue size checker. time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) // ihave duplicate message ids beyond threshold should trigger invalid control message notification. @@ -606,214 +589,307 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { }) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold", func(t *testing.T) { - cacheMissCheckSize := 1000 - inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWant.CacheMissCheckSize = cacheMissCheckSize - // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - params.Config.IWant.CacheMissThreshold = .9 - }) - // oracle must be set even though iWant messages do not have topic IDs - inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(cacheMissCheckSize+1, 100)...)) - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - found := false - for _, iwant := range inspectMsgRpc.GetControl().GetIwant() { - for _, messageID := range iwant.GetMessageIDs() { - if id == messageID { - found = true - } + }) +} + +// TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. +func TestControlMessageInspection_ValidRpc(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, _, topicProviderOracle := inspectorFixture(t) + defer distributor.AssertNotCalled(t, "Distribute") + + topics := []string{ + fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID), + fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID), + fmt.Sprintf("%s/%s", channels.SyncCommittee, sporkID), + fmt.Sprintf("%s/%s", channels.RequestChunks, sporkID), + } + // avoid unknown topics errors + topicProviderOracle.UpdateTopics(topics) + inspector.Start(signalerCtx) + grafts := unittest.P2PRPCGraftFixtures(topics...) + prunes := unittest.P2PRPCPruneFixtures(topics...) + ihaves := unittest.P2PRPCIHaveFixtures(50, topics...) + iwants := unittest.P2PRPCIWantFixtures(2, 5) + pubsubMsgs := unittest.GossipSubMessageFixtures(10, topics[0]) + + // avoid cache misses for iwant messages. + iwants[0].MessageIDs = ihaves[0].MessageIDs[:10] + iwants[1].MessageIDs = ihaves[1].MessageIDs[11:20] + expectedMsgIds := make([]string, 0) + expectedMsgIds = append(expectedMsgIds, ihaves[0].MessageIDs...) + expectedMsgIds = append(expectedMsgIds, ihaves[1].MessageIDs...) + rpc := unittest.P2PRPCFixture( + unittest.WithGrafts(grafts...), + unittest.WithPrunes(prunes...), + unittest.WithIHaves(ihaves...), + unittest.WithIWants(iwants...), + unittest.WithPubsubMessages(pubsubMsgs...)) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, expectedMsgIds, id) + }) + + from := unittest.PeerIdFixture(t) + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestIWantInspection_CacheMiss_AboveThreshold ensures inspector disseminates invalid control message notifications for iWant messages when cache misses exceeds allowed threshold. +func TestIWantInspection_CacheMiss_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.IWant.CacheMissCheckSize = 99 // when size of an iwant message is above this threshold, it is checked for cache misses + // set high cache miss threshold to ensure we only disseminate notification when it is exceeded + params.Config.IWant.CacheMissThreshold = 900 + }) + // 10 iwant messages, each with 100 message ids; total of 1000 message ids, which when imitated as cache misses should trigger notification dissemination. + inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 100)...)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold + allIwantsChecked := sync.WaitGroup{} + allIwantsChecked.Add(901) // 901 message ids + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { + defer allIwantsChecked.Done() + + id, ok := args[0].(string) + require.True(t, ok) + found := false + for _, iwant := range inspectMsgRpc.GetControl().GetIwant() { + for _, messageID := range iwant.GetMessageIDs() { + if id == messageID { + found = true } } - require.True(t, found) - }) + } + require.True(t, found) + }) - inspector.Start(signalerCtx) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) - - t.Run("inspectIWantMessages should not disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold if cache miss check size not exceeded", - func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - // if size of iwants not greater than 10 cache misses will not be checked - params.Config.IWant.CacheMissCheckSize = 10 - // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - params.Config.IWant.CacheMissThreshold = .9 - }) - // oracle must be set even though iWant messages do not have topic IDs - defer distributor.AssertNotCalled(t, "Distribute") - - msgIds := unittest.IdentifierListFixture(100).Strings() - inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - require.Contains(t, msgIds, id) - }) - - from := unittest.PeerIdFixture(t) - inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) + unittest.RequireReturnsBefore(t, allIwantsChecked.Wait, 1*time.Second, "all iwant messages should be checked for cache misses") - require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured RpcMessageErrorThreshold", func(t *testing.T) { - errThreshold := 500 - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.PublishMessages.ErrorThreshold = errThreshold - }) - // create unknown topic - unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() - // create malformed topic - malformedTopic := channels.Topic("!@#$%^&**((").String() - // a topics spork ID is considered invalid if it does not match the current spork ID - invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())).String() - // create 10 normal messages - pubsubMsgs := unittest.GossipSubMessageFixtures(50, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) - // add 550 invalid messages to force notification dissemination - invalidMessageFixtures := []*pubsub_pb.Message{ - {Topic: &unknownTopic}, - {Topic: &malformedTopic}, - {Topic: &invalidSporkIDTopic}, - } - for i := 0; i < errThreshold+1; i++ { - pubsubMsgs = append(pubsubMsgs, invalidMessageFixtures[rand.Intn(len(invalidMessageFixtures))]) - } - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - topics := make([]string, len(pubsubMsgs)) - for i, msg := range pubsubMsgs { - topics[i] = *msg.Topic - } - // set topic oracle to return list of topics to avoid hasSubscription errors and force topic validation - topicProviderOracle.UpdateTopics(topics) - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) +func TestIWantInspection_CacheMiss_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // if size of iwants is below cache miss check size, it is not checked for cache misses + params.Config.IWant.CacheMissCheckSize = 90 + // set high cache miss threshold to ensure that we do not disseminate notification in this test + params.Config.IWant.CacheMissThreshold = 99 + }) + // oracle must be set even though iWant messages do not have topic IDs + defer distributor.AssertNotCalled(t, "Distribute") - inspector.Start(signalerCtx) + msgIds := unittest.IdentifierListFixture(98).Strings() // one less than cache miss threshold + inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + allIwantsChecked := sync.WaitGroup{} + allIwantsChecked.Add(len(msgIds)) + // returns false each time to imitate cache misses; however, since the number of cache misses is below the threshold, no notification should be disseminated. + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { + defer allIwantsChecked.Done() + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, msgIds, id) }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when subscription missing for topic", func(t *testing.T) { - errThreshold := 500 - inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.PublishMessages.ErrorThreshold = errThreshold - }) - pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) - from := unittest.PeerIdFixture(t) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + + from := unittest.PeerIdFixture(t) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) + unittest.RequireReturnsBefore(t, allIwantsChecked.Wait, 1*time.Second, "all iwant messages should be checked for cache misses") + + // waits one more second to ensure no notification is disseminated + time.Sleep(1 * time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_ExceedingErrThreshold ensures inspector disseminates invalid control message notifications for RPCs that exceed the configured error threshold. +func TestPublishMessageInspection_ExceedingErrThreshold(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.PublishMessages.ErrorThreshold = errThreshold }) + // create unknown topic + unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() + // create malformed topic + malformedTopic := channels.Topic("!@#$%^&**((").String() + // a topics spork ID is considered invalid if it does not match the current spork ID + invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())).String() + // create 10 normal messages + pubsubMsgs := unittest.GossipSubMessageFixtures(50, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) + // add 550 invalid messages to force notification dissemination + invalidMessageFixtures := []*pubsub_pb.Message{ + {Topic: &unknownTopic}, + {Topic: &malformedTopic}, + {Topic: &invalidSporkIDTopic}, + } + for i := 0; i < errThreshold+1; i++ { + pubsubMsgs = append(pubsubMsgs, invalidMessageFixtures[rand.Intn(len(invalidMessageFixtures))]) + } + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + topics := make([]string, len(pubsubMsgs)) + for i, msg := range pubsubMsgs { + topics[i] = *msg.Topic + } + // set topic oracle to return list of topics to avoid hasSubscription errors and force topic validation + topicProviderOracle.UpdateTopics(topics) + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when publish messages contain no topic", func(t *testing.T) { - errThreshold := 500 - inspector, signalerCtx, cancel, distributor, _, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - // 5 invalid pubsub messages will force notification dissemination - params.Config.PublishMessages.ErrorThreshold = errThreshold - }) - pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, "") - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - topics := make([]string, len(pubsubMsgs)) - for i, msg := range pubsubMsgs { - topics[i] = *msg.Topic - } - // set topic oracle to return list of topics excluding first topic sent - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_MissingSubscription ensures inspector disseminates invalid control message notifications for RPCs that the peer is not subscribed to. +func TestPublishMessageInspection_MissingSubscription(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.PublishMessages.ErrorThreshold = errThreshold }) - t.Run("inspectRpcPublishMessages should not inspect pubsub message sender on public networks", func(t *testing.T) { - inspector, signalerCtx, cancel, _, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t) - from := unittest.PeerIdFixture(t) - defer idProvider.AssertNotCalled(t, "ByPeerID", from) - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - topicProviderOracle.UpdateTopics([]string{topic}) - pubsubMsgs := unittest.GossipSubMessageFixtures(10, topic, unittest.WithFrom(from)) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestPublishMessageInspection_MissingTopic ensures inspector disseminates invalid control message notifications for published messages with missing topics. +func TestPublishMessageInspection_MissingTopic(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // 5 invalid pubsub messages will force notification dissemination + params.Config.PublishMessages.ErrorThreshold = errThreshold }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from unstaked peer", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - // override the inspector and params, run the inspector in private mode - params.NetworkingType = network.PrivateNetwork - }) - from := unittest.PeerIdFixture(t) - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - topicProviderOracle.UpdateTopics([]string{topic}) - // default RpcMessageErrorThreshold is 500, 501 messages should trigger a notification - pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) - idProvider.On("ByPeerID", from).Return(nil, false).Times(501) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, "") + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + for _, msg := range pubsubMsgs { + msg.Topic = nil + } + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestRpcInspectionDeactivatedOnPublicNetwork ensures inspector does not inspect RPCs on public networks. +func TestRpcInspectionDeactivatedOnPublicNetwork(t *testing.T) { + inspector, signalerCtx, cancel, _, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t) + from := unittest.PeerIdFixture(t) + defer idProvider.AssertNotCalled(t, "ByPeerID", from) + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + topicProviderOracle.UpdateTopics([]string{topic}) + pubsubMsgs := unittest.GossipSubMessageFixtures(10, topic, unittest.WithFrom(from)) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_Unstaked_From ensures inspector disseminates invalid control message notifications for published messages from unstaked peers. +func TestPublishMessageInspection_Unstaked_From(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { + // override the inspector and params, run the inspector in private mode + params.NetworkingType = network.PrivateNetwork }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from ejected peer", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - // override the inspector and params, run the inspector in private mode - params.NetworkingType = network.PrivateNetwork - }) - from := unittest.PeerIdFixture(t) - id := unittest.IdentityFixture() - id.Ejected = true - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - topicProviderOracle.UpdateTopics([]string{topic}) - pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) - idProvider.On("ByPeerID", from).Return(id, true).Times(501) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + from := unittest.PeerIdFixture(t) + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + topicProviderOracle.UpdateTopics([]string{topic}) + // default RpcMessageErrorThreshold is 500, 501 messages should trigger a notification + pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) + idProvider.On("ByPeerID", from).Return(nil, false).Times(501) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_Ejected_From ensures inspector disseminates invalid control message notifications for published messages from ejected peers. +func TestPublishMessageInspection_Ejected_From(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { + // override the inspector and params, run the inspector in private mode + params.NetworkingType = network.PrivateNetwork }) + from := unittest.PeerIdFixture(t) + id := unittest.IdentityFixture() + id.Ejected = true + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + topicProviderOracle.UpdateTopics([]string{topic}) + pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) + idProvider.On("ByPeerID", from).Return(id, true).Times(501) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } // TestNewControlMsgValidationInspector_validateClusterPrefixedTopic ensures cluster prefixed topics are validated as expected. @@ -829,10 +905,13 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) inspector.ActiveClustersChanged(flow.ChainIDList{clusterID, flow.ChainID(unittest.IdentifierFixture().String()), flow.ChainID(unittest.IdentifierFixture().String())}) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("validateClusterPrefixedTopic should not return error if cluster prefixed hard threshold not exceeded for unknown cluster ids", func(t *testing.T) { @@ -848,11 +927,13 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin id := unittest.IdentityFixture() idProvider.On("ByPeerID", from).Return(id, true).Once() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("validateClusterPrefixedTopic should return an error when sender is unstaked", func(t *testing.T) { @@ -866,10 +947,13 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("validateClusterPrefixedTopic should return error if cluster prefixed hard threshold exceeded for unknown cluster ids", func(t *testing.T) { @@ -888,12 +972,15 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + for i := 0; i < 11; i++ { require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) } // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) } @@ -909,6 +996,8 @@ func TestControlMessageValidationInspector_ActiveClustersChanged(t *testing.T) { } inspector.ActiveClustersChanged(activeClusterIds) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + from := unittest.PeerIdFixture(t) for _, id := range activeClusterIds { topic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(id), sporkID)).String() @@ -917,7 +1006,8 @@ func TestControlMessageValidationInspector_ActiveClustersChanged(t *testing.T) { } // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } // invalidTopics returns 3 invalid topics. @@ -989,8 +1079,3 @@ func inspectorFixture(t *testing.T, opts ...func(params *validation.InspectorPar signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) return validationInspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, idProvider, topicProviderOracle } - -func stopInspector(t *testing.T, cancel context.CancelFunc, inspector *validation.ControlMsgValidationInspector) { - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") -} From 6d20dbbade0c63a5ea2952c01fca8f17c16df90b Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 16 Jan 2024 11:02:41 -0800 Subject: [PATCH 33/57] fixes valid rpc test --- ...ntrol_message_validation_inspector_test.go | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 3b13b0e94f5..9b40e6c6b7f 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -597,10 +597,6 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cancel() unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - - t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold", func(t *testing.T) { - - }) } // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. @@ -616,19 +612,19 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { } // avoid unknown topics errors topicProviderOracle.UpdateTopics(topics) + cfg, err := config.DefaultConfig() + require.NoError(t, err) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + grafts := unittest.P2PRPCGraftFixtures(topics...) prunes := unittest.P2PRPCPruneFixtures(topics...) ihaves := unittest.P2PRPCIHaveFixtures(50, topics...) - iwants := unittest.P2PRPCIWantFixtures(2, 5) + // makes sure each iwant has enough message ids to trigger cache misses check + iwants := unittest.P2PRPCIWantFixtures(2, int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.CacheMissCheckSize)+1) pubsubMsgs := unittest.GossipSubMessageFixtures(10, topics[0]) - // avoid cache misses for iwant messages. - iwants[0].MessageIDs = ihaves[0].MessageIDs[:10] - iwants[1].MessageIDs = ihaves[1].MessageIDs[11:20] - expectedMsgIds := make([]string, 0) - expectedMsgIds = append(expectedMsgIds, ihaves[0].MessageIDs...) - expectedMsgIds = append(expectedMsgIds, ihaves[1].MessageIDs...) rpc := unittest.P2PRPCFixture( unittest.WithGrafts(grafts...), unittest.WithPrunes(prunes...), @@ -639,7 +635,14 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { id, ok := args[0].(string) require.True(t, ok) - require.Contains(t, expectedMsgIds, id) + for _, iwant := range iwants { + for _, messageID := range iwant.GetMessageIDs() { + if id == messageID { + return + } + } + } + require.Fail(t, "message id not found in iwant messages") }) from := unittest.PeerIdFixture(t) From a95eb29c2fdc1ca60bc1cf7969d47850910d26a0 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 16 Jan 2024 11:29:28 -0800 Subject: [PATCH 34/57] adds TestIWantInspection_DuplicateMessageIds_AboveThreshold --- ...ntrol_message_validation_inspector_test.go | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 9b40e6c6b7f..6ece4e52ee0 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -566,37 +566,6 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cancel() unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - - // ihave duplicate message ids beyond threshold should trigger invalid control message notification. - t.Run("iwant duplicate message ids above threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) - // oracle must be set even though iWant messages do not have topic IDs - duplicateMsgID := unittest.IdentifierFixture() - duplicates := flow.IdentifierList{duplicateMsgID, duplicateMsgID} - msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() - duplicateMsgIDIWant := unittest.P2PRPCIWantFixture(msgIds...) - - duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) - - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - require.Contains(t, msgIds, id) - }) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - - require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) } // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. @@ -653,6 +622,43 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestIWantInspection_DuplicateMessageIds_AboveThreshold ensures inspector disseminates invalid control message notifications for iWant messages when duplicate message ids exceeds allowed threshold. +func TestIWantInspection_DuplicateMessageIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) + // oracle must be set even though iWant messages do not have topic IDs + duplicateMsgID := unittest.IdentifierFixture() + duplicates := flow.IdentifierList{} + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // includes as many duplicates as allowed by the threshold + for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold)+2; i++ { + duplicates = append(duplicates, duplicateMsgID) + } + msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() + duplicateMsgIDIWant := unittest.P2PRPCIWantFixture(msgIds...) + + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, msgIds, id) + }).Maybe() // if iwant message ids count are not bigger than cache miss check size, this method is not called, anyway in this test we do not care about this method. + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestIWantInspection_CacheMiss_AboveThreshold ensures inspector disseminates invalid control message notifications for iWant messages when cache misses exceeds allowed threshold. func TestIWantInspection_CacheMiss_AboveThreshold(t *testing.T) { inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { From 52cbd02681829ee4b64161524a029cd3a522f102 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 16 Jan 2024 11:31:45 -0800 Subject: [PATCH 35/57] adds TestIWantInspection_DuplicateMessageIds_BelowThreshold --- ...ntrol_message_validation_inspector_test.go | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 6ece4e52ee0..99da7f6d9d6 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -622,6 +622,43 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestIWantInspection_DuplicateMessageIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for +// iWant messages when duplicate message ids are below allowed threshold. +func TestIWantInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) + // oracle must be set even though iWant messages do not have topic IDs + duplicateMsgID := unittest.IdentifierFixture() + duplicates := flow.IdentifierList{} + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // includes as many duplicates as allowed by the threshold + for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold)-2; i++ { + duplicates = append(duplicates, duplicateMsgID) + } + msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() + duplicateMsgIDIWant := unittest.P2PRPCIWantFixture(msgIds...) + + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) + + from := unittest.PeerIdFixture(t) + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, msgIds, id) + }).Maybe() // if iwant message ids count are not bigger than cache miss check size, this method is not called, anyway in this test we do not care about this method. + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestIWantInspection_DuplicateMessageIds_AboveThreshold ensures inspector disseminates invalid control message notifications for iWant messages when duplicate message ids exceeds allowed threshold. func TestIWantInspection_DuplicateMessageIds_AboveThreshold(t *testing.T) { inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) From 26adb68fc5d5a72f1e8d4ba9dd0cd20cccd5e293 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 16 Jan 2024 11:45:38 -0800 Subject: [PATCH 36/57] adds TestIHaveInspection_DuplicateMessageIds_BelowThreshold --- ...ntrol_message_validation_inspector_test.go | 128 +++++++++--------- 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 99da7f6d9d6..5b0cca5cfd5 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -503,69 +503,6 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cancel() unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - - // ihave duplicate topic ids below threshold should NOT trigger invalid control message notification. - t.Run("ihave duplicate message ids below threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{validTopic}) - duplicateMsgID := unittest.IdentifierFixture() - - cfg, err := config.DefaultConfig() - require.NoError(t, err) - msgIds := flow.IdentifierList{} - // includes as many duplicates as allowed by the threshold - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold; i++ { - msgIds = append(msgIds, duplicateMsgID) - } - duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) - duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) - from := unittest.PeerIdFixture(t) - - // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold - distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - - require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) - // TODO: this sleeps should be replaced with a queue size checker. - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) - - // ihave duplicate topic ids beyond threshold should trigger invalid control message notification. - t.Run("ihave duplicate message ids above threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{validTopic}) - duplicateMsgID := unittest.IdentifierFixture() - - cfg, err := config.DefaultConfig() - require.NoError(t, err) - msgIds := flow.IdentifierList{} - // includes as many duplicates as beyond the threshold - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold+2; i++ { - msgIds = append(msgIds, duplicateMsgID) - } - duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) - duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) - from := unittest.PeerIdFixture(t) - - // one notification should be disseminated for invalid messages when the number of duplicates exceeds the threshold - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateMessageIDErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - - require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) - // TODO: this sleeps should be replaced with a queue size checker. - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) } // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. @@ -622,6 +559,71 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestIHaveInspection_DuplicateMessageIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for +// iHave messages when duplicate message ids are below allowed threshold. +func TestIHaveInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) + duplicateMsgID := unittest.IdentifierFixture() + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + msgIds := flow.IdentifierList{} + // includes as many duplicates as allowed by the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold; i++ { + msgIds = append(msgIds, duplicateMsgID) + } + duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) + from := unittest.PeerIdFixture(t) + + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestIHaveInspection_DuplicateMessageIds_AboveThreshold ensures inspector disseminates an invalid control message notification for +// iHave messages when duplicate message ids are above allowed threshold. +func TestIHaveInspection_DuplicateMessageIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) + duplicateMsgID := unittest.IdentifierFixture() + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + msgIds := flow.IdentifierList{} + // includes as many duplicates as beyond the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold+2; i++ { + msgIds = append(msgIds, duplicateMsgID) + } + duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) + from := unittest.PeerIdFixture(t) + + // one notification should be disseminated for invalid messages when the number of duplicates exceeds the threshold + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateMessageIDErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestIWantInspection_DuplicateMessageIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for // iWant messages when duplicate message ids are below allowed threshold. func TestIWantInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { From ccd0ad50d71cd7456192b8cb2575548724b613ff Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 09:40:36 -0800 Subject: [PATCH 37/57] adds TestIHaveInspection_DuplicateTopicIds_BelowThreshold --- ...ntrol_message_validation_inspector_test.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 5b0cca5cfd5..37b6141c83a 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -559,6 +559,38 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestIHaveInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for +// iHave messages when duplicate topic ids are below allowed threshold. +func TestIHaveInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + validTopicIHave := unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...) + ihaves := []*pubsub_pb.ControlIHave{validTopicIHave} + // duplicate the valid topic id on other iHave messages but with different message ids + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold-1; i++ { + ihaves = append(ihaves, unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...)) + } + // creates an RPC with duplicate topic ids but different message ids + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(ihaves...)) + from := unittest.PeerIdFixture(t) + + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestIHaveInspection_DuplicateMessageIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for // iHave messages when duplicate message ids are below allowed threshold. func TestIHaveInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { From 8279cd9b6b0517580d25f403b4e12169b49ee56e Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 09:43:50 -0800 Subject: [PATCH 38/57] adds TestIHaveInspection_DuplicateTopicIds_AboveThreshold --- ...ntrol_message_validation_inspector_test.go | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 37b6141c83a..a27c6c66eaa 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -591,6 +591,39 @@ func TestIHaveInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestIHaveInspection_DuplicateTopicIds_AboveThreshold ensures inspector disseminate an invalid control message notification for +// iHave messages when duplicate topic ids are above allowed threshold. +func TestIHaveInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + validTopicIHave := unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...) + ihaves := []*pubsub_pb.ControlIHave{validTopicIHave} + // duplicate the valid topic id on other iHave messages but with different message ids up to the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold+2; i++ { + ihaves = append(ihaves, unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...)) + } + // creates an RPC with duplicate topic ids but different message ids + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(ihaves...)) + from := unittest.PeerIdFixture(t) + + // one notification should be disseminated for invalid messages when the number of duplicates exceeds the threshold + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestIHaveInspection_DuplicateMessageIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for // iHave messages when duplicate message ids are below allowed threshold. func TestIHaveInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { From 3baf643a744ea6bd933dae6cc8aa39f3ce360898 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 10:25:01 -0800 Subject: [PATCH 39/57] adds TestGraftInspection_InvalidTopic --- ...ntrol_message_validation_inspector_test.go | 185 +++++++++--------- 1 file changed, 94 insertions(+), 91 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index a27c6c66eaa..0cea9ef9266 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -412,97 +412,6 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cancel() unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - - // invalid graft topic ids should trigger invalid control message notification - t.Run("invalid graft topic", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) - unknownTopicGraft := unittest.P2PRPCGraftFixture(&unknownTopic) - malformedTopicGraft := unittest.P2PRPCGraftFixture(&malformedTopic) - invalidSporkIDTopicGraft := unittest.P2PRPCGraftFixture(&invalidSporkIDTopic) - - unknownTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(unknownTopicGraft)) - malformedTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(malformedTopicGraft)) - invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) - - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - - require.NoError(t, inspector.Inspect(from, unknownTopicReq)) - require.NoError(t, inspector.Inspect(from, malformedTopicReq)) - require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) - - // invalid prune topic ids should trigger invalid control message notification - t.Run("invalid prune topic", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) - unknownTopicPrune := unittest.P2PRPCPruneFixture(&unknownTopic) - malformedTopicPrune := unittest.P2PRPCPruneFixture(&malformedTopic) - invalidSporkIDTopicPrune := unittest.P2PRPCPruneFixture(&invalidSporkIDTopic) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) - unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(unknownTopicPrune)) - malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(malformedTopicPrune)) - invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(invalidSporkIDTopicPrune)) - - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - - require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) - require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) - require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) - - // invalid ihave topic ids should trigger invalid control message notification - t.Run("invalid ihave topic", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) - unknownTopicIhave := unittest.P2PRPCIHaveFixture(&unknownTopic, unittest.IdentifierListFixture(5).Strings()...) - malformedTopicIhave := unittest.P2PRPCIHaveFixture(&malformedTopic, unittest.IdentifierListFixture(5).Strings()...) - invalidSporkIDTopicIhave := unittest.P2PRPCIHaveFixture(&invalidSporkIDTopic, unittest.IdentifierListFixture(5).Strings()...) - - unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(unknownTopicIhave)) - malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(malformedTopicIhave)) - invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(invalidSporkIDTopicIhave)) - - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - - require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) - require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) - require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) } // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. @@ -559,6 +468,100 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestGraftInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for +// graft messages when the topic is invalid. +func TestGraftInspection_InvalidTopic(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + // create unknown topic + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) + unknownTopicGraft := unittest.P2PRPCGraftFixture(&unknownTopic) + malformedTopicGraft := unittest.P2PRPCGraftFixture(&malformedTopic) + invalidSporkIDTopicGraft := unittest.P2PRPCGraftFixture(&invalidSporkIDTopic) + + unknownTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(unknownTopicGraft)) + malformedTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(malformedTopicGraft)) + invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, unknownTopicReq)) + require.NoError(t, inspector.Inspect(from, malformedTopicReq)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestPruneInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for +// prune messages when the topic is invalid. +func TestPruneInspection_InvalidTopic(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + // create unknown topic + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) + unknownTopicPrune := unittest.P2PRPCPruneFixture(&unknownTopic) + malformedTopicPrune := unittest.P2PRPCPruneFixture(&malformedTopic) + invalidSporkIDTopicPrune := unittest.P2PRPCPruneFixture(&invalidSporkIDTopic) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) + unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(unknownTopicPrune)) + malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(malformedTopicPrune)) + invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(invalidSporkIDTopicPrune)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) + require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestIHaveInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for +// iHave messages when the topic is invalid. +func TestIHaveInspection_InvalidTopic(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + // create unknown topic + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) + unknownTopicIhave := unittest.P2PRPCIHaveFixture(&unknownTopic, unittest.IdentifierListFixture(5).Strings()...) + malformedTopicIhave := unittest.P2PRPCIHaveFixture(&malformedTopic, unittest.IdentifierListFixture(5).Strings()...) + invalidSporkIDTopicIhave := unittest.P2PRPCIHaveFixture(&invalidSporkIDTopic, unittest.IdentifierListFixture(5).Strings()...) + + unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(unknownTopicIhave)) + malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(malformedTopicIhave)) + invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(invalidSporkIDTopicIhave)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) + require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestIHaveInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for // iHave messages when duplicate topic ids are below allowed threshold. func TestIHaveInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { From 1e7d3fb748b9f249ae172af2efb66926d6d8ddb2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 10:32:01 -0800 Subject: [PATCH 40/57] adds TestPrueInspection_DuplicateTopicIds_BelowThreshold --- ...ntrol_message_validation_inspector_test.go | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 0cea9ef9266..27467e55a6a 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -385,33 +385,6 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cancel() unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - - // duplicate prune topic ids below threshold should NOT trigger invalid control message notification - t.Run("duplicate prune topic ids below threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{duplicateTopic}) - var prunes []*pubsub_pb.ControlPrune - cfg, err := config.DefaultConfig() - require.NoError(t, err) - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { - prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) - } - from := unittest.PeerIdFixture(t) - rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) - // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold - distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) - - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) } // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. @@ -468,6 +441,34 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestPruneInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate invalid control message notifications +// for a valid RPC with duplicate prune topic ids below the threshold. +func TestPrueInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var prunes []*pubsub_pb.ControlPrune + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { + prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestGraftInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for // graft messages when the topic is invalid. func TestGraftInspection_InvalidTopic(t *testing.T) { From 6f2ce0bbba8765427b54cb5f2688b66490ebdf57 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 10:33:40 -0800 Subject: [PATCH 41/57] adds TestPruneInspection_DuplicateTopicIds_AboveThreshold --- ...ntrol_message_validation_inspector_test.go | 69 ++++++++++--------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 27467e55a6a..8c893a3209a 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -351,40 +351,6 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cancel() unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - - // duplicate prune topic ids beyond threshold should trigger invalid control message notification - t.Run("duplicate prune topic ids beyond threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{duplicateTopic}) - var prunes []*pubsub_pb.ControlPrune - cfg, err := config.DefaultConfig() - require.NoError(t, err) - // we need threshold + 1 to trigger the invalid control message notification; as the first duplicate topic id is not counted - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { - prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) - } - from := unittest.PeerIdFixture(t) - rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { - notification, ok := args[0].(*p2p.InvCtrlMsgNotif) - require.True(t, ok) - require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") - require.Equal(t, from, notification.PeerID) - require.Equal(t, p2pmsg.CtrlMsgPrune, notification.MsgType) - require.True(t, validation.IsDuplicateTopicErr(notification.Error)) - }) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) - - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) } // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. @@ -441,6 +407,41 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestPruneInspection_DuplicateTopicIds_AboveThreshold ensures inspector disseminates an invalid control message notification for +// prune messages when the number of duplicate topic ids is above the threshold. +func TestPruneInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var prunes []*pubsub_pb.ControlPrune + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // we need threshold + 1 to trigger the invalid control message notification; as the first duplicate topic id is not counted + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { + prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, from, notification.PeerID) + require.Equal(t, p2pmsg.CtrlMsgPrune, notification.MsgType) + require.True(t, validation.IsDuplicateTopicErr(notification.Error)) + }) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestPruneInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate invalid control message notifications // for a valid RPC with duplicate prune topic ids below the threshold. func TestPrueInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { From 2c72e3d46c189dec72a4e586e6cd6877ab76d424 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 10:45:05 -0800 Subject: [PATCH 42/57] adds TestGraftInspection_DuplicateTopicIds_BelowThreshold --- ...ntrol_message_validation_inspector_test.go | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 8c893a3209a..9fe1517aa24 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -324,33 +324,6 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { cancel() unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - - // duplicate graft topic ids below threshold should NOT trigger invalid control message notification - t.Run("duplicate graft topic ids below threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{duplicateTopic}) - var grafts []*pubsub_pb.ControlGraft - cfg, err := config.DefaultConfig() - require.NoError(t, err) - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { - grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) - } - from := unittest.PeerIdFixture(t) - rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) - // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold - distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) - - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) } // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. @@ -407,6 +380,34 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestGraftInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate invalid control message notifications +// for a valid RPC with duplicate graft topic ids below the threshold. +func TestGraftInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var grafts []*pubsub_pb.ControlGraft + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { + grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestPruneInspection_DuplicateTopicIds_AboveThreshold ensures inspector disseminates an invalid control message notification for // prune messages when the number of duplicate topic ids is above the threshold. func TestPruneInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { From 176c42f5fcc698e66929d48058a3e3ba32ed7177 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 10:47:07 -0800 Subject: [PATCH 43/57] adds TestGraftInspection_DuplicateTopicIds_AboveThreshold --- ...ntrol_message_validation_inspector_test.go | 134 +++++++++--------- 1 file changed, 64 insertions(+), 70 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 9fe1517aa24..2eff317c450 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -288,44 +288,6 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { }) } -// TestControlMessageValidationInspector_processInspectRPCReq verifies the correct behavior of control message validation. -// It ensures that valid RPC control messages do not trigger erroneous invalid control message notifications, -// while all types of invalid control messages trigger expected notifications. -func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { - // duplicate graft topic ids beyond threshold should trigger invalid control message notification - t.Run("duplicate graft topic ids beyond threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{duplicateTopic}) - var grafts []*pubsub_pb.ControlGraft - cfg, err := config.DefaultConfig() - require.NoError(t, err) - for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { - grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) - } - from := unittest.PeerIdFixture(t) - rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { - notification, ok := args[0].(*p2p.InvCtrlMsgNotif) - require.True(t, ok) - require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") - require.Equal(t, from, notification.PeerID) - require.Equal(t, p2pmsg.CtrlMsgGraft, notification.MsgType) - require.True(t, validation.IsDuplicateTopicErr(notification.Error)) - }) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) - - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") - }) -} - // TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. func TestControlMessageInspection_ValidRpc(t *testing.T) { inspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, _, topicProviderOracle := inspectorFixture(t) @@ -380,6 +342,38 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +// TestGraftInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for +// graft messages when the topic is invalid. +func TestGraftInspection_InvalidTopic(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + // create unknown topic + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) + unknownTopicGraft := unittest.P2PRPCGraftFixture(&unknownTopic) + malformedTopicGraft := unittest.P2PRPCGraftFixture(&malformedTopic) + invalidSporkIDTopicGraft := unittest.P2PRPCGraftFixture(&invalidSporkIDTopic) + + unknownTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(unknownTopicGraft)) + malformedTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(malformedTopicGraft)) + invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, unknownTopicReq)) + require.NoError(t, inspector.Inspect(from, malformedTopicReq)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestGraftInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate invalid control message notifications // for a valid RPC with duplicate graft topic ids below the threshold. func TestGraftInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { @@ -408,6 +402,38 @@ func TestGraftInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } +func TestGraftInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var grafts []*pubsub_pb.ControlGraft + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { + grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, from, notification.PeerID) + require.Equal(t, p2pmsg.CtrlMsgGraft, notification.MsgType) + require.True(t, validation.IsDuplicateTopicErr(notification.Error)) + }) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + // TestPruneInspection_DuplicateTopicIds_AboveThreshold ensures inspector disseminates an invalid control message notification for // prune messages when the number of duplicate topic ids is above the threshold. func TestPruneInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { @@ -471,38 +497,6 @@ func TestPrueInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } -// TestGraftInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for -// graft messages when the topic is invalid. -func TestGraftInspection_InvalidTopic(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) - unknownTopicGraft := unittest.P2PRPCGraftFixture(&unknownTopic) - malformedTopicGraft := unittest.P2PRPCGraftFixture(&malformedTopic) - invalidSporkIDTopicGraft := unittest.P2PRPCGraftFixture(&invalidSporkIDTopic) - - unknownTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(unknownTopicGraft)) - malformedTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(malformedTopicGraft)) - invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) - - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - - inspector.Start(signalerCtx) - unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - - require.NoError(t, inspector.Inspect(from, unknownTopicReq)) - require.NoError(t, inspector.Inspect(from, malformedTopicReq)) - require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") -} - // TestPruneInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for // prune messages when the topic is invalid. func TestPruneInspection_InvalidTopic(t *testing.T) { From 98d1294156f1495011ecfeb214ab130fd97e55fb Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 11:09:39 -0800 Subject: [PATCH 44/57] removes cache miss threshold --- config/default-config.yml | 2 -- network/netconf/flags.go | 10 +++------ .../p2p/config/gossipsub_rpc_inspectors.go | 21 +++++++++++-------- .../control_message_validation_inspector.go | 13 ++++++------ ...ntrol_message_validation_inspector_test.go | 13 +++--------- network/p2p/inspector/validation/errors.go | 8 +++---- 6 files changed, 28 insertions(+), 39 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index 2d2c896f34f..73c3519fac1 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -196,8 +196,6 @@ network-config: # The allowed threshold of iWant messages received without a corresponding tracked iHave message that was sent. # If the cache miss threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. cache-miss-threshold: 500 - # The iWants size at which message id cache misses will be checked. - cache-miss-check-size: 1000 # The max allowed number of duplicate message ids in a single iwant message. # Note that ideally there should be no duplicate message ids in a single iwant message but # we allow for some tolerance to avoid penalizing peers that are not malicious diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 8c75ca00d49..32e57d10c67 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -98,7 +98,6 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageCountThreshold), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageIdCountThreshold), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissCheckSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MaxSampleSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), @@ -285,14 +284,11 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageIdCountThreshold), config.GossipSub.RpcInspector.Validation.IWant.MessageIdCountThreshold, "threshold for the number of message ids on a single iwant control message to accept, if exceeded the RPC message ids will be sampled and truncated") - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), config.GossipSub.RpcInspector.Validation.IWant.CacheMissThreshold, "max number of cache misses (untracked) allowed in a single iWant control message, if exceeded a misbehavior report will be created") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissCheckSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.CacheMissCheckSize, - "threshold for the size of iwant control message that triggers cache miss check") - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), - config.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), + config.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIdThreshold, "max allowed duplicate message IDs in a single iWant control message, if exceeded a misbehavior report will be created") flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MaxSampleSizeKey), config.GossipSub.RpcInspector.Validation.PublishMessages.MaxSampleSize, diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index 6e51090e716..fec58ce5fe1 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -85,7 +85,6 @@ const ( MessageCountThreshold = "message-count-threshold" MessageIdCountThreshold = "message-id-count-threshold" CacheMissThresholdKey = "cache-miss-threshold" - CacheMissCheckSizeKey = "cache-miss-check-size" DuplicateMsgIDThresholdKey = "duplicate-message-id-threshold" ) @@ -104,14 +103,18 @@ type IWantRpcInspectionParameters struct { // a random sample of message ids will be taken and the iWant message will be truncated to this sample size. // The sample size is equal to the configured MessageIdCountThreshold. MessageIdCountThreshold int `validate:"gte=0" mapstructure:"message-id-count-threshold"` - // CacheMissThreshold the threshold of missing corresponding iHave messages for iWant messages received before an invalid control message notification is disseminated. - // If the cache miss threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. - CacheMissThreshold float64 `validate:"gt=0" mapstructure:"cache-miss-threshold"` - // CacheMissCheckSize the iWants size at which message id cache misses will be checked. - CacheMissCheckSize int `validate:"gt=0" mapstructure:"cache-miss-check-size"` - // DuplicateMsgIDThreshold maximum allowed duplicate message IDs in a single iWant control message. - // If the duplicate message threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. - DuplicateMsgIDThreshold float64 `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` + // CacheMissThreshold is the threshold of tolerance for the total cache misses in all iWant messages in a single RPC message. + // When the total number of cache misses in all iWant messages in a single RPC message exceeds this threshold, the inspection of message will fail. + // An iWant message is considered a cache miss if it contains a message id that is not present in the local cache for iHave messages, i.e., the node + // does not have a record of an iHave message for this message id. + // When the total number of cache misses in all iWant messages in a single RPC message exceeds this threshold, the inspection of message will fail, and + // a single misbehavior notification will be reported. + CacheMissThreshold int `validate:"gt=0" mapstructure:"cache-miss-threshold"` + // DuplicateMsgIdThreshold is the maximum allowed number of duplicate message ids in a all iWant messages in a single RPC message. + // Each iWant message represents the list of message ids, and this parameter controls the maximum number of duplicate message ids + // that can be included in all iWant messages in a single RPC message. When the total number of duplicate message ids in a single iWant message exceeds this threshold, + // a single misbehavior notification will be reported, and the inspection of message will fail. + DuplicateMsgIdThreshold int `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` } const ( diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 767f0d5fc66..1ae7657255f 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -491,30 +491,29 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant lg = lg.With(). Int("iwant_msg_count", len(iWants)). - Float64("cache_misses_threshold", c.config.IWant.CacheMissThreshold). - Float64("duplicates_threshold", c.config.IWant.DuplicateMsgIDThreshold).Logger() + Int("cache_misses_threshold", c.config.IWant.CacheMissThreshold). + Int("duplicates_threshold", c.config.IWant.DuplicateMsgIdThreshold).Logger() lg.Trace().Msg("validating sample of message ids from iwant control message") totalMessageIds := 0 for _, iWant := range iWants { messageIds := iWant.GetMessageIDs() - checkCacheMisses := len(messageIds) >= c.config.IWant.CacheMissCheckSize messageIDCount := uint(len(messageIds)) for _, messageID := range messageIds { // check duplicate allowed threshold if duplicateMsgIdTracker.track(messageID) > 1 { // ideally, an iWant message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once. duplicateMessageIds++ - if float64(duplicateMessageIds) > c.config.IWant.DuplicateMsgIDThreshold { + if duplicateMessageIds > c.config.IWant.DuplicateMsgIdThreshold { c.metrics.OnIWantDuplicateMessageIdsExceedThreshold() - return NewIWantDuplicateMsgIDThresholdErr(duplicateMessageIds, messageIDCount, c.config.IWant.DuplicateMsgIDThreshold) + return NewIWantDuplicateMsgIDThresholdErr(duplicateMessageIds, messageIDCount, c.config.IWant.DuplicateMsgIdThreshold) } } // check cache miss threshold - if checkCacheMisses && !c.rpcTracker.WasIHaveRPCSent(messageID) { + if !c.rpcTracker.WasIHaveRPCSent(messageID) { cacheMisses++ - if float64(cacheMisses) > c.config.IWant.CacheMissThreshold { + if cacheMisses > c.config.IWant.CacheMissThreshold { c.metrics.OnIWantCacheMissMessageIdsExceedThreshold() return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWant.CacheMissThreshold) } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 2eff317c450..71682d6778e 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -301,17 +301,13 @@ func TestControlMessageInspection_ValidRpc(t *testing.T) { } // avoid unknown topics errors topicProviderOracle.UpdateTopics(topics) - cfg, err := config.DefaultConfig() - require.NoError(t, err) - inspector.Start(signalerCtx) unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) grafts := unittest.P2PRPCGraftFixtures(topics...) prunes := unittest.P2PRPCPruneFixtures(topics...) ihaves := unittest.P2PRPCIHaveFixtures(50, topics...) - // makes sure each iwant has enough message ids to trigger cache misses check - iwants := unittest.P2PRPCIWantFixtures(2, int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.CacheMissCheckSize)+1) + iwants := unittest.P2PRPCIWantFixtures(2, 50) pubsubMsgs := unittest.GossipSubMessageFixtures(10, topics[0]) rpc := unittest.P2PRPCFixture( @@ -699,7 +695,7 @@ func TestIWantInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) // includes as many duplicates as allowed by the threshold - for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold)-2; i++ { + for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIdThreshold)-2; i++ { duplicates = append(duplicates, duplicateMsgID) } msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() @@ -735,7 +731,7 @@ func TestIWantInspection_DuplicateMessageIds_AboveThreshold(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) // includes as many duplicates as allowed by the threshold - for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold)+2; i++ { + for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIdThreshold)+2; i++ { duplicates = append(duplicates, duplicateMsgID) } msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() @@ -766,7 +762,6 @@ func TestIWantInspection_DuplicateMessageIds_AboveThreshold(t *testing.T) { // TestIWantInspection_CacheMiss_AboveThreshold ensures inspector disseminates invalid control message notifications for iWant messages when cache misses exceeds allowed threshold. func TestIWantInspection_CacheMiss_AboveThreshold(t *testing.T) { inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWant.CacheMissCheckSize = 99 // when size of an iwant message is above this threshold, it is checked for cache misses // set high cache miss threshold to ensure we only disseminate notification when it is exceeded params.Config.IWant.CacheMissThreshold = 900 }) @@ -810,8 +805,6 @@ func TestIWantInspection_CacheMiss_AboveThreshold(t *testing.T) { func TestIWantInspection_CacheMiss_BelowThreshold(t *testing.T) { inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - // if size of iwants is below cache miss check size, it is not checked for cache misses - params.Config.IWant.CacheMissCheckSize = 90 // set high cache miss threshold to ensure that we do not disseminate notification in this test params.Config.IWant.CacheMissThreshold = 99 }) diff --git a/network/p2p/inspector/validation/errors.go b/network/p2p/inspector/validation/errors.go index 0215e18be27..b7e897c9ffc 100644 --- a/network/p2p/inspector/validation/errors.go +++ b/network/p2p/inspector/validation/errors.go @@ -12,7 +12,7 @@ import ( type IWantDuplicateMsgIDThresholdErr struct { duplicates int sampleSize uint - threshold float64 + threshold int } func (e IWantDuplicateMsgIDThresholdErr) Error() string { @@ -20,7 +20,7 @@ func (e IWantDuplicateMsgIDThresholdErr) Error() string { } // NewIWantDuplicateMsgIDThresholdErr returns a new IWantDuplicateMsgIDThresholdErr. -func NewIWantDuplicateMsgIDThresholdErr(duplicates int, sampleSize uint, threshold float64) IWantDuplicateMsgIDThresholdErr { +func NewIWantDuplicateMsgIDThresholdErr(duplicates int, sampleSize uint, threshold int) IWantDuplicateMsgIDThresholdErr { return IWantDuplicateMsgIDThresholdErr{duplicates, sampleSize, threshold} } @@ -34,7 +34,7 @@ func IsIWantDuplicateMsgIDThresholdErr(err error) bool { type IWantCacheMissThresholdErr struct { cacheMissCount int // total iwant cache misses sampleSize uint - threshold float64 + threshold int } func (e IWantCacheMissThresholdErr) Error() string { @@ -42,7 +42,7 @@ func (e IWantCacheMissThresholdErr) Error() string { } // NewIWantCacheMissThresholdErr returns a new IWantCacheMissThresholdErr. -func NewIWantCacheMissThresholdErr(cacheMissCount int, sampleSize uint, threshold float64) IWantCacheMissThresholdErr { +func NewIWantCacheMissThresholdErr(cacheMissCount int, sampleSize uint, threshold int) IWantCacheMissThresholdErr { return IWantCacheMissThresholdErr{cacheMissCount, sampleSize, threshold} } From 3116787339667622252aa402e8d6e2e5848f080a Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 11:22:42 -0800 Subject: [PATCH 45/57] lint fix --- network/p2p/inspector/validation/errors_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/p2p/inspector/validation/errors_test.go b/network/p2p/inspector/validation/errors_test.go index a180badf28f..9e67f120140 100644 --- a/network/p2p/inspector/validation/errors_test.go +++ b/network/p2p/inspector/validation/errors_test.go @@ -59,7 +59,7 @@ func TestDuplicateMessageIDErrRoundTrip(t *testing.T) { // TestIWantCacheMissThresholdErrRoundTrip ensures correct error formatting for IWantCacheMissThresholdErr. func TestIWantCacheMissThresholdErrRoundTrip(t *testing.T) { - err := NewIWantCacheMissThresholdErr(5, 10, .4) + err := NewIWantCacheMissThresholdErr(5, 10, 5) // tests the error message formatting. expectedErrMsg := "5/10 iWant cache misses exceeds the allowed threshold: 0.400000" @@ -75,7 +75,7 @@ func TestIWantCacheMissThresholdErrRoundTrip(t *testing.T) { // TestIWantDuplicateMsgIDThresholdErrRoundTrip ensures correct error formatting for IWantDuplicateMsgIDThresholdErr. func TestIWantDuplicateMsgIDThresholdErrRoundTrip(t *testing.T) { - err := NewIWantDuplicateMsgIDThresholdErr(5, 10, .4) + err := NewIWantDuplicateMsgIDThresholdErr(5, 10, 5) // tests the error message formatting. expectedErrMsg := "5/10 iWant duplicate message ids exceeds the allowed threshold: 0.400000" From 9b7ed6a0c8183cbbc7deb4571db9e4f01d01d8d2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 11:23:18 -0800 Subject: [PATCH 46/57] lint fix --- network/p2p/inspector/validation/errors.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/p2p/inspector/validation/errors.go b/network/p2p/inspector/validation/errors.go index b7e897c9ffc..bb73f9cba9b 100644 --- a/network/p2p/inspector/validation/errors.go +++ b/network/p2p/inspector/validation/errors.go @@ -16,7 +16,7 @@ type IWantDuplicateMsgIDThresholdErr struct { } func (e IWantDuplicateMsgIDThresholdErr) Error() string { - return fmt.Sprintf("%d/%d iWant duplicate message ids exceeds the allowed threshold: %f", e.duplicates, e.sampleSize, e.threshold) + return fmt.Sprintf("%d/%d iWant duplicate message ids exceeds the allowed threshold: %d", e.duplicates, e.sampleSize, e.threshold) } // NewIWantDuplicateMsgIDThresholdErr returns a new IWantDuplicateMsgIDThresholdErr. @@ -38,7 +38,7 @@ type IWantCacheMissThresholdErr struct { } func (e IWantCacheMissThresholdErr) Error() string { - return fmt.Sprintf("%d/%d iWant cache misses exceeds the allowed threshold: %f", e.cacheMissCount, e.sampleSize, e.threshold) + return fmt.Sprintf("%d/%d iWant cache misses exceeds the allowed threshold: %d", e.cacheMissCount, e.sampleSize, e.threshold) } // NewIWantCacheMissThresholdErr returns a new IWantCacheMissThresholdErr. From 030604e3030e3e964565352116fa39c2a6a826fa Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 15:12:12 -0800 Subject: [PATCH 47/57] fixes metrics --- config/default-config.yml | 2 +- module/metrics.go | 78 +++---- .../gossipsub_rpc_validation_inspector.go | 218 +++++++++--------- module/metrics/noop.go | 7 +- module/mock/gossip_sub_metrics.go | 6 +- ...ip_sub_rpc_validation_inspector_metrics.go | 6 +- module/mock/lib_p2_p_metrics.go | 6 +- module/mock/network_metrics.go | 6 +- .../control_message_validation_inspector.go | 17 +- 9 files changed, 173 insertions(+), 173 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index 73c3519fac1..3668c2ca215 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -166,7 +166,7 @@ network-config: # in a single RPC message exceeds this threshold, a random sample of iHave messages will be taken and the RPC message will be truncated to this sample size. # The sample size is equal to the configured message-count-threshold. message-count-threshold: 1000 - # MessageIdCountThreshold is the maximum allowed number of message ids in a single iHave message. + # The maximum allowed number of message ids in a single iHave message. # Each iHave message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids # that can be included in a single iHave message. When the total number of message ids in a single iHave message exceeds this threshold, # a random sample of message ids will be taken and the iHave message will be truncated to this sample size. diff --git a/module/metrics.go b/module/metrics.go index 2c925272dd5..4315d6c017b 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -277,88 +277,82 @@ type GossipSubRpcValidationInspectorMetrics interface { // diff: the number of control messages truncated. OnControlMessagesTruncated(messageType p2pmsg.ControlMessageType, diff int) - // OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on an iWant message at the end of the async inspection iWants - // across one RPC. - // Args: - // duplicateCount: the number of duplicate message ids received by the node on an iWant message at the end of the async inspection iWants. - // cacheMissCount: the number of cache miss message ids received by the node on an iWant message at the end of the async inspection iWants. + // OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on iWant messages at the end of the async inspection iWants + // across one RPC, regardless of the result of the inspection. + // + // duplicateCount: the total number of duplicate message ids received by the node on the iWant messages at the end of the async inspection of the RPC. + // cacheMissCount: the total number of cache miss message ids received by the node on the iWant message at the end of the async inspection of the RPC. OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) - // OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of duplicate message ids - // received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. + // OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total number of duplicate message ids + // received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. OnIWantDuplicateMessageIdsExceedThreshold() - // OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of cache miss message ids - // received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. + // OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total + // number of cache miss message ids received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. OnIWantCacheMissMessageIdsExceedThreshold() - // OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages, regardless of the result of the inspection. - // It tracks the number of duplicate topic ids and duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. + // OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages of a single RPC, regardless of the result of the inspection. + // It tracks the number of duplicate topic ids and duplicate message ids received by the node on the iHave messages of that single RPC at the end of the async inspection iHaves. // Args: - // duplicateTopicIds: the number of duplicate topic ids received by the node on an iHave message at the end of the async inspection iHaves. - // duplicateMessageIds: the number of duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. + // + // duplicateTopicIds: the total number of duplicate topic ids received by the node on the iHave messages at the end of the async inspection of the RPC. + // duplicateMessageIds: the number of duplicate message ids received by the node on the iHave messages at the end of the async inspection of the RPC. OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) - // OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate topic ids - // received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. + // OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate topic ids + // received by the node on the iHave messages of that RPC exceeding the threshold, which results in a misbehaviour report. OnIHaveDuplicateTopicIdsExceedThreshold() - // OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate message ids + // OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate message ids // received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. OnIHaveDuplicateMessageIdsExceedThreshold() - // OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message failed due to an invalid topic id. + // OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message type on a single RPC failed due to an invalid topic id. // Args: // - messageType: the type of the control message that was truncated. OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) - // OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message failed due to active cluster ids not set inspection failure. + // OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message type on a single RPC failed due to active cluster ids not set inspection failure. // This is not causing a misbehaviour report. OnActiveClusterIDsNotSetErr() - // OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message failed due to unstaked peer inspection failure. + // OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message type on a single RPC failed due to unstaked peer inspection failure. // This is not causing a misbehaviour report. OnUnstakedPeerInspectionFailed() - // OnInvalidControlMessageSent tracks the number of times that the async inspection of a control message failed and an invalid control message was sent. - OnInvalidControlMessageSent() - - // OnInvalidSenderForPublishMessage tracks the number of times that the async inspection of a publish message detected an invalid sender. - // Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. - OnInvalidSenderForPublishMessage() + // OnInvalidControlMessageNotificationSent tracks the number of times that the async inspection of a control message failed and resulted in dissemination of an invalid control message was sent. + OnInvalidControlMessageNotificationSent() // OnPublishMessagesInspectionErrorExceedsThreshold tracks the number of times that async inspection of publish messages failed due to the number of errors. OnPublishMessagesInspectionErrorExceedsThreshold() - // OnPublishMessageInvalidSubscription tracks the number of times that the async inspection of a publish message detected an invalid subscription. - // Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. - OnPublishMessageInvalidSubscription() - - // OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a prune message failed due to the number of duplicate topic ids - // received by the node on prune messages of the same rpc excesses threshold, which results in a misbehaviour report. - // Note that it does cause a misbehaviour report. + // OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of prune messages for an RPC failed due to the number of duplicate topic ids + // received by the node on prune messages of the same RPC excesses threshold, which results in a misbehaviour report. OnPruneDuplicateTopicIdsExceedThreshold() - // OnPruneMessageInspected is called at the end of the async inspection of prune messages, regardless of the result of the inspection. + // OnPruneMessageInspected is called at the end of the async inspection of prune messages of the RPC, regardless of the result of the inspection. // Args: - // duplicateTopicIds: the number of duplicate topic ids received by the node on a prune message at the end of the async inspection prunes. + // duplicateTopicIds: the number of duplicate topic ids received by the node on the prune messages of the RPC at the end of the async inspection prunes. OnPruneMessageInspected(duplicateTopicIds int) - // OnGraftDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a graft message failed due to the number of duplicate topic ids. - // received by the node on graft messages of the same rpc excesses threshold, which results in a misbehaviour report. + // OnGraftDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of the graft messages of a single RPC failed due to the number of duplicate topic ids + // received by the node on graft messages of the same RPC excesses threshold, which results in a misbehaviour report. OnGraftDuplicateTopicIdsExceedThreshold() - // OnGraftMessageInspected is called at the end of the async inspection of graft messages, regardless of the result of the inspection. + // OnGraftMessageInspected is called at the end of the async inspection of graft messages of a single RPC, regardless of the result of the inspection. // Args: - // duplicateTopicIds: the number of duplicate topic ids received by the node on a graft message at the end of the async inspection grafts. + // duplicateTopicIds: the number of duplicate topic ids received by the node on the graft messages at the end of the async inspection of a single RPC. OnGraftMessageInspected(duplicateTopicIds int) - // OnPublishMessageInspected tracks the number of errors that occurred during the async inspection of publish messages. - // Note that this function is called on each publish message received by the node regardless of the result of the inspection. - // If the number of errors exceeds the threshold, a misbehaviour report is sent, but this function is still called. + // OnPublishMessageInspected is called at the end of the async inspection of publish messages of a single RPC, regardless of the result of the inspection. + // It tracks the total number of errors detected during the async inspection of the rpc together with their individual breakdown. // Args: // - errCount: the number of errors that occurred during the async inspection of publish messages. - OnPublishMessageInspected(errCount int) + // - invalidTopicIdsCount: the number of times that an invalid topic id was detected during the async inspection of publish messages. + // - invalidSubscriptionsCount: the number of times that an invalid subscription was detected during the async inspection of publish messages. + // - invalidSendersCount: the number of times that an invalid sender was detected during the async inspection of publish messages. + OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index b8dd232d36c..6b79e8c477d 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -52,14 +52,15 @@ type GossipSubRpcValidationInspectorMetrics struct { iWantCacheMissMessageIdExceedThresholdCount prometheus.Counter // inspection result - errActiveClusterIdsNotSetCount prometheus.Counter - errUnstakedPeerInspectionFailedCount prometheus.Counter - invalidControlMessageSentCount prometheus.Counter + errActiveClusterIdsNotSetCount prometheus.Counter + errUnstakedPeerInspectionFailedCount prometheus.Counter + invalidControlMessageNotificationSentCount prometheus.Counter // publish messages publishMessageInspectionErrExceedThresholdCount prometheus.Counter - publishMessageInvalidSenderCount prometheus.Counter - publishMessageInvalidSubscriptionsCount prometheus.Counter + publishMessageInvalidSenderCountHistogram prometheus.Histogram + publishMessageInvalidSubscriptionsHistogram prometheus.Histogram + publishMessageInvalidTopicIdHistogram prometheus.Histogram publishMessageInspectedErrHistogram prometheus.Histogram } @@ -112,7 +113,7 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_iwant_total", - Help: "number of received iwant messages from gossipsub protocol", + Help: "total number of received iwant messages from gossipsub protocol", }) gc.receivedIWantMsgIDsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ @@ -126,172 +127,185 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_graft_total", - Help: "number of received graft messages from gossipsub protocol", + Help: "total number of received graft messages from gossipsub protocol", }) gc.receivedPruneCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_prune_total", - Help: "number of received prune messages from gossipsub protocol", + Help: "total number of received prune messages from gossipsub protocol", }) gc.receivedPublishMessageCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_publish_message_total", - Help: "number of received publish messages from gossipsub protocol", + Help: "total number of received publish messages from gossipsub protocol", }) gc.incomingRpcCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_incoming_rpc_total", - Help: "number of incoming rpc messages from gossipsub protocol", + Help: "total number of incoming rpc messages from gossipsub protocol", }) gc.iHaveDuplicateMessageIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Buckets: []float64{1, 100, 1000}, - Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids", - Help: "number of duplicate message ids received from gossipsub protocol during the async inspection", + Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids_count", + Help: "number of duplicate message ids received from gossipsub protocol during the async inspection of a single RPC", }) gc.iHaveDuplicateTopicIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Buckets: []float64{1, 100, 1000}, - Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids", - Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection", + Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids_count", + Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection of a single RPC", }) gc.iHaveDuplicateMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids_exceed_threshold_total", - Help: "number of times that the async inspection of iHave messages failed due to the number of duplicate message ids exceeding the threshold", + Help: "total number of times that the async inspection of iHave messages failed due to the number of duplicate message ids exceeding the threshold", }) gc.iHaveDuplicateTopicIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids_exceed_threshold_total", - Help: "number of times that the async inspection of iHave messages failed due to the number of duplicate topic ids exceeding the threshold", + Help: "total number of times that the async inspection of iHave messages failed due to the number of duplicate topic ids exceeding the threshold", }) gc.iWantDuplicateMessageIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids", + Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids_count", Buckets: []float64{1, 100, 1000}, - Help: "number of duplicate message ids received from gossipsub protocol during the async inspection", + Help: "number of duplicate message ids received from gossipsub protocol during the async inspection of a single RPC", }) gc.iWantCacheMissHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids", + Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids_count", Buckets: []float64{1, 100, 1000}, - Help: "number of cache miss message ids received from gossipsub protocol during the async inspection", + Help: "total number of cache miss message ids received from gossipsub protocol during the async inspection of a single RPC", }) gc.iWantDuplicateMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids_exceed_threshold_total", - Help: "number of times that the async inspection of iWant messages failed due to the number of duplicate message ids ", + Help: "total number of times that the async inspection of iWant messages failed due to the number of duplicate message ids ", }) gc.iWantCacheMissMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids_exceed_threshold_total", - Help: "number of times that the async inspection of iWant messages failed due to the number of cache miss message ids ", + Help: "total number of times that the async inspection of iWant messages failed due to the number of cache miss message ids ", }) gc.ctrlMsgInvalidTopicIdCount = *promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "control_message_invalid_topic_id_count", - Help: "number of control messages with invalid topic id received from gossipsub protocol during the async inspection", + Name: gc.prefix + "control_message_invalid_topic_id_total", + Help: "total number of control messages with invalid topic id received from gossipsub protocol during the async inspection", }, []string{LabelMessage}) gc.errActiveClusterIdsNotSetCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "active_cluster_ids_not_inspection_err_count", - Help: "number of inspection errors due to active cluster ids not set inspection failure", + Name: gc.prefix + "active_cluster_ids_not_inspection_error_total", + Help: "total number of inspection errors due to active cluster ids not set inspection failure", }) gc.errUnstakedPeerInspectionFailedCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "unstaked_peer_inspection_err_count", - Help: "number of inspection errors due to unstaked peer inspection failure", + Name: gc.prefix + "unstaked_peer_inspection_error_total", + Help: "total number of inspection errors due to unstaked peer inspection failure", }) - gc.invalidControlMessageSentCount = promauto.NewCounter(prometheus.CounterOpts{ + gc.invalidControlMessageNotificationSentCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "invalid_control_message_sent_count", - Help: "number of invalid control messages sent due to async inspection failure", + Name: gc.prefix + "invalid_control_message_notification_sent_total", + Help: "number of invalid control message notifications (i.e., misbehavior report) sent due to async inspection of rpcs failure", }) - gc.publishMessageInvalidSenderCount = promauto.NewCounter(prometheus.CounterOpts{ + gc.graftDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "publish_message_invalid_sender_count", - Help: "number of invalid sender detected for publish messages", + Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of duplicate topic ids on graft messages of a single RPC during the async inspection, regardless of the result of the inspection", }) - gc.publishMessageInspectionErrExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + gc.graftDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "publish_message_inspection_err_exceed_threshold_count", - Help: "number of publish messages inspection errors exceeding the threshold", + Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids_exceed_threshold_total", + Help: "number of times that the async inspection of graft messages of an rpc failed due to the number of duplicate topic ids exceeding the threshold", }) - gc.publishMessageInvalidSubscriptionsCount = promauto.NewCounter(prometheus.CounterOpts{ + gc.pruneDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "publish_message_invalid_subscriptions_count", - Help: "number of publish messages with invalid subscriptions", + Buckets: []float64{1, 100, 1000}, + Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids_count", + Help: "number of duplicate topic ids on prune messages of a single RPC during the async inspection, regardless of the result of the inspection", }) - gc.graftDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + gc.pruneDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids", - Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection of graft messages", + Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids_exceed_threshold_total", + Help: "number of times that the async inspection of prune messages failed due to the number of duplicate topic ids exceeding the threshold", }) - gc.graftDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + gc.publishMessageInspectedErrHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids_exceed_threshold_total", - Help: "number of times that the async inspection of graft messages failed due to the number of duplicate topic ids exceeding the threshold", + Name: gc.prefix + "publish_message_inspected_error_count", + Buckets: []float64{10, 100, 1000}, + Help: "number of errors that occurred during the async inspection of publish messages on a single RPC, regardless pof the result", }) - gc.pruneDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + gc.publishMessageInvalidSenderCountHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids", - Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection of prune messages", + Name: gc.prefix + "rpc_inspection_publish_message_invalid_sender_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of invalid senders observed during the async inspection of publish messages on a single RPC, regardless of the result", }) - gc.pruneDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + gc.publishMessageInvalidTopicIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids_exceed_threshold_total", - Help: "number of times that the async inspection of prune messages failed due to the number of duplicate topic ids exceeding the threshold", + Name: gc.prefix + "rpc_inspection_publish_message_invalid_topic_id_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of invalid topic ids observed during the async inspection of publish messages on a single RPC, regardless of the result", }) - gc.publishMessageInspectedErrHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + gc.publishMessageInvalidSubscriptionsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_publish_message_invalid_subscriptions_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of invalid subscriptions observed during the async inspection of publish messages on a single RPC, regardless of the result", + }) + + gc.publishMessageInspectionErrExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, - Name: gc.prefix + "publish_message_inspected_err_histogram", - Help: "histogram of publish message inspection errors", + Name: gc.prefix + "publish_message_inspection_err_exceed_threshold_total", + Help: "number of rpcs fail on inspection due to published message inspection errors exceeding the threshold", }) return gc @@ -365,103 +379,86 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnIncomingRpcReceived(iHaveCoun c.receivedIHaveMsgCount.Add(float64(iHaveCount)) } -// OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on an iWant message at the end of the async inspection iWants -// across one RPC. +// OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on iWant messages at the end of the async inspection iWants +// across one RPC, regardless of the result of the inspection. // -// duplicateCount: the number of duplicate message ids received by the node on an iWant message at the end of the async inspection iWants. -// cacheMissCount: the number of cache miss message ids received by the node on an iWant message at the end of the async inspection iWants. +// duplicateCount: the total number of duplicate message ids received by the node on the iWant messages at the end of the async inspection of the RPC. +// cacheMissCount: the total number of cache miss message ids received by the node on the iWant message at the end of the async inspection of the RPC. func (c *GossipSubRpcValidationInspectorMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { c.iWantDuplicateMessageIdHistogram.Observe(float64(duplicateCount)) c.iWantCacheMissHistogram.Observe(float64(cacheMissCount)) } -// OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of duplicate message ids -// received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. +// OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total number of duplicate message ids +// received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. func (c *GossipSubRpcValidationInspectorMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { c.iWantDuplicateMessageIdExceedThresholdCount.Inc() } -// OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the number of cache miss message ids -// received by the node on an iWant message exceeding the threshold, which results in a misbehaviour report. +// OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total +// number of cache miss message ids received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. func (c *GossipSubRpcValidationInspectorMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { c.iWantCacheMissMessageIdExceedThresholdCount.Inc() } -// OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages, regardless of the result of the inspection. -// It tracks the number of duplicate topic ids and duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. +// OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages of a single RPC, regardless of the result of the inspection. +// It tracks the number of duplicate topic ids and duplicate message ids received by the node on the iHave messages of that single RPC at the end of the async inspection iHaves. // Args: // -// duplicateTopicIds: the number of duplicate topic ids received by the node on an iHave message at the end of the async inspection iHaves. -// duplicateMessageIds: the number of duplicate message ids received by the node on an iHave message at the end of the async inspection iHaves. +// duplicateTopicIds: the total number of duplicate topic ids received by the node on the iHave messages at the end of the async inspection of the RPC. +// duplicateMessageIds: the number of duplicate message ids received by the node on the iHave messages at the end of the async inspection of the RPC. func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { c.iHaveDuplicateTopicIdHistogram.Observe(float64(duplicateTopicIds)) c.iHaveDuplicateMessageIdHistogram.Observe(float64(duplicateMessageIds)) } -// OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate topic ids -// received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. +// OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate topic ids +// received by the node on the iHave messages of that RPC exceeding the threshold, which results in a misbehaviour report. func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { c.iHaveDuplicateTopicIdExceedThresholdCount.Inc() } -// OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times the number times that the async inspection of iHave messages failed due to the number of duplicate message ids +// OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate message ids // received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { c.iHaveDuplicateMessageIdExceedThresholdCount.Inc() } -// OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message failed due to an invalid topic id. +// OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message type on a single RPC failed due to an invalid topic id. // Args: // - messageType: the type of the control message that was truncated. func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { c.ctrlMsgInvalidTopicIdCount.WithLabelValues(messageType.String()).Inc() } -// OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message failed due to active cluster ids not set inspection failure. +// OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message type on a single RPC failed due to active cluster ids not set inspection failure. // This is not causing a misbehaviour report. func (c *GossipSubRpcValidationInspectorMetrics) OnActiveClusterIDsNotSetErr() { c.errActiveClusterIdsNotSetCount.Inc() } -// OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message failed due to unstaked peer inspection failure. +// OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message type on a single RPC failed due to unstaked peer inspection failure. // This is not causing a misbehaviour report. func (c *GossipSubRpcValidationInspectorMetrics) OnUnstakedPeerInspectionFailed() { c.errUnstakedPeerInspectionFailedCount.Inc() } -// OnInvalidControlMessageSent tracks the number of times that the async inspection of a control message failed and an invalid control message was sent. -func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageSent() { - c.invalidControlMessageSentCount.Inc() -} - -// OnInvalidSenderForPublishMessage tracks the number of times that the async inspection of a publish message detected an invalid sender. -// Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. -func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidSenderForPublishMessage() { - c.publishMessageInvalidSenderCount.Inc() +// OnInvalidControlMessageNotificationSent tracks the number of times that the async inspection of a control message failed and resulted in dissemination of an invalid control message was sent (i.e., a +// misbehavior report). +func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageNotificationSent() { + c.invalidControlMessageNotificationSentCount.Inc() } -// OnPublishMessagesInspectionErrorExceedsThreshold tracks the number of times that async inspection of publish messages failed due to the number of errors. -func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { - c.publishMessageInspectionErrExceedThresholdCount.Inc() -} - -// OnPublishMessageInvalidSubscription tracks the number of times that the async inspection of a publish message detected an invalid subscription. -// Note that it does not cause a misbehaviour report; unless the number of times that this happens exceeds the threshold. -func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInvalidSubscription() { - c.publishMessageInvalidSubscriptionsCount.Inc() -} - -// OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a prune message failed due to the number of duplicate topic ids -// received by the node on prune messages of the same rpc excesses threshold, which results in a misbehaviour report. -// Note that it does cause a misbehaviour report. +// OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of prune messages for an RPC failed due to the number of duplicate topic ids +// received by the node on prune messages of the same RPC excesses threshold, which results in a misbehaviour report. func (c *GossipSubRpcValidationInspectorMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { c.pruneDuplicateTopicIdsExceedThresholdCount.Inc() } -// OnPruneMessageInspected is called at the end of the async inspection of prune messages, regardless of the result of the inspection. +// OnPruneMessageInspected is called at the end of the async inspection of prune messages of the RPC, regardless of the result of the inspection. // Args: // -// duplicateTopicIds: the number of duplicate topic ids received by the node on a prune message at the end of the async inspection prunes. +// duplicateTopicIds: the number of duplicate topic ids received by the node on the prune messages of the RPC at the end of the async inspection prunes. func (c *GossipSubRpcValidationInspectorMetrics) OnPruneMessageInspected(duplicateTopicIds int) { c.pruneDuplicateTopicIdsHistogram.Observe(float64(duplicateTopicIds)) } @@ -472,19 +469,30 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnGraftDuplicateTopicIdsExceedT c.graftDuplicateTopicIdsExceedThresholdCount.Inc() } -// OnGraftMessageInspected is called at the end of the async inspection of graft messages, regardless of the result of the inspection. +// OnGraftMessageInspected is called at the end of the async inspection of graft messages of a single RPC, regardless of the result of the inspection. // Args: // -// duplicateTopicIds: the number of duplicate topic ids received by the node on a graft message at the end of the async inspection grafts. +// duplicateTopicIds: the number of duplicate topic ids received by the node on the graft messages at the end of the async inspection of a single RPC. func (c *GossipSubRpcValidationInspectorMetrics) OnGraftMessageInspected(duplicateTopicIds int) { c.graftDuplicateTopicIdsHistogram.Observe(float64(duplicateTopicIds)) } -// OnPublishMessageInspected tracks the number of errors that occurred during the async inspection of publish messages. -// Note that this function is called on each publish message received by the node regardless of the result of the inspection. -// If the number of errors exceeds the threshold, a misbehaviour report is sent, but this function is still called. +// OnPublishMessageInspected is called at the end of the async inspection of publish messages of a single RPC, regardless of the result of the inspection. +// It tracks the total number of errors detected during the async inspection of the rpc together with their individual breakdown. // Args: // - errCount: the number of errors that occurred during the async inspection of publish messages. -func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(errCount int) { - c.publishMessageInspectedErrHistogram.Observe(float64(errCount)) +// - invalidTopicIdsCount: the number of times that an invalid topic id was detected during the async inspection of publish messages. +// - invalidSubscriptionsCount: the number of times that an invalid subscription was detected during the async inspection of publish messages. +// - invalidSendersCount: the number of times that an invalid sender was detected during the async inspection of publish messages. +func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + c.publishMessageInspectedErrHistogram.Observe(float64(totalErrCount)) + c.publishMessageInvalidSenderCountHistogram.Observe(float64(invalidSendersCount)) + c.publishMessageInvalidSubscriptionsHistogram.Observe(float64(invalidSubscriptionsCount)) + c.publishMessageInvalidTopicIdHistogram.Observe(float64(invalidTopicIdsCount)) +} + +// OnPublishMessagesInspectionErrorExceedsThreshold tracks the number of times that async inspection of publish messages failed due to the number of errors exceeding the threshold. +// Note that it causes a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + c.publishMessageInspectionErrExceedThresholdCount.Inc() } diff --git a/module/metrics/noop.go b/module/metrics/noop.go index 99b03f36a8a..14ce9bb3994 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -335,15 +335,14 @@ func (nc *NoopCollector) OnInvalidTopicIdDetectedForControlMessage(messageType p } func (nc *NoopCollector) OnActiveClusterIDsNotSetErr() {} func (nc *NoopCollector) OnUnstakedPeerInspectionFailed() {} -func (nc *NoopCollector) OnInvalidControlMessageSent() {} -func (nc *NoopCollector) OnInvalidSenderForPublishMessage() {} +func (nc *NoopCollector) OnInvalidControlMessageNotificationSent() {} func (nc *NoopCollector) OnPublishMessagesInspectionErrorExceedsThreshold() {} -func (nc *NoopCollector) OnPublishMessageInvalidSubscription() {} func (nc *NoopCollector) OnPruneDuplicateTopicIdsExceedThreshold() {} func (nc *NoopCollector) OnPruneMessageInspected(duplicateTopicIds int) {} func (nc *NoopCollector) OnGraftDuplicateTopicIdsExceedThreshold() {} func (nc *NoopCollector) OnGraftMessageInspected(duplicateTopicIds int) {} -func (nc *NoopCollector) OnPublishMessageInspected(errCount int) {} +func (nc *NoopCollector) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { +} func (nc *NoopCollector) OnMisbehaviorReported(string, string) {} func (nc *NoopCollector) OnViolationReportSkipped() {} diff --git a/module/mock/gossip_sub_metrics.go b/module/mock/gossip_sub_metrics.go index 152b8337fa6..bf7f08ac914 100644 --- a/module/mock/gossip_sub_metrics.go +++ b/module/mock/gossip_sub_metrics.go @@ -122,7 +122,7 @@ func (_m *GossipSubMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int } // OnInvalidControlMessageSent provides a mock function with given fields: -func (_m *GossipSubMetrics) OnInvalidControlMessageSent() { +func (_m *GossipSubMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } @@ -227,8 +227,8 @@ func (_m *GossipSubMetrics) OnPruneMessageInspected(duplicateTopicIds int) { } // OnPublishMessageInspected provides a mock function with given fields: errCount -func (_m *GossipSubMetrics) OnPublishMessageInspected(errCount int) { - _m.Called(errCount) +func (_m *GossipSubMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount) } // OnPublishMessageInvalidSubscription provides a mock function with given fields: diff --git a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go index 533694ea029..6268472eb27 100644 --- a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go +++ b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go @@ -101,7 +101,7 @@ func (_m *GossipSubRpcValidationInspectorMetrics) OnIncomingRpcReceived(iHaveCou } // OnInvalidControlMessageSent provides a mock function with given fields: -func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageSent() { +func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } @@ -126,8 +126,8 @@ func (_m *GossipSubRpcValidationInspectorMetrics) OnPruneMessageInspected(duplic } // OnPublishMessageInspected provides a mock function with given fields: errCount -func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(errCount int) { - _m.Called(errCount) +func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount) } // OnPublishMessageInvalidSubscription provides a mock function with given fields: diff --git a/module/mock/lib_p2_p_metrics.go b/module/mock/lib_p2_p_metrics.go index 27c3e102cd6..3420e4985c3 100644 --- a/module/mock/lib_p2_p_metrics.go +++ b/module/mock/lib_p2_p_metrics.go @@ -243,7 +243,7 @@ func (_m *LibP2PMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, g } // OnInvalidControlMessageSent provides a mock function with given fields: -func (_m *LibP2PMetrics) OnInvalidControlMessageSent() { +func (_m *LibP2PMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } @@ -358,8 +358,8 @@ func (_m *LibP2PMetrics) OnPruneMessageInspected(duplicateTopicIds int) { } // OnPublishMessageInspected provides a mock function with given fields: errCount -func (_m *LibP2PMetrics) OnPublishMessageInspected(errCount int) { - _m.Called(errCount) +func (_m *LibP2PMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount) } // OnPublishMessageInvalidSubscription provides a mock function with given fields: diff --git a/module/mock/network_metrics.go b/module/mock/network_metrics.go index 14799f9f64b..fa5c8af68b2 100644 --- a/module/mock/network_metrics.go +++ b/module/mock/network_metrics.go @@ -273,7 +273,7 @@ func (_m *NetworkMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, } // OnInvalidControlMessageSent provides a mock function with given fields: -func (_m *NetworkMetrics) OnInvalidControlMessageSent() { +func (_m *NetworkMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } @@ -393,8 +393,8 @@ func (_m *NetworkMetrics) OnPruneMessageInspected(duplicateTopicIds int) { } // OnPublishMessageInspected provides a mock function with given fields: errCount -func (_m *NetworkMetrics) OnPublishMessageInspected(errCount int) { - _m.Called(errCount) +func (_m *NetworkMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount) } // OnPublishMessageInvalidSubscription provides a mock function with given fields: diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 1ae7657255f..418d8c6fe2e 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -567,19 +567,18 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, return false } var errs *multierror.Error + invalidTopicIdsCount := 0 + invalidSubscriptionsCount := 0 + invalidSendersCount := 0 defer func() { // regardless of inspection result, update metrics - if errs != nil { - c.metrics.OnPublishMessageInspected(errs.Len()) - } else { - c.metrics.OnPublishMessageInspected(0) - } + c.metrics.OnPublishMessageInspected(errs.Len(), invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) }() for _, message := range messages[:sampleSize] { if c.networkingType == network.PrivateNetwork { err := c.checkPubsubMessageSender(message) if err != nil { - c.metrics.OnInvalidSenderForPublishMessage() + invalidSendersCount++ errs = multierror.Append(errs, err) continue } @@ -591,13 +590,13 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, err, _ := c.validateTopic(from, topic, activeClusterIDS) if err != nil { // we can skip checking for subscription of topic that failed validation and continue - c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.RpcPublishMessage) + invalidTopicIdsCount++ errs = multierror.Append(errs, err) continue } if !hasSubscription(topic.String()) { - c.metrics.OnPublishMessageInvalidSubscription() + invalidSubscriptionsCount++ errs = multierror.Append(errs, fmt.Errorf("subscription for topic %s not found", topic)) } } @@ -940,7 +939,7 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In c.metrics.OnUnstakedPeerInspectionFailed() lg.Warn().Msg("control message received from unstaked peer") default: - c.metrics.OnInvalidControlMessageSent() + c.metrics.OnInvalidControlMessageNotificationSent() distErr := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err, count, topicType)) if distErr != nil { lg.Error(). From bc4c41a5ea3e353e0aaca4e5c0dedbe1de406a2b Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 15:13:52 -0800 Subject: [PATCH 48/57] updates mocks --- module/mock/gossip_sub_metrics.go | 16 +++------------- ...ossip_sub_rpc_validation_inspector_metrics.go | 16 +++------------- module/mock/lib_p2_p_metrics.go | 16 +++------------- module/mock/network_metrics.go | 16 +++------------- 4 files changed, 12 insertions(+), 52 deletions(-) diff --git a/module/mock/gossip_sub_metrics.go b/module/mock/gossip_sub_metrics.go index bf7f08ac914..f7e057ea5ba 100644 --- a/module/mock/gossip_sub_metrics.go +++ b/module/mock/gossip_sub_metrics.go @@ -121,7 +121,7 @@ func (_m *GossipSubMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } -// OnInvalidControlMessageSent provides a mock function with given fields: +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: func (_m *GossipSubMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } @@ -131,11 +131,6 @@ func (_m *GossipSubMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _m.Called(_a0, _a1) } -// OnInvalidSenderForPublishMessage provides a mock function with given fields: -func (_m *GossipSubMetrics) OnInvalidSenderForPublishMessage() { - _m.Called() -} - // OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType func (_m *GossipSubMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { _m.Called(messageType) @@ -226,14 +221,9 @@ func (_m *GossipSubMetrics) OnPruneMessageInspected(duplicateTopicIds int) { _m.Called(duplicateTopicIds) } -// OnPublishMessageInspected provides a mock function with given fields: errCount +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount func (_m *GossipSubMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { - _m.Called(totalErrCount) -} - -// OnPublishMessageInvalidSubscription provides a mock function with given fields: -func (_m *GossipSubMetrics) OnPublishMessageInvalidSubscription() { - _m.Called() + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) } // OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: diff --git a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go index 6268472eb27..84eef02f7ea 100644 --- a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go +++ b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go @@ -100,16 +100,11 @@ func (_m *GossipSubRpcValidationInspectorMetrics) OnIncomingRpcReceived(iHaveCou _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } -// OnInvalidControlMessageSent provides a mock function with given fields: +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } -// OnInvalidSenderForPublishMessage provides a mock function with given fields: -func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidSenderForPublishMessage() { - _m.Called() -} - // OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { _m.Called(messageType) @@ -125,14 +120,9 @@ func (_m *GossipSubRpcValidationInspectorMetrics) OnPruneMessageInspected(duplic _m.Called(duplicateTopicIds) } -// OnPublishMessageInspected provides a mock function with given fields: errCount +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { - _m.Called(totalErrCount) -} - -// OnPublishMessageInvalidSubscription provides a mock function with given fields: -func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInvalidSubscription() { - _m.Called() + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) } // OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: diff --git a/module/mock/lib_p2_p_metrics.go b/module/mock/lib_p2_p_metrics.go index 3420e4985c3..f91d247d6bf 100644 --- a/module/mock/lib_p2_p_metrics.go +++ b/module/mock/lib_p2_p_metrics.go @@ -242,7 +242,7 @@ func (_m *LibP2PMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, g _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } -// OnInvalidControlMessageSent provides a mock function with given fields: +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: func (_m *LibP2PMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } @@ -252,11 +252,6 @@ func (_m *LibP2PMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _a _m.Called(_a0, _a1) } -// OnInvalidSenderForPublishMessage provides a mock function with given fields: -func (_m *LibP2PMetrics) OnInvalidSenderForPublishMessage() { - _m.Called() -} - // OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType func (_m *LibP2PMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { _m.Called(messageType) @@ -357,14 +352,9 @@ func (_m *LibP2PMetrics) OnPruneMessageInspected(duplicateTopicIds int) { _m.Called(duplicateTopicIds) } -// OnPublishMessageInspected provides a mock function with given fields: errCount +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount func (_m *LibP2PMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { - _m.Called(totalErrCount) -} - -// OnPublishMessageInvalidSubscription provides a mock function with given fields: -func (_m *LibP2PMetrics) OnPublishMessageInvalidSubscription() { - _m.Called() + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) } // OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: diff --git a/module/mock/network_metrics.go b/module/mock/network_metrics.go index fa5c8af68b2..e86d63fb03a 100644 --- a/module/mock/network_metrics.go +++ b/module/mock/network_metrics.go @@ -272,7 +272,7 @@ func (_m *NetworkMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } -// OnInvalidControlMessageSent provides a mock function with given fields: +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: func (_m *NetworkMetrics) OnInvalidControlMessageNotificationSent() { _m.Called() } @@ -282,11 +282,6 @@ func (_m *NetworkMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _ _m.Called(_a0, _a1) } -// OnInvalidSenderForPublishMessage provides a mock function with given fields: -func (_m *NetworkMetrics) OnInvalidSenderForPublishMessage() { - _m.Called() -} - // OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType func (_m *NetworkMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { _m.Called(messageType) @@ -392,14 +387,9 @@ func (_m *NetworkMetrics) OnPruneMessageInspected(duplicateTopicIds int) { _m.Called(duplicateTopicIds) } -// OnPublishMessageInspected provides a mock function with given fields: errCount +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount func (_m *NetworkMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { - _m.Called(totalErrCount) -} - -// OnPublishMessageInvalidSubscription provides a mock function with given fields: -func (_m *NetworkMetrics) OnPublishMessageInvalidSubscription() { - _m.Called() + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) } // OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: From f2a14b85144b0a9eb12e4459359b72fd2cb0c30f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 15:17:48 -0800 Subject: [PATCH 49/57] updates mocks --- .../validation/control_message_validation_inspector.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 418d8c6fe2e..d05a1de90e8 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -939,14 +939,15 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In c.metrics.OnUnstakedPeerInspectionFailed() lg.Warn().Msg("control message received from unstaked peer") default: - c.metrics.OnInvalidControlMessageNotificationSent() distErr := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err, count, topicType)) if distErr != nil { lg.Error(). Err(distErr). Msg("failed to distribute invalid control message notification") + return } lg.Error().Msg("rpc control message async inspection failed") + c.metrics.OnInvalidControlMessageNotificationSent() } } From bd0dbdb1d2608f62d9074d932f24bf4b8e1b371b Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 15:25:07 -0800 Subject: [PATCH 50/57] fixes insecure package tests --- .../gossipsub/rpc_inspector/validation_inspector_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index d33c6dede6f..da672560a11 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -771,11 +771,9 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { flowConfig, err := config.DefaultConfig() require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - // force all cache miss checks - inspectorConfig.IWant.CacheMissCheckSize = 1 inspectorConfig.InspectionQueue.NumberOfWorkers = 1 - inspectorConfig.IWant.CacheMissThreshold = .5 // set cache miss threshold to 50% - messageCount := 1 + inspectorConfig.IWant.CacheMissThreshold = 10 + messageCount := 10 controlMessageCount := int64(1) cacheMissThresholdNotifCount := atomic.NewUint64(0) done := make(chan struct{}) From f2fddf317c9736cd645d72bb3a4d64f117a80cd3 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 15:48:00 -0800 Subject: [PATCH 51/57] fixes insecure package tests --- .../validation_inspector_test.go | 27 ++++++++++++------- .../control_message_validation_inspector.go | 8 +++++- network/p2p/test/fixtures.go | 12 +++++++++ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index da672560a11..67e68c2763b 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -3,6 +3,7 @@ package rpc_inspector import ( "context" "fmt" + "math" "os" "testing" "time" @@ -181,7 +182,9 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { inspectorConfig.InspectionQueue.NumberOfWorkers = 1 - messageCount := 10 + // sets the message count to the max of the duplicate topic id threshold for graft and prune control messages to ensure + // a successful attack + messageCount := int(math.Max(float64(inspectorConfig.IHave.DuplicateTopicIdThreshold), float64(inspectorConfig.GraftPrune.DuplicateTopicIdThreshold))) + 2 controlMessageCount := int64(1) count := atomic.NewInt64(0) @@ -301,7 +304,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") - require.True(t, validation.IsDuplicateTopicErr(notification.Error)) + require.True(t, validation.IsDuplicateMessageIDErr(notification.Error)) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, notification.MsgType == p2pmsg.CtrlMsgIHave, @@ -361,21 +364,25 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { validationInspector.Start(signalerCtx) nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + // to suppress peers provider not set + p2ptest.RegisterPeerProviders(t, nodes) spammer.Start(t) defer stopComponents(t, cancel, nodes, validationInspector) // generate 2 control messages with iHaves for different topics - ihaveCtlMsgs1 := spammer.GenerateCtlMessages(1, p2ptest.WithIHave(1, 1, pushBlocks.String())) - ihaveCtlMsgs2 := spammer.GenerateCtlMessages(1, p2ptest.WithIHave(1, 1, reqChunks.String())) - - // duplicate message ids for a single topic is invalid and will cause an error - ihaveCtlMsgs1[0].Ihave[0].MessageIDs = append(ihaveCtlMsgs1[0].Ihave[0].MessageIDs, ihaveCtlMsgs1[0].Ihave[0].MessageIDs[0]) - // duplicate message ids across different topics is valid - ihaveCtlMsgs2[0].Ihave[0].MessageIDs[0] = ihaveCtlMsgs1[0].Ihave[0].MessageIDs[0] + messageIdCount := inspectorConfig.IHave.DuplicateMessageIdThreshold + 2 + messageIds := unittest.IdentifierListFixture(1) + for i := 0; i < messageIdCount; i++ { + messageIds = append(messageIds, messageIds[0]) + } + // prepares 2 control messages with iHave messages for different topics with duplicate message IDs + ihaveCtlMsgs1 := spammer.GenerateCtlMessages( + 1, + p2ptest.WithIHaveMessageIDs(messageIds.Strings(), pushBlocks.String()), + p2ptest.WithIHaveMessageIDs(messageIds.Strings(), reqChunks.String())) // start spamming the victim peer spammer.SpamControlMessage(t, victimNode, ihaveCtlMsgs1) - spammer.SpamControlMessage(t, victimNode, ihaveCtlMsgs2) unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") // ensure we receive the expected number of invalid control message notifications diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index d05a1de90e8..491d550da1e 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -418,10 +418,12 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave defer func() { // regardless of inspection result, update metrics c.metrics.OnIHaveMessagesInspected(totalDuplicateTopicIds, totalDuplicateMessageIds) + fmt.Println("IHaveMessagesInspected", totalDuplicateTopicIds, totalDuplicateMessageIds, totalMessageIds) }() for _, ihave := range ihaves { messageIds := ihave.GetMessageIDs() topic := ihave.GetTopicID() + totalMessageIds += len(messageIds) // first check if the topic is valid, fail fast if it is not err, ctrlMsgType := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) @@ -572,7 +574,11 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, invalidSendersCount := 0 defer func() { // regardless of inspection result, update metrics - c.metrics.OnPublishMessageInspected(errs.Len(), invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) + errCnt := 0 + if errs != nil { + errCnt = errs.Len() + } + c.metrics.OnPublishMessageInspected(errCnt, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) }() for _, message := range messages[:sampleSize] { if c.networkingType == network.PrivateNetwork { diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 8b8f33067d4..09eef2dbba3 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -917,6 +917,18 @@ func WithIHave(msgCount, msgIDCount int, topicId string) GossipSubCtrlOption { } } +// WithIHaveMessageIDs adds iHave control messages with the given message IDs to the control message. +func WithIHaveMessageIDs(msgIDs []string, topicId string) GossipSubCtrlOption { + return func(msg *pb.ControlMessage) { + msg.Ihave = []*pb.ControlIHave{ + { + TopicID: &topicId, + MessageIDs: msgIDs, + }, + } + } +} + // WithIWant adds iWant control messages of the given size and number to the control message. // The message IDs are generated randomly. // Args: From a6ae09ea6fc96278acde1158e0ed6e326411b458 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 15:53:05 -0800 Subject: [PATCH 52/57] relaxes inspection conditions --- config/default-config.yml | 8 ++++---- network/p2p/inspector/validation/errors_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index 3668c2ca215..9dab8af4c0a 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -159,7 +159,7 @@ network-config: # Maximum number of total duplicate topic ids in a single GRAFT or PRUNE message, ideally this should be 0 but we allow for some tolerance # to avoid penalizing peers that are not malicious but are misbehaving due to bugs or other issues. # A topic id is considered duplicate if it appears more than once in a single GRAFT or PRUNE message. - duplicate-topic-id-threshold: 10 + duplicate-topic-id-threshold: 50 ihave: # The maximum allowed number of iHave messages in a single RPC message. # Each iHave message represents the list of message ids. When the total number of iHave messages @@ -175,12 +175,12 @@ network-config: # The tolerance threshold for having duplicate topics in an iHave message under inspection. # When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. # Note that a topic ID is counted as a duplicate only if it is repeated more than once. - duplicate-topic-id-threshold: 10 + duplicate-topic-id-threshold: 50 # Threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. # When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. # Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once # within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. - duplicate-message-id-threshold: 10 + duplicate-message-id-threshold: 100 iwant: # The maximum allowed number of iWant messages in a single RPC message. # Each iWant message represents the list of message ids. When the total number of iWant messages @@ -199,7 +199,7 @@ network-config: # The max allowed number of duplicate message ids in a single iwant message. # Note that ideally there should be no duplicate message ids in a single iwant message but # we allow for some tolerance to avoid penalizing peers that are not malicious - duplicate-message-id-threshold: 10 + duplicate-message-id-threshold: 100 cluster-prefixed-messages: # Cluster prefixed control message validation configs # The size of the cache used to track the amount of cluster prefixed topics received by peers diff --git a/network/p2p/inspector/validation/errors_test.go b/network/p2p/inspector/validation/errors_test.go index 9e67f120140..29072fbd5f7 100644 --- a/network/p2p/inspector/validation/errors_test.go +++ b/network/p2p/inspector/validation/errors_test.go @@ -62,7 +62,7 @@ func TestIWantCacheMissThresholdErrRoundTrip(t *testing.T) { err := NewIWantCacheMissThresholdErr(5, 10, 5) // tests the error message formatting. - expectedErrMsg := "5/10 iWant cache misses exceeds the allowed threshold: 0.400000" + expectedErrMsg := "5/10 iWant cache misses exceeds the allowed threshold: 5" assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") // tests the IsIWantCacheMissThresholdErr function. @@ -78,7 +78,7 @@ func TestIWantDuplicateMsgIDThresholdErrRoundTrip(t *testing.T) { err := NewIWantDuplicateMsgIDThresholdErr(5, 10, 5) // tests the error message formatting. - expectedErrMsg := "5/10 iWant duplicate message ids exceeds the allowed threshold: 0.400000" + expectedErrMsg := "5/10 iWant duplicate message ids exceeds the allowed threshold: 5" assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") // tests the IsIWantDuplicateMsgIDThresholdErr function. From 7bdedb2033a0691eacbcd77772492f662a196492 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 17 Jan 2024 15:54:12 -0800 Subject: [PATCH 53/57] removes debug line --- .../inspector/validation/control_message_validation_inspector.go | 1 - 1 file changed, 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 491d550da1e..fdbaadc5e0d 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -418,7 +418,6 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave defer func() { // regardless of inspection result, update metrics c.metrics.OnIHaveMessagesInspected(totalDuplicateTopicIds, totalDuplicateMessageIds) - fmt.Println("IHaveMessagesInspected", totalDuplicateTopicIds, totalDuplicateMessageIds, totalMessageIds) }() for _, ihave := range ihaves { messageIds := ihave.GetMessageIDs() From 90bfcec008ebf7f112391a5813ee48606b13211d Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 18 Jan 2024 10:58:50 -0800 Subject: [PATCH 54/57] fixes TestGossipSubSpamMitigationIntegration --- .../gossipsub/rpc_inspector/validation_inspector_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 67e68c2763b..0f68de8a4d7 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -1092,12 +1092,14 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { graftCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, unknownTopic.String())) graftCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, malformedTopic.String())) graftCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, invalidSporkIDTopic.String())) - graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(3, duplicateTopic.String())) + graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), // sets duplicate to +2 above the threshold to ensure that the victim node will penalize the spammer node + p2ptest.WithGraft(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2, duplicateTopic.String())) pruneCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithPrune(spamRpcCount, unknownTopic.String())) pruneCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithPrune(spamRpcCount, malformedTopic.String())) pruneCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, invalidSporkIDTopic.String())) - pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithPrune(3, duplicateTopic.String())) + pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), // sets duplicate to +2 above the threshold to ensure that the victim node will penalize the spammer node + p2ptest.WithPrune(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2, duplicateTopic.String())) // start spamming the victim peer spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithUnknownTopic) From 0b07c2b0e7788bb72fc219a73f883dd61e8de048 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 18 Jan 2024 11:12:21 -0800 Subject: [PATCH 55/57] fixes broken promises tests --- .../test/gossipsub/scoring/ihave_spam_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go index fb3f4e9ea27..da6581d6230 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go @@ -67,6 +67,12 @@ func TestGossipSubIHaveBrokenPromises_Below_Threshold(t *testing.T) { conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.DecayInterval = 1 * time.Second // score tracer interval is set to 500 milliseconds to speed up the test, it should be shorter than the heartbeat interval (1 second) of gossipsub to catch the score updates in time. conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 500 * time.Millisecond + + // relaxing the scoring parameters to fit the test scenario. + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyDecay = 0.99 + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyThreshold = 10 + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyWeight = -1 + victimNode, victimIdentity := p2ptest.NodeFixture( t, sporkId, @@ -209,6 +215,11 @@ func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { // score tracer interval is set to 500 milliseconds to speed up the test, it should be shorter than the heartbeat interval (1 second) of gossipsub to catch the score updates in time. conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 500 * time.Millisecond + // relaxing the scoring parameters to fit the test scenario. + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyDecay = 0.99 + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyThreshold = 10 + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyWeight = -1 + ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. From 20effabf0940dcd02e03c3c582a78616fc9565dc Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 18 Jan 2024 17:57:45 -0800 Subject: [PATCH 56/57] empty commit --- .../validation/control_message_validation_inspector_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 71682d6778e..3afe6897e75 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -30,6 +30,7 @@ import ( ) func TestNewControlMsgValidationInspector(t *testing.T) { + t.Run("should create validation inspector without error", func(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() From db642071f3bac3c6e8ea3888bdc8107d372be864 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 22 Jan 2024 09:08:29 -0800 Subject: [PATCH 57/57] lint fix --- .../validation/control_message_validation_inspector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 3afe6897e75..672f89c057e 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -30,7 +30,7 @@ import ( ) func TestNewControlMsgValidationInspector(t *testing.T) { - + t.Run("should create validation inspector without error", func(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig()