keep order of check MaxTx

CHANGELOG.md

@@ -61,7 +61,6 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (baseapp) [#20107](https://github.com/cosmos/cosmos-sdk/pull/20107) Avoid header height overwrite block height.
 * (cli) [#20020](https://github.com/cosmos/cosmos-sdk/pull/20020) Make bootstrap-state command support both new and legacy genesis format.
 * [#507](https://github.com/crypto-org-chain/cosmos-sdk/pull/507) mempool respect gas wanted returned by ante handler
-* [#536](https://github.com/crypto-org-chain/cosmos-sdk/pull/536) Reduce scope of mutex in `InsertWithGasWanted`.
 
 ### Bug Fixes
 

types/mempool/priority_nonce.go

@@ -202,6 +202,16 @@ func (mp *PriorityNonceMempool[C]) NextSenderTx(sender string) sdk.Tx {
 // Inserting a duplicate tx with a different priority overwrites the existing tx,
 // changing the total order of the mempool.
 func (mp *PriorityNonceMempool[C]) InsertWithGasWanted(ctx context.Context, tx sdk.Tx, gasWanted uint64) error {
+	mp.mtx.Lock()
+	defer mp.mtx.Unlock()
+	if mp.cfg.MaxTx > 0 && mp.priorityIndex.Len() >= mp.cfg.MaxTx {
+		return ErrMempoolTxMaxCapacity
+	} else if mp.cfg.MaxTx < 0 {
+		return nil
+	}
+
+	memTx := NewMempoolTx(tx, gasWanted)
+
 	sigs, err := mp.cfg.SignerExtractor.GetSigners(tx)
 	if err != nil {
 		return err
@@ -216,16 +226,6 @@ func (mp *PriorityNonceMempool[C]) InsertWithGasWanted(ctx context.Context, tx s
 	nonce := sig.Sequence
 	key := txMeta[C]{nonce: nonce, priority: priority, sender: sender}
 
-	mp.mtx.Lock()
-	defer mp.mtx.Unlock()
-	if mp.cfg.MaxTx > 0 && mp.priorityIndex.Len() >= mp.cfg.MaxTx {
-		return ErrMempoolTxMaxCapacity
-	} else if mp.cfg.MaxTx < 0 {
-		return nil
-	}
-
-	memTx := NewMempoolTx(tx, gasWanted)
-
 	senderIndex, ok := mp.senderIndices[sender]
 	if !ok {
 		senderIndex = skiplist.New(skiplist.LessThanFunc(func(a, b any) int {