Check txpool_status after every Load Test (#236)

* Added `txpool status` after every Load Test

* Minor fixes

* load test fix

* load test fixes

* Change `txpool_timeout`; Remove unused files

* Added MIXED Load Testing

* Fix Load Test command

* Increased ERC721 LT timeout

---------

Co-authored-by: Goran Rojovic <goran.rojovic@ethernal.tech>

.github/workflows/load-test.yml

@@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy
       type:
         description: The type of load test to run
         type: choice
-        options: [EOA, ERC20, ERC721]
+        options: [EOA, ERC20, ERC721, MIXED]
       dynamic:
         description: Indicates whether the load test should generate dynamic transactions
         type: boolean
@@ -117,6 +117,12 @@ on: # yamllint disable-line rule:truthy
       max_gas_utilization:
         description: Maximum Gas Utilization
         value: ${{ jobs.load_test.outputs.max_gas_utilization }}
+      txpool_pending:
+        description: Pending Transactions Count
+        value: ${{ jobs.txpool_status.outputs.txpool_pending }}
+      txpool_queued:
+        description: Queued Transactions Count
+        value: ${{ jobs.txpool_status.outputs.txpool_queued }}
     secrets:
       AWS_ROLE_ARN:
         required: true
@@ -239,9 +245,25 @@ jobs:
         if: success()
         id: load_test_results_success
         run: echo "test_output=true" >> $GITHUB_OUTPUT
+  txpool_status:
+    name: Check txpool status after Load Test
+    runs-on: ubuntu-latest
+    needs: [check_network, load_test]
+    outputs:
+      txpool_pending: ${{ steps.txpool_status_results.outputs.txpool_pending }}
+      txpool_queued: ${{ steps.txpool_status_results.outputs.txpool_queued }}
+    steps:
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - name: Get txpool status
+        id: txpool_status_results
+        run: |
+          txpool_status=$(cast rpc --rpc-url "http://${{ needs.check_network.outputs.rpc_url }}" txpool_status | jq)
+          echo "txpool_pending=$(echo $txpool_status | jq -r '.pending')" >> $GITHUB_OUTPUT
+          echo "txpool_queued=$(echo $txpool_status | jq -r '.queued')" >> $GITHUB_OUTPUT
   notification:
     name: Load Test Notification
-    needs: load_test
+    needs: [load_test, txpool_status]
     uses: ./.github/workflows/notification-load-test.yml
     if: (always() && inputs.notification && needs.load_test.outputs.test_output_success == 'true')
     with:
@@ -263,12 +285,14 @@ jobs:
       avg_gas_per_tx: ${{ needs.load_test.outputs.avg_gas_per_tx }}
       avg_gas_utilization: ${{ needs.load_test.outputs.avg_gas_utilization }}
       max_gas_utilization: ${{ needs.load_test.outputs.max_gas_utilization }}
+      txpool_pending: ${{ needs.txpool_status.outputs.txpool_pending }}
+      txpool_queued: ${{ needs.txpool_status.outputs.txpool_queued }}
     secrets:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
   destroy_load_test_runner:
     name: Destroy Load Test Runner
     environment: ${{ inputs.environment }}
-    needs: [load_test_runner, load_test]
+    needs: [load_test_runner, load_test, txpool_status]
     if: always()
     runs-on: ubuntu-latest
     steps:

.github/workflows/nightly.yml

@@ -46,7 +46,7 @@ jobs:
       dynamic: true
       vus: "100"
       txs_per_user: "10000"
-      txpool_timeout: "60m0s"
+      txpool_timeout: "15m0s"
       wait_txpool: false
       receipts_timeout: "2m0s"
       batch_size: "10000"
@@ -70,7 +70,7 @@ jobs:
       dynamic: true
       vus: "100"
       txs_per_user: "10000"
-      txpool_timeout: "60m0s"
+      txpool_timeout: "20m0s"
       wait_txpool: false
       receipts_timeout: "2m0s"
       batch_size: "10000"
@@ -94,6 +94,30 @@ jobs:
       dynamic: true
       vus: "100"
       txs_per_user: "10000"
+      txpool_timeout: "55m0s"
+      wait_txpool: false
+      receipts_timeout: "2m0s"
+      batch_size: "10000"
+      notification: false
+    secrets:
+      AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
+      AWS_S3_BLADE_BUCKET: ${{ secrets.AWS_S3_BLADE_BUCKET }}
+      AWS_LOADTESTRUNNER_AMI_ID: ${{ secrets.AWS_LOADTESTRUNNER_AMI_ID }}
+      AWS_LOADTESTRUNNER_SUBNET_ID: ${{ secrets.AWS_LOADTESTRUNNER_SUBNET_ID }}
+      AWS_LOADTESTRUNNER_SG_ID: ${{ secrets.AWS_LOADTESTRUNNER_SG_ID }}
+      AWS_LOADTESTRUNNER_MNEMONIC: ${{ secrets.AWS_LOADTESTRUNNER_MNEMONIC }}
+      PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  load_test_mixed:
+    name: Load Test MIXED
+    uses: ./.github/workflows/load-test.yml
+    needs: [deploy_network, load_test_eoa, load_test_erc20, load_test_erc721]
+    with:
+      environment: nightly
+      type: MIXED
+      dynamic: true
+      vus: "100"
+      txs_per_user: "10000"
       txpool_timeout: "60m0s"
       wait_txpool: false
       receipts_timeout: "2m0s"
@@ -111,7 +135,7 @@ jobs:
   destroy_network:
     name: Destroy Network
     uses: ./.github/workflows/destroy-network.yml
-    needs: [deploy_network, load_test_eoa, load_test_erc20, load_test_erc721]
+    needs: [deploy_network, load_test_eoa, load_test_erc20, load_test_erc721, load_test_mixed]
     if: always()
     with:
       environment: nightly
@@ -124,7 +148,7 @@ jobs:
   notification_nightly:
     name: Nightly Notification
     uses: ./.github/workflows/notification-nightly.yml
-    needs: [ci, deploy_network, load_test_eoa, load_test_erc20, load_test_erc721, destroy_network]
+    needs: [ci, deploy_network, load_test_eoa, load_test_erc20, load_test_erc721, load_test_mixed, destroy_network]
     if: success() || failure()
     with:
       environment: nightly
@@ -147,8 +171,17 @@ jobs:
       deploy_network_terraform_output: ${{ needs.deploy_network.outputs.terraform_output }}
       deploy_network_ansible_output: ${{ needs.deploy_network.outputs.ansible_output }}
       load_test_eoa_output: ${{ needs.load_test_eoa.outputs.load_test_output }}
+      load_test_eoa_txpool_pending_output: ${{ needs.load_test_eoa.outputs.txpool_pending }}
+      load_test_eoa_txpool_queued_output: ${{ needs.load_test_eoa.outputs.txpool_queued }}
       load_test_erc20_output: ${{ needs.load_test_erc20.outputs.load_test_output }}
+      load_test_erc20_txpool_pending_output: ${{ needs.load_test_erc20.outputs.txpool_pending }}
+      load_test_erc20_txpool_queued_output: ${{ needs.load_test_erc20.outputs.txpool_queued }}
       load_test_erc721_output: ${{ needs.load_test_erc721.outputs.load_test_output }}
+      load_test_erc721_txpool_pending_output: ${{ needs.load_test_erc721.outputs.txpool_pending }}
+      load_test_erc721_txpool_queued_output: ${{ needs.load_test_erc721.outputs.txpool_queued }}
+      load_test_mixed_output: ${{ needs.load_test_mixed.outputs.load_test_output }}
+      load_test_mixed_txpool_pending_output: ${{ needs.load_test_mixed.outputs.txpool_pending }}
+      load_test_mixed_txpool_queued_output: ${{ needs.load_test_mixed.outputs.txpool_queued }}
       destroy_network_logs_output: ${{ needs.destroy_network.outputs.logs_output }}
       destroy_network_terraform_output: ${{ needs.destroy_network.outputs.terraform_output }}
     secrets:
@@ -165,7 +198,7 @@ jobs:
       dynamic: true
       vus: "100"
       txs_per_user: "10000"
-      txpool_timeout: "60m0s"
+      txpool_timeout: "15m0s"
       wait_txpool: false
       receipts_timeout: "2m0s"
       batch_size: "10000"
@@ -178,6 +211,8 @@ jobs:
       avg_gas_per_tx: ${{ needs.load_test_eoa.outputs.avg_gas_per_tx }}
       avg_gas_utilization: ${{ needs.load_test_eoa.outputs.avg_gas_utilization }}
       max_gas_utilization: ${{ needs.load_test_eoa.outputs.max_gas_utilization }}
+      txpool_pending: ${{ needs.load_test_eoa.outputs.txpool_pending }}
+      txpool_queued: ${{ needs.load_test_eoa.outputs.txpool_queued }}
     secrets:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
   notification_load_test_erc20:
@@ -191,7 +226,7 @@ jobs:
       dynamic: true
       vus: "100"
       txs_per_user: "10000"
-      txpool_timeout: "60m0s"
+      txpool_timeout: "20m0s"
       wait_txpool: false
       receipts_timeout: "2m0s"
       batch_size: "10000"
@@ -204,6 +239,8 @@ jobs:
       avg_gas_per_tx: ${{ needs.load_test_erc20.outputs.avg_gas_per_tx }}
       avg_gas_utilization: ${{ needs.load_test_erc20.outputs.avg_gas_utilization }}
       max_gas_utilization: ${{ needs.load_test_erc20.outputs.max_gas_utilization }}
+      txpool_pending: ${{ needs.load_test_erc20.outputs.txpool_pending }}
+      txpool_queued: ${{ needs.load_test_erc20.outputs.txpool_queued }}
     secrets:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
   notification_load_test_erc721:
@@ -217,7 +254,7 @@ jobs:
       dynamic: true
       vus: "100"
       txs_per_user: "10000"
-      txpool_timeout: "60m0s"
+      txpool_timeout: "55m0s"
       wait_txpool: false
       receipts_timeout: "2m0s"
       batch_size: "10000"
@@ -230,5 +267,35 @@ jobs:
       avg_gas_per_tx: ${{ needs.load_test_erc721.outputs.avg_gas_per_tx }}
       avg_gas_utilization: ${{ needs.load_test_erc721.outputs.avg_gas_utilization }}
       max_gas_utilization: ${{ needs.load_test_erc721.outputs.max_gas_utilization }}
+      txpool_pending: ${{ needs.load_test_erc721.outputs.txpool_pending }}
+      txpool_queued: ${{ needs.load_test_erc721.outputs.txpool_queued }}
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  notification_load_test_mixed:
+    name: Load Test MIXED Notification
+    uses: ./.github/workflows/notification-load-test.yml
+    needs: [load_test_mixed, notification_nightly, notification_load_test_eoa, notification_load_test_erc20, notification_load_test_erc721]
+    if: (always() && needs.load_test_mixed.outputs.load_test_output == 'true')
+    with:
+      environment: nightly
+      type: MIXED
+      dynamic: true
+      vus: "100"
+      txs_per_user: "10000"
+      txpool_timeout: "60m0s"
+      wait_txpool: false
+      receipts_timeout: "2m0s"
+      batch_size: "10000"
+      total_time: ${{ needs.load_test_mixed.outputs.total_time }}
+      total_txs: ${{ needs.load_test_mixed.outputs.total_txs }}
+      total_blocks: ${{ needs.load_test_mixed.outputs.total_blocks }}
+      avg_txs_per_second: ${{ needs.load_test_mixed.outputs.avg_txs_per_second }}
+      max_txs_per_second: ${{ needs.load_test_mixed.outputs.max_txs_per_second }}
+      total_gas_used: ${{ needs.load_test_mixed.outputs.total_gas_used }}
+      avg_gas_per_tx: ${{ needs.load_test_mixed.outputs.avg_gas_per_tx }}
+      avg_gas_utilization: ${{ needs.load_test_mixed.outputs.avg_gas_utilization }}
+      max_gas_utilization: ${{ needs.load_test_mixed.outputs.max_gas_utilization }}
+      txpool_pending: ${{ needs.load_test_mixed.outputs.txpool_pending }}
+      txpool_queued: ${{ needs.load_test_mixed.outputs.txpool_queued }}
     secrets:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/notification-deploy-network.yml

@@ -103,22 +103,10 @@ jobs:
                 {
                   "type": "context",
                   "elements": [
-                    {
-                      "type": "mrkdwn",
-                      "text": "Triggered by: *${{ github.triggering_actor }}*"
-                    },
                     {
                       "type": "mrkdwn",
                       "text": "Commit: *<https://github.com/${{ github.repository }}/tree/${{ github.sha }}|${{ steps.short_sha.outputs.value }}>*"
-                    }
-                  ]
-                },
-                {
-                  "type": "divider"
-                },
-                {
-                  "type": "context",
-                  "elements": [
+                    },
                     {
                       "type": "mrkdwn",
                       "text": "Environment: *${{ inputs.environment }}*"
@@ -129,6 +117,9 @@ jobs:
                     }
                   ]
                 },
+                {
+                  "type": "divider"
+                },
                 {
                   "type": "context",
                   "elements": [
@@ -175,11 +166,11 @@ jobs:
                     },
                     {
                       "type": "mrkdwn",
-                      "text": "Gossip Message Size: *${{ inputs.gossip_msg_size }}*"
+                      "text": "Log Level: *${{ vars.LOG_LEVEL }}*"
                     },
                     {
                       "type": "mrkdwn",
-                      "text": "Log Level: *${{ vars.LOG_LEVEL }}*"
+                      "text": "Gossip Message Size: *${{ inputs.gossip_msg_size }}*"
                     }
                   ]
                 }

.github/workflows/notification-destroy-network.yml

@@ -74,10 +74,6 @@ jobs:
                 {
                   "type": "context",
                   "elements": [
-                    {
-                      "type": "mrkdwn",
-                      "text": "Triggered by: *${{ github.triggering_actor }}*"
-                    },
                     {
                       "type": "mrkdwn",
                       "text": "Environment: *${{ inputs.environment }}*"

.github/workflows/notification-load-test.yml

@@ -75,6 +75,14 @@ on: # yamllint disable-line rule:truthy
         description: Maximum Gas Utilization
         type: string
         required: true
+      txpool_pending:
+        description: Pending Transactions Count
+        type: string
+        required: true
+      txpool_queued:
+        description: Queued Transactions Count
+        type: string
+        required: true
     secrets:
       SLACK_WEBHOOK_URL:
         required: true
@@ -94,6 +102,7 @@ jobs:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
           SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
           succeed_bnt: 'primary'
+          failed_bnt: 'danger'
         with:
           payload: |
             {
@@ -114,30 +123,18 @@ jobs:
                         "type": "plain_text",
                         "text": "Workflow Run"
                       },
-                      "style": "${{ env.succeed_bnt }}",
+                      "style": "${{ inputs.txpool_pending == '0' && inputs.txpool_queued == '0' && env.succeed_bnt || env.failed_bnt }}",
                       "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                     }
                   ]
                 },
                 {
                   "type": "context",
                   "elements": [
-                    {
-                      "type": "mrkdwn",
-                      "text": "Triggered by: *${{ github.triggering_actor }}*"
-                    },
                     {
                       "type": "mrkdwn",
                       "text": "Commit: *<https://github.com/${{ github.repository }}/tree/${{ github.sha }}|${{ steps.short_sha.outputs.value }}>*"
-                    }
-                  ]
-                },
-                {
-                  "type": "divider"
-                },
-                {
-                  "type": "context",
-                  "elements": [
+                    },
                     {
                       "type": "mrkdwn",
                       "text": "Environment: *${{ inputs.environment }}*"
@@ -148,6 +145,9 @@ jobs:
                     }
                   ]
                 },
+                {
+                  "type": "divider"
+                },
                 {
                   "type": "context",
                   "elements": [
@@ -171,13 +171,29 @@ jobs:
                       "type": "mrkdwn",
                       "text": "Batch Size: *${{ inputs.batch_size }}*"
                     },
+                    {
+                      "type": "mrkdwn",
+                      "text": "Receipts Timeout: *${{ inputs.receipts_timeout }}*"
+                    },
                     {
                       "type": "mrkdwn",
                       "text": "Wait TX Pool To Empty: *${{ inputs.wait_txpool }}*"
+                    }
+                  ]
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "Pending TXs: *${{ inputs.txpool_pending }}*"
                     },
                     {
                       "type": "mrkdwn",
-                      "text": "Receipts Timeout: *${{ inputs.receipts_timeout }}*"
+                      "text": "Queued TXs: *${{ inputs.txpool_queued }}*"
                     }
                   ]
                 },