Add Sanity Check Tests

.github/workflows/deploy-network.yml

@@ -194,6 +194,8 @@ jobs:
           sed 's/gossip_msg_size: .*/gossip_msg_size: ${{ inputs.gossip_msg_size }}/g' group_vars/all.yml > group_vars/all.yml.tmp && mv group_vars/all.yml.tmp group_vars/all.yml
           sed 's/json_rpc_batch_request_limit: .*/json_rpc_batch_request_limit: 0/g' group_vars/all.yml > group_vars/all.yml.tmp && mv group_vars/all.yml.tmp group_vars/all.yml
           sed 's/log_level: .*/log_level: ${{ vars.LOG_LEVEL }}/g' group_vars/all.yml > group_vars/all.yml.tmp && mv group_vars/all.yml.tmp group_vars/all.yml
+          sed 's/epoch_reward: .*/epoch_reward: 1000000000/g' group_vars/all.yml > group_vars/all.yml.tmp && mv group_vars/all.yml.tmp group_vars/all.yml
+          sed 's/reward_wallet_balance: .*/reward_wallet_balance: 0xD3C21BCECCEDA1000000/g' group_vars/all.yml > group_vars/all.yml.tmp && mv group_vars/all.yml.tmp group_vars/all.yml
       - name: Setup Ansible
         working-directory: ansible
         run: |

.github/workflows/nightly.yml

@@ -36,10 +36,20 @@ jobs:
       AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
       AWS_S3_BLADE_BUCKET: ${{ secrets.AWS_S3_BLADE_BUCKET }}
       VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }}
+  sanity_check_tests:
+    name: Sanity Check Tests
+    uses: ./.github/workflows/sanity-check-test.yml
+    needs: deploy_network
+    with:
+      environment: nightly
+    secrets:
+      AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
+      AWS_S3_BLADE_BUCKET: ${{ secrets.AWS_S3_BLADE_BUCKET }}
+      AWS_LOADTESTRUNNER_MNEMONIC: ${{ secrets.AWS_LOADTESTRUNNER_MNEMONIC }}
   load_test_eoa:
     name: Load Test EOA
     uses: ./.github/workflows/load-test.yml
-    needs: deploy_network
+    needs: [deploy_network, sanity_check_tests]
     with:
       environment: nightly
       type: EOA
@@ -63,7 +73,7 @@ jobs:
   load_test_erc20:
     name: Load Test ERC20
     uses: ./.github/workflows/load-test.yml
-    needs: [deploy_network, load_test_eoa]
+    needs: [deploy_network, sanity_check_tests, load_test_eoa]
     with:
       environment: nightly
       type: ERC20
@@ -87,7 +97,7 @@ jobs:
   load_test_erc721:
     name: Load Test ERC721
     uses: ./.github/workflows/load-test.yml
-    needs: [deploy_network, load_test_eoa, load_test_erc20]
+    needs: [deploy_network, sanity_check_tests, load_test_eoa, load_test_erc20]
     with:
       environment: nightly
       type: ERC721
@@ -111,7 +121,7 @@ jobs:
   load_test_mixed:
     name: Load Test MIXED
     uses: ./.github/workflows/load-test.yml
-    needs: [deploy_network, load_test_eoa, load_test_erc20, load_test_erc721]
+    needs: [deploy_network, sanity_check_tests, load_test_eoa, load_test_erc20, load_test_erc721]
     with:
       environment: nightly
       type: MIXED
@@ -135,7 +145,7 @@ jobs:
   destroy_network:
     name: Destroy Network
     uses: ./.github/workflows/destroy-network.yml
-    needs: [deploy_network, load_test_eoa, load_test_erc20, load_test_erc721, load_test_mixed]
+    needs: [deploy_network, sanity_check_tests, load_test_eoa, load_test_erc20, load_test_erc721, load_test_mixed]
     if: always()
     with:
       environment: nightly
@@ -148,7 +158,7 @@ jobs:
   notification_nightly:
     name: Nightly Notification
     uses: ./.github/workflows/notification-nightly.yml
-    needs: [ci, deploy_network, load_test_eoa, load_test_erc20, load_test_erc721, load_test_mixed, destroy_network]
+    needs: [ci, deploy_network, sanity_check_tests, load_test_eoa, load_test_erc20, load_test_erc721, load_test_mixed, destroy_network]
     if: success() || failure()
     with:
       environment: nightly
@@ -168,6 +178,12 @@ jobs:
       property_polybft_test_output: ${{ needs.ci.outputs.property_polybft_test }}
       fuzz_test_output: ${{ needs.ci.outputs.fuzz_test }}
       benchmark_test_output: ${{ needs.ci.outputs.benchmark_test }}
+      sanity_check_tests_output: ${{ needs.sanity_check_tests.outputs.sanity_check_tests_output }}
+      stake_test_output: ${{ needs.sanity_check_tests.outputs.stake_test_output }}
+      unstake_test_output: ${{ needs.sanity_check_tests.outputs.unstake_test_output }}
+      register_validator_test_output: ${{ needs.sanity_check_tests.outputs.register_validator_test_output }}
+      withdraw_rewards_test_output: ${{ needs.sanity_check_tests.outputs.withdraw_rewards_test_output }}
+      unstake_all_test_output: ${{ needs.sanity_check_tests.outputs.unstake_all_test_output }}
       deploy_network_terraform_output: ${{ needs.deploy_network.outputs.terraform_output }}
       deploy_network_ansible_output: ${{ needs.deploy_network.outputs.ansible_output }}
       load_test_eoa_output: ${{ needs.load_test_eoa.outputs.load_test_output }}

.github/workflows/notification-nightly.yml

@@ -71,6 +71,30 @@ on: # yamllint disable-line rule:truthy
         description: Benchmark Tests output
         type: string
         required: true
+      sanity_check_tests_output:
+        description: Sanity Check Tests output
+        type: string
+        required: true
+      stake_test_output:
+        description: Stake Test output
+        type: string
+        required: true
+      unstake_test_output:
+        description: Unstake Test output
+        type: string
+        required: true
+      register_validator_test_output:
+        description: Register Validator Test output
+        type: string
+        required: true
+      withdraw_rewards_test_output:
+        description: Withdraw Rewards Test output
+        type: string
+        required: true
+      unstake_all_test_output:
+        description: Unstake All Test output
+        type: string
+        required: true
       deploy_network_terraform_output:
         description: Deploy Network - Terraform output
         type: string
@@ -187,7 +211,7 @@ jobs:
                         "type": "plain_text",
                         "text": "Workflow Run"
                       },
-                      "style": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.property_polybft_test_output == '' && inputs.fuzz_test_output == '' && inputs.benchmark_test_output == '' && inputs.deploy_network_terraform_output == '' && inputs.deploy_network_ansible_output == '' && inputs.load_test_eoa_output == 'true' && inputs.load_test_eoa_txpool_pending_output == '0' && inputs.load_test_eoa_txpool_queued_output == '0' && inputs.load_test_erc20_output == 'true' && inputs.load_test_erc20_txpool_pending_output == '0' && inputs.load_test_erc20_txpool_queued_output == '0' && inputs.load_test_erc721_output == 'true' && inputs.load_test_erc721_txpool_pending_output == '0' && inputs.load_test_erc721_txpool_queued_output == '0' && inputs.load_test_mixed_output == 'true' && inputs.load_test_mixed_txpool_pending_output == '0' && inputs.load_test_mixed_txpool_queued_output == '0' && inputs.destroy_network_logs_output == '' && inputs.destroy_network_terraform_output == '' && env.succeed_bnt || env.failed_bnt }}",
+                      "style": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.property_polybft_test_output == '' && inputs.fuzz_test_output == '' && inputs.benchmark_test_output == '' && inputs.sanity_check_tests_output == 'true' && inputs.deploy_network_terraform_output == '' && inputs.deploy_network_ansible_output == '' && inputs.load_test_eoa_output == 'true' && inputs.load_test_eoa_txpool_pending_output == '0' && inputs.load_test_eoa_txpool_queued_output == '0' && inputs.load_test_erc20_output == 'true' && inputs.load_test_erc20_txpool_pending_output == '0' && inputs.load_test_erc20_txpool_queued_output == '0' && inputs.load_test_erc721_output == 'true' && inputs.load_test_erc721_txpool_pending_output == '0' && inputs.load_test_erc721_txpool_queued_output == '0' && inputs.load_test_mixed_output == 'true' && inputs.load_test_mixed_txpool_pending_output == '0' && inputs.load_test_mixed_txpool_queued_output == '0' && inputs.destroy_network_logs_output == '' && inputs.destroy_network_terraform_output == '' && env.succeed_bnt || env.failed_bnt }}",
                       "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                     },
                     {
@@ -315,6 +339,31 @@ jobs:
                     }
                   ]
                 },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ inputs.stake_test_output == 'PASSED' && env.succeed_job || env.failed_job }} *Stake Test*"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ inputs.unstake_test_output == 'PASSED' && env.succeed_job || env.failed_job }} *Unstake Test*"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ inputs.register_validator_test_output == 'PASSED' && env.succeed_job || env.failed_job }} *Register Validator Test*"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ inputs.withdraw_rewards_test_output == 'PASSED' && env.succeed_job || env.failed_job }} *Withdraw Rewards Test*"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ inputs.unstake_all_test_output == 'PASSED' && env.succeed_job || env.failed_job }} *Unstake All Test*"
+                    }
+                  ]
+                },
                 {
                   "type": "divider"
                 },
@@ -327,7 +376,7 @@ jobs:
                     },
                     {
                       "type": "mrkdwn",
-                      "text": "${{ inputs.benchmark_test_output == '' && env.succeed_job || env.failed_job }} *Destroy Network*"
+                      "text": "${{ inputs.destroy_network_terraform_output == '' && inputs.destroy_network_logs_output == '' && env.succeed_job || env.failed_job }} *Destroy Network*"
                     },
                     {
                       "type": "mrkdwn",

.github/workflows/sanity-check-test.yml

@@ -0,0 +1,114 @@
+---
+name: Sanity Check Tests
+on: # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: The environment to run against
+        type: choice
+        options: [dev, test] # nightly should not be initiated manually
+  workflow_call:
+    inputs:
+      environment:
+        description: The environment to run against
+        type: string
+        required: true
+    outputs:
+      sanity_check_tests_output:
+        description: Sanity Check Tests output
+        value: ${{ jobs.sanity_check_tests.outputs.test_output_success }}
+      stake_test_output:
+        description: Stake Test output
+        value: ${{ jobs.sanity_check_tests.outputs.stake_test }}
+      unstake_test_output:
+        description: Unstake Test output
+        value: ${{ jobs.sanity_check_tests.outputs.unstake_test }}
+      register_validator_test_output:
+        description: Register Validator Test output
+        value: ${{ jobs.sanity_check_tests.outputs.register_validator_test }}
+      withdraw_rewards_test_output:
+        description: Withdraw Rewards Test output
+        value: ${{ jobs.sanity_check_tests.outputs.withdraw_rewards_test }}
+      unstake_all_test_output:
+        description: Unstake All Test output
+        value: ${{ jobs.sanity_check_tests.outputs.unstake_all_test }}
+    secrets:
+      AWS_ROLE_ARN:
+        required: true
+      AWS_S3_BLADE_BUCKET:
+        required: true
+      AWS_LOADTESTRUNNER_MNEMONIC:
+        required: true
+
+permissions:
+  id-token: write
+  contents: read
+  security-events: write
+
+jobs:
+  check_network:
+    name: Check if the network is already deployed
+    runs-on: ubuntu-latest
+    outputs:
+      rpc_url: ${{ steps.rpc_url.outputs.url }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.1
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+      - name: Retrieve state file from s3
+        id: retrieve_state
+        run: echo "retrieve_state_output=$(aws s3 cp s3://${{ secrets.AWS_S3_BLADE_BUCKET }}/states/${{ inputs.environment }} state.json)" >> $GITHUB_OUTPUT
+      - name: Set RPC URL
+        id: rpc_url
+        if: contains(steps.retrieve_state.outputs.retrieve_state_output, 'download')
+        run: echo "url=$(cat state.json | jq -r '.outputs.aws_lb_ext_domain.value // empty')" >> $GITHUB_OUTPUT
+  sanity_check_tests:
+    name: Run Sanity Check
+    runs-on: ubuntu-latest
+    needs: check_network
+    if: needs.check_network.outputs.rpc_url != ''
+    environment: ${{ inputs.environment }}
+    outputs:
+      test_output_success: ${{ steps.sanity_check_tests_results.outputs.test_output }}
+      stake_test: ${{ steps.sanity_check_tests_results.outputs.stake_test }}
+      unstake_test: ${{ steps.sanity_check_tests_results.outputs.unstake_test }}
+      register_validator_test: ${{ steps.sanity_check_tests_results.outputs.register_validator_test }}
+      withdraw_rewards_test: ${{ steps.sanity_check_tests_results.outputs.withdraw_rewards_test }}
+      unstake_all_test: ${{ steps.sanity_check_tests_results.outputs.unstake_all_test }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Install Go
+        uses: actions/setup-go@v5.0.0
+        with:
+          go-version: 1.21.x
+      - name: Build Blade
+        run: make build
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.1
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+      - name: Get validator keys
+        run: aws ssm get-parameters --names `aws ec2 describe-instances --filters "Name=tag:BaseDN,Values=${{ inputs.environment }}${{ vars.BASEDN_SUFIX }}" "Name=tag:Role,Values=validator" "Name=instance-state-name,Values=running" --query "Reservations[*].Instances[*].Tags[?Key=='Name'].Value[]" --output text | sed 's/.*/\/${{ inputs.environment }}\/&\/validator-key/'` --with-decryption --region ${{ vars.AWS_REGION }} > keys.json
+      - name: Start Sanity Check Test
+        id: sanity_check_tests_results
+        run: |
+          ./blade sanity-check --jsonrpc "http://${{ needs.check_network.outputs.rpc_url }}" --mnemonic "${{ secrets.AWS_LOADTESTRUNNER_MNEMONIC }}" --receipts-timeout 2m --validator-keys "`cat keys.json | jq -r '.Parameters[].Value' | tr '\n' ',' | sed 's/,$//'`" --to-json
+          echo "test_output=`cat sanity_check_results.json | jq 'map(select(.Result != "PASSED")) | length == 0'`" >> $GITHUB_OUTPUT
+          echo "stake_test=`cat sanity_check_results.json | jq -r '.[] | select(.Name == "Stake Test") | .Result'`" >> $GITHUB_OUTPUT
+          echo "unstake_test=`cat sanity_check_results.json | jq -r '.[] | select(.Name == "Unstake Test") | .Result'`" >> $GITHUB_OUTPUT
+          echo "register_validator_test=`cat sanity_check_results.json | jq -r '.[] | select(.Name == "Register Validator Test") | .Result'`" >> $GITHUB_OUTPUT
+          echo "withdraw_rewards_test=`cat sanity_check_results.json | jq -r '.[] | select(.Name == "Withdraw Rewards Test") | .Result'`" >> $GITHUB_OUTPUT
+          echo "unstake_all_test=`cat sanity_check_results.json | jq -r '.[] | select(.Name == "Unstake All Test") | .Result'`" >> $GITHUB_OUTPUT
+      - name: Archive Sanity Check Test results
+        if: always()
+        uses: actions/upload-artifact@v4.3.0
+        with:
+          name: sanity-check-results
+          path: sanity_check_results.json
+          retention-days: 30

blockchain/storagev2/leveldb/leveldb_perf_test.go

@@ -39,7 +39,7 @@ func Benchmark(b *testing.B) {
 	}()
 
 	blockCount := 1000
-	storagev2.BenchmarkStorage(b, blockCount, s, 25, 15) // CI times
+	storagev2.BenchmarkStorage(b, blockCount, s, 26, 15) // CI times
 
 	size, err := dbSize(path)
 	require.NoError(b, err)

loadtest/sanitycheck/base_sanity_check.go

@@ -161,11 +161,15 @@ func (t *BaseSanityCheckTest) waitForEpochEnding(fromBlock *uint64) (*types.Head
 		case <-timer.C:
 			return nil, fmt.Errorf("timed out waiting for end of epoch")
 		case <-ticker.C:
-			if currentBlock.Number()%t.config.EpochSize == 0 {
-				return currentBlock.Header, nil
+			if currentBlock != nil {
+				if currentBlock.Number()%t.config.EpochSize == 0 {
+					return currentBlock.Header, nil
+				}
+
+				rpcBlock = jsonrpc.BlockNumber(currentBlock.Number() + 1)
 			}
 
-			block, err := t.client.GetBlockByNumber(jsonrpc.BlockNumber(currentBlock.Number()+1), false)
+			block, err := t.client.GetBlockByNumber(rpcBlock, false)
 			if err != nil {
 				return nil, err
 			}

loadtest/sanitycheck/sanity_check_register_validator.go

@@ -85,6 +85,11 @@ func (t *RegisterValidatorTest) runTest() (*wallet.Account, error) {
 		return nil, fmt.Errorf("failed to register new validator: %w", err)
 	}
 
+	if blockNum%t.config.EpochSize == 0 {
+		// if validator was registered on the epoch ending block, it will become active on the next epoch
+		blockNum++
+	}
+
 	epochEndingBlock, err := t.waitForEpochEnding(&blockNum)
 	if err != nil {
 		return nil, err
@@ -98,11 +103,13 @@ func (t *RegisterValidatorTest) runTest() (*wallet.Account, error) {
 	fmt.Println("Checking if new validator is added to validator set with its stake")
 
 	if extra.Validators == nil || extra.Validators.IsEmpty() {
-		return nil, fmt.Errorf("validator set delta is empty on an epoch ending block")
+		return nil, fmt.Errorf("validator set delta is empty on an epoch ending block. Block: %d. EpochSize: %d",
+			epochEndingBlock.Number, t.config.EpochSize)
 	}
 
 	if !extra.Validators.Added.ContainsAddress(newValidatorAcc.Address()) {
-		return nil, fmt.Errorf("validator %s is not in the added validators", newValidatorAcc.Address())
+		return nil, fmt.Errorf("validator %s is not in the added validators. Block: %d. EpochSize: %d",
+			newValidatorAcc.Address(), epochEndingBlock.Number, t.config.EpochSize)
 	}
 
 	validatorMetaData := extra.Validators.Added.GetValidatorMetadata(newValidatorAcc.Address())

loadtest/sanitycheck/sanity_check_runner.go

@@ -22,14 +22,14 @@ type SanityCheckTestResult struct {
 	Name          string
 	ExecutionTime string
 	Result        string
-	Err           error
+	Err           string
 }
 
 // String returns a string representation of the SanityCheckTestResult.
 func (s SanityCheckTestResult) String() string {
-	if s.Err != nil {
+	if s.Err != "" {
 		return fmt.Sprintf("%s: Execution time: %s. Error: %s. Result: %s.",
-			s.Name, s.ExecutionTime, s.Err.Error(), s.Result)
+			s.Name, s.ExecutionTime, s.Err, s.Result)
 	}
 
 	return fmt.Sprintf("%s: Execution time: %s. Result: %s.", s.Name, s.ExecutionTime, s.Result)
@@ -147,16 +147,19 @@ func (r *SanityCheckTestRunner) Run() error {
 		result := passed
 		t := time.Now().UTC()
 
+		errMsg := ""
+
 		err := test.Run()
 		if err != nil {
 			result = failed
+			errMsg = err.Error()
 		}
 
 		results = append(results, SanityCheckTestResult{
 			Name:          test.Name(),
 			ExecutionTime: time.Since(t).String(),
 			Result:        result,
-			Err:           err,
+			Err:           errMsg,
 		})
 	}