Merge PR #4861: Simulation Fixes

.pending/bugfixes/simulation/_Fix-non-determinism

@@ -0,0 +1,2 @@
+[\#4861](https://github.com/cosmos/cosmos-sdk/pull/4861) Fix non-determinism simulation
+by using CLI flags as input and updating Makefile target.

Makefile

@@ -86,8 +86,9 @@ test_race:
 	@VERSION=$(VERSION) go test -mod=readonly -race $(PACKAGES_NOSIMULATION)
 
 test_sim_nondeterminism:
-	@echo "Running nondeterminism test..."
-	@go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true -v -timeout 10m
+	@echo "Running non-determinism test..."
+	@go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true \
+	    -NumBlocks=100 -BlockSize=200 -Commit=true -v -timeout 24h
 
 test_sim_custom_genesis_fast:
 	@echo "Running custom genesis simulation..."
@@ -125,7 +126,6 @@ test_sim_benchmark_invariants:
 .PHONY: test \
 test_sim_nondeterminism \
 test_sim_custom_genesis_fast \
-test_sim_fast \
 test_sim_import_export \
 test_sim_after_import \
 test_sim_custom_genesis_multi_seed \

simapp/sim_test.go

@@ -681,7 +681,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 
 	defer func() {
 		newDB.Close()
-		os.RemoveAll(newDir)
+		_ = os.RemoveAll(newDir)
 	}()
 
 	newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, 0, fauxMerkleModeOpt)
@@ -708,21 +708,29 @@ func TestAppStateDeterminism(t *testing.T) {
 
 	for i := 0; i < numSeeds; i++ {
 		seed := rand.Int63()
+
 		for j := 0; j < numTimesToRunPerSeed; j++ {
 			logger := log.NewNopLogger()
 			db := dbm.NewMemDB()
 			app := NewSimApp(logger, db, nil, true, 0)
 
-			// Run randomized simulation
-			simulation.SimulateFromSeed(
-				t, os.Stdout, app.BaseApp, appStateFn, seed,
-				testAndRunTxs(app), []sdk.Invariant{},
-				1, 50, 100, 0, "",
-				false, true, false, false, false, app.ModuleAccountAddrs(),
+			fmt.Printf(
+				"Running non-determinism simulation; seed: %d/%d (%d), attempt: %d/%d\n",
+				i+1, numSeeds, seed, j+1, numTimesToRunPerSeed,
+			)
+
+			_, _, err := simulation.SimulateFromSeed(
+				t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
+				[]sdk.Invariant{}, 1, numBlocks, exportParamsHeight,
+				blockSize, "", false, commit, lean,
+				false, false, app.ModuleAccountAddrs(),
 			)
+			require.NoError(t, err)
+
 			appHash := app.LastCommitID().Hash
 			appHashList[j] = appHash
 		}
+
 		for k := 1; k < numTimesToRunPerSeed; k++ {
 			require.Equal(t, appHashList[0], appHashList[k], "appHash list: %v", appHashList)
 		}

x/simulation/simulate.go

@@ -7,7 +7,6 @@ import (
 	"math/rand"
 	"os"
 	"os/signal"
-	"runtime/debug"
 	"syscall"
 	"testing"
 	"time"
@@ -126,14 +125,12 @@ func SimulateFromSeed(
 	if !testingMode {
 		b.ResetTimer()
 	} else {
-		// Recover logs in case of panic
+		// recover logs in case of panic
 		defer func() {
 			if r := recover(); r != nil {
-				fmt.Fprintf(w, "panic with err: %v\n", r)
-				stackTrace := string(debug.Stack())
-				fmt.Println(stackTrace)
+				_, _ = fmt.Fprintf(w, "simulation halted due to panic on block %d; %v\n", header.Height, r)
 				logWriter.PrintLogs()
-				err = fmt.Errorf("Simulation halted due to panic on block %d", header.Height)
+				panic(r)
 			}
 		}()
 	}
@@ -258,22 +255,27 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr
 	operationQueue OperationQueue, timeOperationQueue []FutureOperation,
 	totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn {
 
-	lastBlocksizeState := 0 // state for [4 * uniform distribution]
+	lastBlockSizeState := 0 // state for [4 * uniform distribution]
 	blocksize := 0
 	selectOp := ops.getSelectOpFn()
 
-	return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
-		accounts []Account, header abci.Header) (opCount int) {
+	return func(
+		r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accounts []Account, header abci.Header,
+	) (opCount int) {
 
-		fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
-			header.Height, totalNumBlocks, opCount, blocksize)
-		lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize)
+		_, _ = fmt.Fprintf(
+			w, "\rSimulating... block %d/%d, operation %d/%d.",
+			header.Height, totalNumBlocks, opCount, blocksize,
+		)
+		lastBlockSizeState, blocksize = getBlockSize(r, params, lastBlockSizeState, avgBlockSize)
 
 		type opAndR struct {
 			op   Operation
 			rand *rand.Rand
 		}
+
 		opAndRz := make([]opAndR, 0, blocksize)
+
 		// Predetermine the blocksize slice so that we can do things like block
 		// out certain operations without changing the ops that follow.
 		for i := 0; i < blocksize; i++ {

x/simulation/util.go

@@ -53,21 +53,23 @@ func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B
 //  - "over stuffed" blocks with average size of 2 * avgblocksize,
 //  - normal sized blocks, hitting avgBlocksize on average,
 //  - and empty blocks, with no txs / only txs scheduled from the past.
-func getBlockSize(r *rand.Rand, params Params,
-	lastBlockSizeState, avgBlockSize int) (state, blocksize int) {
-
+func getBlockSize(r *rand.Rand, params Params, lastBlockSizeState, avgBlockSize int) (state, blockSize int) {
 	// TODO: Make default blocksize transition matrix actually make the average
 	// blocksize equal to avgBlockSize.
 	state = params.BlockSizeTransitionMatrix.NextState(r, lastBlockSizeState)
+
 	switch state {
 	case 0:
-		blocksize = r.Intn(avgBlockSize * 4)
+		blockSize = r.Intn(avgBlockSize * 4)
+
 	case 1:
-		blocksize = r.Intn(avgBlockSize * 2)
+		blockSize = r.Intn(avgBlockSize * 2)
+
 	default:
-		blocksize = 0
+		blockSize = 0
 	}
-	return state, blocksize
+
+	return state, blockSize
 }
 
 // PeriodicInvariants  returns an array of wrapped Invariants. Where each