diff --git a/.github/workflows/simulation.yml b/.github/workflows/simulation.yml new file mode 100644 index 000000000..e0edfedbf --- /dev/null +++ b/.github/workflows/simulation.yml @@ -0,0 +1,91 @@ +name: Simulation +on: pull_request + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: "1.19" + check-latest: true + - run: make build + - name: Install runsim + run: go install github.com/cosmos/tools/cmd/runsim@v1.0.0 + - uses: actions/cache@v3 + with: + path: ~/go/bin + key: ${{ runner.os }}-go-runsim-binary + + test-sim-import-export: + runs-on: ubuntu-latest + needs: [build] + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: "1.19" + check-latest: true + - uses: actions/cache@v3 + with: + path: ~/go/bin + key: ${{ runner.os }}-go-runsim-binary + - name: test-sim-import-export + run: | + make test-sim-import-export + + test-sim-after-import: + runs-on: ubuntu-latest + needs: [build] + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: "1.19" + check-latest: true + - uses: actions/cache@v3 + with: + path: ~/go/bin + key: ${{ runner.os }}-go-runsim-binary + - name: test-sim-after-import + run: | + make test-sim-after-import + + test-sim-multi-seed-short: + runs-on: ubuntu-latest + needs: [build] + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: "1.19" + check-latest: true + - uses: actions/cache@v3 + with: + path: ~/go/bin + key: ${{ runner.os }}-go-runsim-binary + - name: test-sim-multi-seed-short + run: | + make test-sim-multi-seed-short + + test-sim-deterministic: + runs-on: ubuntu-latest + needs: [build] + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: "1.19" + check-latest: true + - uses: actions/cache@v3 + with: + path: ~/go/bin + key: ${{ runner.os }}-go-runsim-binary + - name: test-sim-deterministic + run: | + make test-sim-deterministic diff --git a/app/sim_test.go b/app/sim_test.go new file mode 100644 index 000000000..372ad08f4 --- /dev/null +++ b/app/sim_test.go @@ -0,0 +1,510 @@ +package band + +import ( + "encoding/json" + "fmt" + "math/rand" + "os" + "runtime/debug" + "strings" + "testing" + + dbm "github.com/cometbft/cometbft-db" + abci "github.com/cometbft/cometbft/abci/types" + "github.com/cometbft/cometbft/libs/log" + tmproto "github.com/cometbft/cometbft/proto/tendermint/types" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" + "github.com/cosmos/cosmos-sdk/store" + storetypes "github.com/cosmos/cosmos-sdk/store/types" + simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + sdk "github.com/cosmos/cosmos-sdk/types" + simtypes "github.com/cosmos/cosmos-sdk/types/simulation" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + authzkeeper "github.com/cosmos/cosmos-sdk/x/authz/keeper" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" + capabilitytypes "github.com/cosmos/cosmos-sdk/x/capability/types" + distrtypes "github.com/cosmos/cosmos-sdk/x/distribution/types" + evidencetypes "github.com/cosmos/cosmos-sdk/x/evidence/types" + govtypes "github.com/cosmos/cosmos-sdk/x/gov/types" + minttypes "github.com/cosmos/cosmos-sdk/x/mint/types" + paramtypes "github.com/cosmos/cosmos-sdk/x/params/types" + "github.com/cosmos/cosmos-sdk/x/simulation" + simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" + slashingtypes "github.com/cosmos/cosmos-sdk/x/slashing/types" + stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" + "github.com/stretchr/testify/require" + + oracletypes "github.com/bandprotocol/chain/v2/x/oracle/types" +) + +// BandAppChainID hardcoded chainID for simulation +const BandAppChainID = "simulation-app" + +// Get flags every time the simulator is run +func init() { + simcli.GetSimulatorFlags() +} + +type StoreKeysPrefixes struct { + A storetypes.StoreKey + B storetypes.StoreKey + Prefixes [][]byte +} + +// fauxMerkleModeOpt returns a BaseApp option to use a dbStoreAdapter instead of +// an IAVLStore for faster simulation speed. +func fauxMerkleModeOpt(bapp *baseapp.BaseApp) { + bapp.SetFauxMerkleMode() +} + +// interBlockCacheOpt returns a BaseApp option function that sets the persistent +// inter-block write-through cache. +func interBlockCacheOpt() func(*baseapp.BaseApp) { + return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager()) +} + +func TestFullAppSimulation(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = BandAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation( + config, + "leveldb-app-sim", + "Simulation", + simcli.FlagVerboseValue, + simcli.FlagEnabledValue, + ) + if skip { + t.Skip("skipping application simulation") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + app := NewBandApp( + logger, + db, + nil, + true, + map[int64]bool{}, + appOptions, + 100, + fauxMerkleModeOpt, + baseapp.SetChainID(BandAppChainID), + ) + require.Equal(t, "BandApp", app.Name()) + + // run randomized simulation + _, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), NewDefaultGenesisState()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.ModuleAccountAddrs(), + config, + app.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } +} + +func TestAppImportExport(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = BandAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation( + config, + "leveldb-app-sim", + "Simulation", + simcli.FlagVerboseValue, + simcli.FlagEnabledValue, + ) + if skip { + t.Skip("skipping application import/export simulation") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + app := NewBandApp( + logger, + db, + nil, + true, + map[int64]bool{}, + appOptions, + 100, + fauxMerkleModeOpt, + baseapp.SetChainID(BandAppChainID), + ) + require.Equal(t, "BandApp", app.Name()) + + // Run randomized simulation + _, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), NewDefaultGenesisState()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.ModuleAccountAddrs(), + config, + app.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } + + fmt.Printf("exporting genesis...\n") + + exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) + require.NoError(t, err) + + fmt.Printf("importing genesis...\n") + + newDB, newDir, _, _, err := simtestutil.SetupSimulation( + config, + "leveldb-app-sim-2", + "Simulation-2", + simcli.FlagVerboseValue, + simcli.FlagEnabledValue, + ) + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, newDB.Close()) + require.NoError(t, os.RemoveAll(newDir)) + }() + + newApp := NewBandApp( + log.NewNopLogger(), + newDB, + nil, + true, + map[int64]bool{}, + appOptions, + 100, + fauxMerkleModeOpt, + baseapp.SetChainID(BandAppChainID), + ) + require.Equal(t, "BandApp", newApp.Name()) + + var genesisState GenesisState + err = json.Unmarshal(exported.AppState, &genesisState) + require.NoError(t, err) + + defer func() { + if r := recover(); r != nil { + err := fmt.Sprintf("%v", r) + if !strings.Contains(err, "validator set is empty after InitGenesis") { + panic(r) + } + logger.Info("Skipping simulation as all validators have been unbonded") + logger.Info("err", err, "stacktrace", string(debug.Stack())) + } + }() + + ctxA := app.NewContext(true, tmproto.Header{Height: app.LastBlockHeight()}) + ctxB := newApp.NewContext(true, tmproto.Header{Height: app.LastBlockHeight()}) + newApp.mm.InitGenesis(ctxB, app.AppCodec(), genesisState) + newApp.StoreConsensusParams(ctxB, exported.ConsensusParams) + + fmt.Printf("comparing stores...\n") + + storeKeysPrefixes := []StoreKeysPrefixes{ + {app.GetKey(authtypes.StoreKey), newApp.GetKey(authtypes.StoreKey), [][]byte{}}, + { + app.GetKey(stakingtypes.StoreKey), newApp.GetKey(stakingtypes.StoreKey), + [][]byte{ + stakingtypes.UnbondingQueueKey, stakingtypes.RedelegationQueueKey, stakingtypes.ValidatorQueueKey, + stakingtypes.HistoricalInfoKey, stakingtypes.UnbondingIDKey, stakingtypes.UnbondingIndexKey, stakingtypes.UnbondingTypeKey, stakingtypes.ValidatorUpdatesKey, + }, + }, // ordering may change but it doesn't matter + {app.GetKey(slashingtypes.StoreKey), newApp.GetKey(slashingtypes.StoreKey), [][]byte{}}, + {app.GetKey(minttypes.StoreKey), newApp.GetKey(minttypes.StoreKey), [][]byte{}}, + {app.GetKey(distrtypes.StoreKey), newApp.GetKey(distrtypes.StoreKey), [][]byte{}}, + {app.GetKey(banktypes.StoreKey), newApp.GetKey(banktypes.StoreKey), [][]byte{banktypes.BalancesPrefix}}, + {app.GetKey(paramtypes.StoreKey), newApp.GetKey(paramtypes.StoreKey), [][]byte{}}, + {app.GetKey(govtypes.StoreKey), newApp.GetKey(govtypes.StoreKey), [][]byte{}}, + {app.GetKey(evidencetypes.StoreKey), newApp.GetKey(evidencetypes.StoreKey), [][]byte{}}, + {app.GetKey(capabilitytypes.StoreKey), newApp.GetKey(capabilitytypes.StoreKey), [][]byte{}}, + { + app.GetKey(authzkeeper.StoreKey), + newApp.GetKey(authzkeeper.StoreKey), + [][]byte{authzkeeper.GrantKey, authzkeeper.GrantQueuePrefix}, + }, + {app.GetKey(oracletypes.StoreKey), newApp.GetKey(oracletypes.StoreKey), [][]byte{ + oracletypes.RequestCountStoreKey, + oracletypes.RequestLastExpiredStoreKey, + oracletypes.PendingResolveListStoreKey, + oracletypes.RequestStoreKeyPrefix, + oracletypes.ReportStoreKeyPrefix, + oracletypes.ValidatorStatusKeyPrefix, + oracletypes.ResultStoreKeyPrefix, + }}, + } + + for _, skp := range storeKeysPrefixes { + storeA := ctxA.KVStore(skp.A) + storeB := ctxB.KVStore(skp.B) + + failedKVAs, failedKVBs := sdk.DiffKVStores(storeA, storeB, skp.Prefixes) + require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare") + + fmt.Printf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), skp.A, skp.B) + require.Equal( + t, + 0, + len(failedKVAs), + simtestutil.GetSimulationLog(skp.A.Name(), app.SimulationManager().StoreDecoders, failedKVAs, failedKVBs), + ) + } +} + +func TestAppSimulationAfterImport(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = BandAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation( + config, + "leveldb-app-sim", + "Simulation", + simcli.FlagVerboseValue, + simcli.FlagEnabledValue, + ) + if skip { + t.Skip("skipping application simulation after import") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + app := NewBandApp( + logger, + db, + nil, + true, + map[int64]bool{}, + appOptions, + 100, + fauxMerkleModeOpt, + baseapp.SetChainID(BandAppChainID), + ) + require.Equal(t, "BandApp", app.Name()) + + // Run randomized simulation + stopEarly, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), NewDefaultGenesisState()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.ModuleAccountAddrs(), + config, + app.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } + + if stopEarly { + fmt.Println("can't export or import a zero-validator genesis, exiting test...") + return + } + + fmt.Printf("exporting genesis...\n") + + exported, err := app.ExportAppStateAndValidators(true, []string{}, []string{}) + require.NoError(t, err) + + fmt.Printf("importing genesis...\n") + + newDB, newDir, _, _, err := simtestutil.SetupSimulation( + config, + "leveldb-app-sim-2", + "Simulation-2", + simcli.FlagVerboseValue, + simcli.FlagEnabledValue, + ) + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, newDB.Close()) + require.NoError(t, os.RemoveAll(newDir)) + }() + + newApp := NewBandApp( + log.NewNopLogger(), + newDB, + nil, + true, + map[int64]bool{}, + appOptions, + 100, + fauxMerkleModeOpt, + baseapp.SetChainID(BandAppChainID), + ) + + require.Equal(t, "BandApp", newApp.Name()) + + newApp.InitChain(abci.RequestInitChain{ + ChainId: BandAppChainID, + AppStateBytes: exported.AppState, + }) + + _, _, err = simulation.SimulateFromSeed( + t, + os.Stdout, + newApp.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), NewDefaultGenesisState()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(newApp, newApp.AppCodec(), config), + app.ModuleAccountAddrs(), + config, + app.AppCodec(), + ) + require.NoError(t, err) +} + +// TODO: Make another test for the fuzzer itself, which just has noOp txs +// and doesn't depend on the application. +func TestAppStateDeterminism(t *testing.T) { + if !simcli.FlagEnabledValue { + t.Skip("skipping application simulation") + } + + config := simcli.NewConfigFromFlags() + config.InitialBlockHeight = 1 + config.ExportParamsPath = "" + config.OnOperation = false + config.AllInvariants = false + config.ChainID = BandAppChainID + + numSeeds := 3 + numTimesToRunPerSeed := 5 + + // We will be overriding the random seed and just run a single simulation on the provided seed value + if config.Seed != simcli.DefaultSeedValue { + numSeeds = 1 + } + + appHashList := make([]json.RawMessage, numTimesToRunPerSeed) + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + for i := 0; i < numSeeds; i++ { + if config.Seed == simcli.DefaultSeedValue { + config.Seed = rand.Int63() + } + + fmt.Println("config.Seed: ", config.Seed) + + for j := 0; j < numTimesToRunPerSeed; j++ { + var logger log.Logger + if simcli.FlagVerboseValue { + logger = log.TestingLogger() + } else { + logger = log.NewNopLogger() + } + + db := dbm.NewMemDB() + app := NewBandApp( + logger, + db, + nil, + true, + map[int64]bool{}, + appOptions, + 100, + interBlockCacheOpt(), + baseapp.SetChainID(BandAppChainID), + ) + require.Equal(t, "BandApp", app.Name()) + + fmt.Printf( + "running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n", + config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, + ) + + _, _, err := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), NewDefaultGenesisState()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.ModuleAccountAddrs(), + config, + app.AppCodec(), + ) + require.NoError(t, err) + + if config.Commit { + simtestutil.PrintStats(db) + } + + appHash := app.LastCommitID().Hash + appHashList[j] = appHash + + if j != 0 { + require.Equal( + t, + string(appHashList[0]), + string(appHashList[j]), + "non-determinism in seed %d: %d/%d, attempt: %d/%d\n", + config.Seed, + i+1, + numSeeds, + j+1, + numTimesToRunPerSeed, + ) + } + } + } +} diff --git a/x/oracle/simulation/operations.go b/x/oracle/simulation/operations.go index 791c16c61..05b84080f 100644 --- a/x/oracle/simulation/operations.go +++ b/x/oracle/simulation/operations.go @@ -1,7 +1,6 @@ package simulation import ( - "fmt" "math/rand" "github.com/cosmos/cosmos-sdk/baseapp" @@ -227,8 +226,6 @@ func SimulateMsgReportData( for i := uint64(1); i <= rCount; i++ { req, _ := keeper.GetRequest(ctx, types.RequestID(i)) - fmt.Printf("req %+v\n", req) - for _, val := range req.RequestedValidators { valAddr, _ := sdk.ValAddressFromBech32(val) acc, ok := simtypes.FindAccount(accs, sdk.AccAddress(valAddr)) @@ -246,7 +243,6 @@ func SimulateMsgReportData( } if rid == 0 { - fmt.Printf("as") return simtypes.NoOpMsg( types.ModuleName, types.MsgReportData{}.Type(),