Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

R4R: Display number of operations which actually got ran #2683

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions PENDING.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ IMPROVEMENTS
- \#2660 [x/mock/simulation] Staking transactions get tested far more frequently
- \#2610 [x/stake] Block redelegation to and from the same validator
- \#2652 [x/auth] Add benchmark for get and set account
- \#2683 [x/mock/simulation] Display event success/failure

* Tendermint

Expand Down
9 changes: 4 additions & 5 deletions x/auth/simulation/fake.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package simulation

import (
"errors"
"fmt"
"math/big"
"math/rand"

Expand All @@ -15,22 +14,22 @@ import (
// SimulateDeductFee
func SimulateDeductFee(m auth.AccountKeeper, f auth.FeeCollectionKeeper) simulation.Operation {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accs []simulation.Account, event func(string)) (
accs []simulation.Account, event simulation.EventFn) (
action string, fOp []simulation.FutureOperation, err error) {

account := simulation.RandomAcc(r, accs)
stored := m.GetAccount(ctx, account.Address)
initCoins := stored.GetCoins()

if len(initCoins) == 0 {
event(fmt.Sprintf("auth/SimulateDeductFee/false"))
event("auth/SimulateDeductFee", false)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️ this

return action, nil, nil
}

denomIndex := r.Intn(len(initCoins))
amt, err := randPositiveInt(r, initCoins[denomIndex].Amount)
if err != nil {
event(fmt.Sprintf("auth/SimulateDeductFee/false"))
event("auth/SimulateDeductFee", false)
return action, nil, nil
}

Expand All @@ -46,7 +45,7 @@ func SimulateDeductFee(m auth.AccountKeeper, f auth.FeeCollectionKeeper) simulat

f.AddCollectedFees(ctx, coins)

event(fmt.Sprintf("auth/SimulateDeductFee/true"))
event("auth/SimulateDeductFee", true)

action = "TestDeductFee"
return action, nil, nil
Expand Down
8 changes: 4 additions & 4 deletions x/bank/simulation/msgs.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
// SingleInputSendTx tests and runs a single msg send w/ auth, with one input and one output, where both
// accounts already exist.
func SingleInputSendTx(mapper auth.AccountKeeper) simulation.Operation {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event func(string)) (action string, fOps []simulation.FutureOperation, err error) {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event simulation.EventFn) (action string, fOps []simulation.FutureOperation, err error) {
fromAcc, action, msg, abort := createSingleInputSendMsg(r, ctx, accs, mapper)
if abort {
return action, nil, nil
Expand All @@ -27,7 +27,7 @@ func SingleInputSendTx(mapper auth.AccountKeeper) simulation.Operation {
if err != nil {
return "", nil, err
}
event("bank/sendAndVerifyTxSend/ok")
event("bank/sendAndVerifyTxSend", true)

return action, nil, nil
}
Expand All @@ -37,7 +37,7 @@ func SingleInputSendTx(mapper auth.AccountKeeper) simulation.Operation {
// accounts already exist.
func SingleInputSendMsg(mapper auth.AccountKeeper, bk bank.Keeper) simulation.Operation {
handler := bank.NewHandler(bk)
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event func(string)) (action string, fOps []simulation.FutureOperation, err error) {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event simulation.EventFn) (action string, fOps []simulation.FutureOperation, err error) {
fromAcc, action, msg, abort := createSingleInputSendMsg(r, ctx, accs, mapper)
if abort {
return action, nil, nil
Expand All @@ -46,7 +46,7 @@ func SingleInputSendMsg(mapper auth.AccountKeeper, bk bank.Keeper) simulation.Op
if err != nil {
return "", nil, err
}
event("bank/sendAndVerifyMsgSend/ok")
event("bank/sendAndVerifyMsgSend", true)

return action, nil, nil
}
Expand Down
16 changes: 8 additions & 8 deletions x/distribution/simulation/msgs.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
handler := distribution.NewHandler(k)
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accs []simulation.Account, event func(string)) (
accs []simulation.Account, event simulation.EventFn) (
action string, fOp []simulation.FutureOperation, err error) {

accountOrigin := simulation.RandomAcc(r, accs)
Expand All @@ -32,7 +32,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper)
write()
}

event(fmt.Sprintf("distribution/MsgSetWithdrawAddress/%v", result.IsOK()))
event("distribution/MsgSetWithdrawAddress", result.IsOK())

action = fmt.Sprintf("TestMsgSetWithdrawAddress: ok %v, msg %s", result.IsOK(), msg.GetSignBytes())
return action, nil, nil
Expand All @@ -43,7 +43,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper)
func SimulateMsgWithdrawDelegatorRewardsAll(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
handler := distribution.NewHandler(k)
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accs []simulation.Account, event func(string)) (
accs []simulation.Account, event simulation.EventFn) (
action string, fOp []simulation.FutureOperation, err error) {

account := simulation.RandomAcc(r, accs)
Expand All @@ -59,7 +59,7 @@ func SimulateMsgWithdrawDelegatorRewardsAll(m auth.AccountKeeper, k distribution
write()
}

event(fmt.Sprintf("distribution/MsgWithdrawDelegatorRewardsAll/%v", result.IsOK()))
event("distribution/MsgWithdrawDelegatorRewardsAll", result.IsOK())

action = fmt.Sprintf("TestMsgWithdrawDelegatorRewardsAll: ok %v, msg %s", result.IsOK(), msg.GetSignBytes())
return action, nil, nil
Expand All @@ -70,7 +70,7 @@ func SimulateMsgWithdrawDelegatorRewardsAll(m auth.AccountKeeper, k distribution
func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
handler := distribution.NewHandler(k)
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accs []simulation.Account, event func(string)) (
accs []simulation.Account, event simulation.EventFn) (
action string, fOp []simulation.FutureOperation, err error) {

delegatorAccount := simulation.RandomAcc(r, accs)
Expand All @@ -87,7 +87,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee
write()
}

event(fmt.Sprintf("distribution/MsgWithdrawDelegatorReward/%v", result.IsOK()))
event("distribution/MsgWithdrawDelegatorReward", result.IsOK())

action = fmt.Sprintf("TestMsgWithdrawDelegatorReward: ok %v, msg %s", result.IsOK(), msg.GetSignBytes())
return action, nil, nil
Expand All @@ -98,7 +98,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee
func SimulateMsgWithdrawValidatorRewardsAll(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
handler := distribution.NewHandler(k)
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accs []simulation.Account, event func(string)) (
accs []simulation.Account, event simulation.EventFn) (
action string, fOp []simulation.FutureOperation, err error) {

account := simulation.RandomAcc(r, accs)
Expand All @@ -114,7 +114,7 @@ func SimulateMsgWithdrawValidatorRewardsAll(m auth.AccountKeeper, k distribution
write()
}

event(fmt.Sprintf("distribution/MsgWithdrawValidatorRewardsAll/%v", result.IsOK()))
event("distribution/MsgWithdrawValidatorRewardsAll/", result.IsOK())

action = fmt.Sprintf("TestMsgWithdrawValidatorRewardsAll: ok %v, msg %s", result.IsOK(), msg.GetSignBytes())
return action, nil, nil
Expand Down
16 changes: 8 additions & 8 deletions x/gov/simulation/msgs.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, sk stake.Keepe
})
statePercentageArray := []float64{1, .9, .75, .4, .15, 0}
curNumVotesState := 1
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event func(string)) (action string, fOps []simulation.FutureOperation, err error) {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event simulation.EventFn) (action string, fOps []simulation.FutureOperation, err error) {
// 1) submit proposal now
sender := simulation.RandomAcc(r, accs)
msg, err := simulationCreateMsgSubmitProposal(r, sender)
Expand Down Expand Up @@ -82,7 +82,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, sk stake.Keepe
// Note: Currently doesn't ensure that the proposal txt is in JSON form
func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operation {
handler := gov.NewHandler(k)
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event func(string)) (action string, fOps []simulation.FutureOperation, err error) {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event simulation.EventFn) (action string, fOps []simulation.FutureOperation, err error) {
sender := simulation.RandomAcc(r, accs)
msg, err := simulationCreateMsgSubmitProposal(r, sender)
if err != nil {
Expand All @@ -93,7 +93,7 @@ func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operati
}
}

func simulateHandleMsgSubmitProposal(msg gov.MsgSubmitProposal, sk stake.Keeper, handler sdk.Handler, ctx sdk.Context, event func(string)) (action string, ok bool) {
func simulateHandleMsgSubmitProposal(msg gov.MsgSubmitProposal, sk stake.Keeper, handler sdk.Handler, ctx sdk.Context, event simulation.EventFn) (action string, ok bool) {
ctx, write := ctx.CacheContext()
result := handler(ctx, msg)
ok = result.IsOK()
Expand All @@ -104,7 +104,7 @@ func simulateHandleMsgSubmitProposal(msg gov.MsgSubmitProposal, sk stake.Keeper,
sk.SetPool(ctx, pool)
write()
}
event(fmt.Sprintf("gov/MsgSubmitProposal/%v", ok))
event("gov/MsgSubmitProposal", ok)
action = fmt.Sprintf("TestMsgSubmitProposal: ok %v, msg %s", ok, msg.GetSignBytes())
return
}
Expand All @@ -126,7 +126,7 @@ func simulationCreateMsgSubmitProposal(r *rand.Rand, sender simulation.Account)

// SimulateMsgDeposit
func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event func(string)) (action string, fOp []simulation.FutureOperation, err error) {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event simulation.EventFn) (action string, fOp []simulation.FutureOperation, err error) {
acc := simulation.RandomAcc(r, accs)
proposalID, ok := randomProposalID(r, k, ctx)
if !ok {
Expand All @@ -146,7 +146,7 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
sk.SetPool(ctx, pool)
write()
}
event(fmt.Sprintf("gov/MsgDeposit/%v", result.IsOK()))
event("gov/MsgDeposit", result.IsOK())
action = fmt.Sprintf("TestMsgDeposit: ok %v, msg %s", result.IsOK(), msg.GetSignBytes())
return action, nil, nil
}
Expand All @@ -160,7 +160,7 @@ func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation {

// nolint: unparam
func operationSimulateMsgVote(k gov.Keeper, sk stake.Keeper, acc simulation.Account, proposalID int64) simulation.Operation {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event func(string)) (action string, fOp []simulation.FutureOperation, err error) {
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account, event simulation.EventFn) (action string, fOp []simulation.FutureOperation, err error) {
if acc.Equals(simulation.Account{}) {
acc = simulation.RandomAcc(r, accs)
}
Expand All @@ -186,7 +186,7 @@ func operationSimulateMsgVote(k gov.Keeper, sk stake.Keeper, acc simulation.Acco
write()
}

event(fmt.Sprintf("gov/MsgVote/%v", result.IsOK()))
event("gov/MsgVote", result.IsOK())
action = fmt.Sprintf("TestMsgVote: ok %v, msg %s", result.IsOK(), msg.GetSignBytes())
return action, nil, nil
}
Expand Down
44 changes: 20 additions & 24 deletions x/mock/simulation/random_simulate_blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
accs := RandomAccounts(r, params.NumKeys)

// Setup event stats
events := make(map[string]uint)
event := func(what string) {
events[what]++
events := make(map[event]uint)
eventFn := func(what string, success bool) {
events[event{what, success}]++
}

validators := initChain(r, params, accs, setups, app, appStateFn)
Expand All @@ -100,7 +100,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
var pastTimes []time.Time
var pastVoteInfos [][]abci.VoteInfo

request := RandomRequestBeginBlock(r, params, validators, pastTimes, pastVoteInfos, event, header)
request := RandomRequestBeginBlock(r, params, validators, pastTimes, pastVoteInfos, eventFn, header)
// These are operations which have been queued by previous operations
operationQueue := make(map[int][]Operation)
timeOperationQueue := []FutureOperation{}
Expand All @@ -110,7 +110,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
blockLogBuilders = make([]*strings.Builder, numBlocks)
}
displayLogs := logPrinter(testingMode, blockLogBuilders)
blockSimulator := createBlockSimulator(testingMode, tb, t, params, event, invariants, ops, operationQueue, timeOperationQueue, numBlocks, blockSize, displayLogs)
blockSimulator := createBlockSimulator(testingMode, tb, t, params, eventFn, invariants, ops, operationQueue, timeOperationQueue, numBlocks, blockSize, displayLogs)
if !testingMode {
b.ResetTimer()
} else {
Expand Down Expand Up @@ -147,8 +147,8 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,

// Run queued operations. Ignores blocksize if blocksize is too small
logWriter("Queued operations")
numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, accs, logWriter, displayLogs, event)
numQueuedTimeOpsRan := runQueuedTimeOperations(timeOperationQueue, header.Time, tb, r, app, ctx, accs, logWriter, displayLogs, event)
numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, accs, logWriter, displayLogs, eventFn)
numQueuedTimeOpsRan := runQueuedTimeOperations(timeOperationQueue, header.Time, tb, r, app, ctx, accs, logWriter, displayLogs, eventFn)
if testingMode && onOperation {
// Make sure invariants hold at end of queued operations
assertAllInvariants(t, app, invariants, "QueuedOperations", displayLogs)
Expand Down Expand Up @@ -183,11 +183,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
}

// Generate a random RequestBeginBlock with the current validator set for the next block
request = RandomRequestBeginBlock(r, params, validators, pastTimes, pastVoteInfos, event, header)
request = RandomRequestBeginBlock(r, params, validators, pastTimes, pastVoteInfos, eventFn, header)

// Update the validator set, which will be reflected in the application on the next block
validators = nextValidators
nextValidators = updateValidators(tb, r, params, validators, res.ValidatorUpdates, event)
nextValidators = updateValidators(tb, r, params, validators, res.ValidatorUpdates, eventFn)
}
if stopEarly {
DisplayEvents(events)
Expand All @@ -206,7 +206,7 @@ type blockSimFn func(
// Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize
// memory overhead
func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params Params,
event func(string), invariants []Invariant,
eventFn EventFn, invariants []Invariant,
ops []WeightedOperation, operationQueue map[int][]Operation, timeOperationQueue []FutureOperation,
totalNumBlocks int, avgBlockSize int, displayLogs func()) blockSimFn {

Expand Down Expand Up @@ -236,7 +236,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize)
lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize)
for j := 0; j < blocksize; j++ {
logUpdate, futureOps, err := selectOp(r)(r, app, ctx, accounts, event)
logUpdate, futureOps, err := selectOp(r)(r, app, ctx, accounts, eventFn)
logWriter(logUpdate)
if err != nil {
displayLogs()
Expand Down Expand Up @@ -313,7 +313,7 @@ func queueOperations(queuedOperations map[int][]Operation, queuedTimeOperations

// nolint: errcheck
func runQueuedOperations(queueOperations map[int][]Operation, height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) {
accounts []Account, logWriter func(string), displayLogs func(), event func(string, bool)) (numOpsRan int) {
if queuedOps, ok := queueOperations[height]; ok {
numOps := len(queuedOps)
for i := 0; i < numOps; i++ {
Expand All @@ -334,7 +334,7 @@ func runQueuedOperations(queueOperations map[int][]Operation, height int, tb tes
}

func runQueuedTimeOperations(queueOperations []FutureOperation, currentTime time.Time, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) {
accounts []Account, logWriter func(string), displayLogs func(), event EventFn) (numOpsRan int) {

numOpsRan = 0
for len(queueOperations) > 0 && currentTime.After(queueOperations[0].BlockTime) {
Expand Down Expand Up @@ -382,7 +382,7 @@ func randomProposer(r *rand.Rand, validators map[string]mockValidator) cmn.HexBy
// RandomRequestBeginBlock generates a list of signing validators according to the provided list of validators, signing fraction, and evidence fraction
// nolint: unparam
func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string]mockValidator,
pastTimes []time.Time, pastVoteInfos [][]abci.VoteInfo, event func(string), header abci.Header) abci.RequestBeginBlock {
pastTimes []time.Time, pastVoteInfos [][]abci.VoteInfo, eventFn EventFn, header abci.Header) abci.RequestBeginBlock {
if len(validators) == 0 {
return abci.RequestBeginBlock{Header: header}
}
Expand All @@ -402,11 +402,7 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string]
// offline
signed = false
}
if signed {
event("beginblock/signing/signed")
} else {
event("beginblock/signing/missed")
}
eventFn("beginblock/signing/signed", signed)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this a useful stat? I really don't think it is, since the information it provides is unhelpful after a very small number of blocks.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's still worth recording for an idea of the overall distribution.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm dubious. I'd opt to remove just b/c it also skews the "number of operations" count, though I guess we could create a fix for this.

This is essentially looking at the result of a binomial distribution after a very large number of runs. Maybe we make this one display the percentage between the two, and not the raw number?

pubkey, err := tmtypes.PB2TM.PubKey(mVal.val.PubKey)
if err != nil {
panic(err)
Expand Down Expand Up @@ -445,7 +441,7 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string]
Time: time,
TotalVotingPower: totalVotingPower,
})
event("beginblock/evidence")
eventFn("beginblock/evidence", true)
}
}
return abci.RequestBeginBlock{
Expand All @@ -459,7 +455,7 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string]

// updateValidators mimicks Tendermint's update logic
// nolint: unparam
func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[string]mockValidator, updates []abci.ValidatorUpdate, event func(string)) map[string]mockValidator {
func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[string]mockValidator, updates []abci.ValidatorUpdate, eventFn EventFn) map[string]mockValidator {

for _, update := range updates {
str := fmt.Sprintf("%v", update.PubKey)
Expand All @@ -469,17 +465,17 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[st
tb.Fatalf("tried to delete a nonexistent validator")
}

event("endblock/validatorupdates/kicked")
eventFn("endblock/validatorupdates/kicked", true)
delete(current, str)
default:
// Does validator already exist?
if mVal, ok := current[str]; ok {
mVal.val = update
event("endblock/validatorupdates/updated")
eventFn("endblock/validatorupdates/updated", true)
} else {
// Set this new validator
current[str] = mockValidator{update, GetMemberOfInitialState(r, params.InitialLivenessWeightings)}
event("endblock/validatorupdates/added")
eventFn("endblock/validatorupdates/added", true)
}
}
}
Expand Down
Loading