From 2cd14f0bacd394162cd2cdf7281d667fa85a582c Mon Sep 17 00:00:00 2001 From: Philip Offtermatt <57488781+p-offtermatt@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:08:30 +0200 Subject: [PATCH 01/11] Add primary branches to build on PR --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b74b94b..dc8919c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,7 +13,7 @@ on: # Publish semver tags as releases. tags: [ 'v*.*.*' ] pull_request: - branches: [ "main" ] + branches: [ "main", "v0.37.x", "v0.34.x" ] env: # Use docker.io for Docker Hub if empty From 73f76bac809064d968e188e6fde980e24da95832 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 21 Aug 2023 13:52:35 +0200 Subject: [PATCH 02/11] Add mutex around state updates --- cometmock/abci_client/client.go | 29 ++++----- cometmock/storage/storage.go | 101 +++++++++++++++++--------------- 2 files changed, 64 insertions(+), 66 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 81d71bc..90a1f16 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -28,6 +28,9 @@ var GlobalClient *AbciClient // store a mutex that allows only running one block at a time var blockMutex = sync.Mutex{} +// store a mutex that disallows accessing state information while it is being updated +var stateUpdateMutex = sync.Mutex{} + var verbose = false // AbciClient facilitates calls to the ABCI interface of multiple nodes. @@ -716,8 +719,6 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( return nil, nil, nil, nil, nil, err } - a.LastBlock = block - commitSigs := []types.CommitSig{} for index, val := range a.CurState.Validators.Validators { @@ -813,25 +814,14 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( deliverTxResponses = append(deliverTxResponses, resDeliverTx) } - // insert entries into the storage - err = a.Storage.InsertBlock(newHeight, block) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - err = a.Storage.InsertCommit(newHeight, a.LastCommit) - if err != nil { - return nil, nil, nil, nil, nil, err - } + // lock the state update mutex while the stores are updated to avoid + // inconsistencies between stores + a.Storage.LockBeforeStateUpdate() + a.LastBlock = block // copy state so that the historical state is not mutated state := a.CurState.Copy() - err = a.Storage.InsertState(newHeight, &state) - if err != nil { - return nil, nil, nil, nil, nil, err - } - // build components of the state update, then call the update function abciResponses := cmtstate.ABCIResponses{ DeliverTxs: deliverTxResponses, @@ -839,7 +829,8 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( BeginBlock: resBeginBlock, } - err = a.Storage.InsertResponses(newHeight, &abciResponses) + // insert entries into the storage + err = a.Storage.UpdateStores(newHeight, block, a.LastCommit, &state, &abciResponses) if err != nil { return nil, nil, nil, nil, nil, err } @@ -849,6 +840,8 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( if err != nil { return nil, nil, nil, nil, nil, err } + // unlock the state mutex, since we are done updating state + a.Storage.UnlockAfterStateUpdate() resCommit, err := a.SendCommit() if err != nil { diff --git a/cometmock/storage/storage.go b/cometmock/storage/storage.go index 2151d22..3e62c7b 100644 --- a/cometmock/storage/storage.go +++ b/cometmock/storage/storage.go @@ -12,51 +12,51 @@ import ( // Storage is an interface for storing blocks, commits and states by height. // All methods are thread-safe. type Storage interface { - // InsertBlock inserts a block at a given height. - // If there is already a block at that height, it should be overwritten. - InsertBlock(height int64, block *types.Block) error // GetBlock returns the block at a given height. GetBlock(height int64) (*types.Block, error) - // InsertCommit inserts a commit at a given height. - // If there is already a commit at that height, it should be overwritten. - InsertCommit(height int64, commit *types.Commit) error // GetCommit returns the commit at a given height. GetCommit(height int64) (*types.Commit, error) - // InsertState inserts a state at a given height. This is the state after - // applying the block at that height. - // If there is already a state at that height, it should be overwritten. - InsertState(height int64, state *cometstate.State) error // GetState returns the state at a given height. This is the state after // applying the block at that height. GetState(height int64) (*cometstate.State, error) - // InsertResponses inserts the ABCI responses from a given height. - // If there are already responses at that height, they should be overwritten. - InsertResponses(height int64, responses *protostate.ABCIResponses) error // GetResponses returns the ABCI responses from a given height. GetResponses(height int64) (*protostate.ABCIResponses, error) + + // LockBeforeStateUpdate locks the storage for state update. + LockBeforeStateUpdate() + + // UnlockAfterStateUpdate unlocks the storage for state update. + UnlockAfterStateUpdate() + + // UpdateStores updates the storage with the given block, commit, state and responses. + // It is assumed that the block, commit, state and responses are all from the same height. + // If they are not, the storage will be in an inconsistent state. + // If the storage is already updated with the given height, the storage will overwrite the existing data. + // This method is *not* thread-safe. + // Before calling this, the caller should call LockForStateUpdate(). + // After calling this, the caller should call UnlockForStateUpdate(). + UpdateStores(height int64, block *types.Block, commit *types.Commit, state *cometstate.State, responses *protostate.ABCIResponses) error } // MapStorage is a simple in-memory implementation of Storage. type MapStorage struct { - blocks map[int64]*types.Block - blocksMutex sync.RWMutex - commits map[int64]*types.Commit - commitMutex sync.RWMutex - states map[int64]*cometstate.State - statesMutex sync.RWMutex - responses map[int64]*protostate.ABCIResponses - responsesMutex sync.RWMutex + // a mutex that gets locked while the state is being updated, + // so that a) updates do not interleave and b) reads do not happen while + // the state is being updated, i.e. two stores might give bogus data. + stateUpdateMutex sync.RWMutex + blocks map[int64]*types.Block + commits map[int64]*types.Commit + states map[int64]*cometstate.State + responses map[int64]*protostate.ABCIResponses } // ensure MapStorage implements Storage var _ Storage = (*MapStorage)(nil) -func (m *MapStorage) InsertBlock(height int64, block *types.Block) error { - m.blocksMutex.Lock() - defer m.blocksMutex.Unlock() +func (m *MapStorage) insertBlock(height int64, block *types.Block) error { if m.blocks == nil { m.blocks = make(map[int64]*types.Block) } @@ -65,9 +65,8 @@ func (m *MapStorage) InsertBlock(height int64, block *types.Block) error { } func (m *MapStorage) GetBlock(height int64) (*types.Block, error) { - m.blocksMutex.RLock() - defer m.blocksMutex.RUnlock() - + m.stateUpdateMutex.RLock() + defer m.stateUpdateMutex.RUnlock() if m.blocks == nil { m.blocks = make(map[int64]*types.Block) } @@ -77,10 +76,7 @@ func (m *MapStorage) GetBlock(height int64) (*types.Block, error) { return nil, fmt.Errorf("block for height %v not found", height) } -func (m *MapStorage) InsertCommit(height int64, commit *types.Commit) error { - m.commitMutex.Lock() - defer m.commitMutex.Unlock() - +func (m *MapStorage) insertCommit(height int64, commit *types.Commit) error { if m.commits == nil { m.commits = make(map[int64]*types.Commit) } @@ -90,9 +86,8 @@ func (m *MapStorage) InsertCommit(height int64, commit *types.Commit) error { } func (m *MapStorage) GetCommit(height int64) (*types.Commit, error) { - m.commitMutex.RLock() - defer m.commitMutex.RUnlock() - + m.stateUpdateMutex.RLock() + defer m.stateUpdateMutex.RUnlock() if m.commits == nil { m.commits = make(map[int64]*types.Commit) } @@ -103,10 +98,7 @@ func (m *MapStorage) GetCommit(height int64) (*types.Commit, error) { return nil, fmt.Errorf("commit for height %v not found", height) } -func (m *MapStorage) InsertState(height int64, state *cometstate.State) error { - m.statesMutex.Lock() - defer m.statesMutex.Unlock() - +func (m *MapStorage) insertState(height int64, state *cometstate.State) error { if m.states == nil { m.states = make(map[int64]*cometstate.State) } @@ -116,9 +108,8 @@ func (m *MapStorage) InsertState(height int64, state *cometstate.State) error { } func (m *MapStorage) GetState(height int64) (*cometstate.State, error) { - m.statesMutex.RLock() - defer m.statesMutex.RUnlock() - + m.stateUpdateMutex.RLock() + defer m.stateUpdateMutex.RUnlock() if m.states == nil { m.states = make(map[int64]*cometstate.State) } @@ -129,10 +120,7 @@ func (m *MapStorage) GetState(height int64) (*cometstate.State, error) { return nil, fmt.Errorf("state for height %v not found", height) } -func (m *MapStorage) InsertResponses(height int64, responses *protostate.ABCIResponses) error { - m.responsesMutex.Lock() - defer m.responsesMutex.Unlock() - +func (m *MapStorage) insertResponses(height int64, responses *protostate.ABCIResponses) error { if m.responses == nil { m.responses = make(map[int64]*protostate.ABCIResponses) } @@ -142,9 +130,8 @@ func (m *MapStorage) InsertResponses(height int64, responses *protostate.ABCIRes } func (m *MapStorage) GetResponses(height int64) (*protostate.ABCIResponses, error) { - m.responsesMutex.RLock() - defer m.responsesMutex.RUnlock() - + m.stateUpdateMutex.RLock() + defer m.stateUpdateMutex.RUnlock() if m.responses == nil { m.responses = make(map[int64]*protostate.ABCIResponses) } @@ -154,3 +141,21 @@ func (m *MapStorage) GetResponses(height int64) (*protostate.ABCIResponses, erro } return nil, fmt.Errorf("responses for height %v not found", height) } + +func (m *MapStorage) LockBeforeStateUpdate() { + m.stateUpdateMutex.Lock() +} + +func (m *MapStorage) UnlockAfterStateUpdate() { + m.stateUpdateMutex.Unlock() +} + +func (m *MapStorage) UpdateStores(height int64, block *types.Block, commit *types.Commit, state *cometstate.State, responses *protostate.ABCIResponses) error { + m.stateUpdateMutex.Lock() + defer m.stateUpdateMutex.Unlock() + m.insertBlock(height, block) + m.insertCommit(height, commit) + m.insertState(height, state) + m.insertResponses(height, responses) + return nil +} From a7cd9ab9491e375fbf28b3b44f58c40303a9589c Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 21 Aug 2023 14:11:39 +0200 Subject: [PATCH 03/11] Fix duplicate mutex lock --- cometmock/storage/storage.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cometmock/storage/storage.go b/cometmock/storage/storage.go index 3e62c7b..e3dfb9f 100644 --- a/cometmock/storage/storage.go +++ b/cometmock/storage/storage.go @@ -151,8 +151,6 @@ func (m *MapStorage) UnlockAfterStateUpdate() { } func (m *MapStorage) UpdateStores(height int64, block *types.Block, commit *types.Commit, state *cometstate.State, responses *protostate.ABCIResponses) error { - m.stateUpdateMutex.Lock() - defer m.stateUpdateMutex.Unlock() m.insertBlock(height, block) m.insertCommit(height, commit) m.insertState(height, state) From 22fb5d3714e009043f35764fbe15d1f696148aff Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 21 Aug 2023 17:30:54 +0200 Subject: [PATCH 04/11] Remove queue from mergify config (cherry picked from commit 2a59a3e478129ce89577d890eb531016e5b83c28) # Conflicts: # .mergify.yml --- .mergify.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 0000000..c3d7575 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,23 @@ +defaults: + actions: + backport: + assignees: + - "{{ author }}" + +pull_request_rules: + - name: Backport patches to the v0.37.x branch + conditions: + - base=main + - label=A:backport/v0.37.x + actions: + backport: + branches: + - release/v0.37.x + - name: Backport patches to the v0.34.x branch + conditions: + - base=main + - label=A:backport/v0.34.x + actions: + backport: + branches: + - release/v0.34.x From 09b964bdb8c17d75d9ce1f705880135fd90b7bb0 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 21 Aug 2023 17:32:47 +0200 Subject: [PATCH 05/11] Fix branch name in mergify config (cherry picked from commit 437a21e82426e229fe135af3bee2820e60aeb81f) --- .mergify.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index c3d7575..5817d97 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -12,7 +12,7 @@ pull_request_rules: actions: backport: branches: - - release/v0.37.x + - v0.37.x - name: Backport patches to the v0.34.x branch conditions: - base=main @@ -20,4 +20,4 @@ pull_request_rules: actions: backport: branches: - - release/v0.34.x + - v0.34.x From 5a8673623ce69b037d1e69a63843c55598a4634b Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 30 Aug 2023 15:07:19 +0200 Subject: [PATCH 06/11] Add light client evidence generation (cherry picked from commit f0a488e2bc96172886f13a7303ac668610b86af7) --- cometmock/abci_client/client.go | 219 +++++++++++++++++++++----------- cometmock/rpc_server/routes.go | 16 ++- go.mod | 1 + go.sum | 2 + 4 files changed, 163 insertions(+), 75 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 90a1f16..99ead85 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/barkimedes/go-deepcopy" db "github.com/cometbft/cometbft-db" abcitypes "github.com/cometbft/cometbft/abci/types" cryptoenc "github.com/cometbft/cometbft/crypto/encoding" @@ -33,6 +34,15 @@ var stateUpdateMutex = sync.Mutex{} var verbose = false +type MisbehaviourType int + +const ( + DuplicateVote MisbehaviourType = iota + Lunatic + Amnesia + Equivocation +) + // AbciClient facilitates calls to the ABCI interface of multiple nodes. // It also tracks the current state and a common logger. type AbciClient struct { @@ -582,15 +592,138 @@ func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { // RunBlock runs a block with a specified transaction through the ABCI application. // It calls RunBlockWithTimeAndProposer with the current time and the LastValidators.Proposer. func (a *AbciClient) RunBlock(tx *[]byte) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { - return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer, make([]*types.Validator, 0)) + return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer, make(map[*types.Validator]MisbehaviourType, 0)) } // RunBlockWithEvidence runs a block with a specified transaction through the ABCI application. -// It also produces duplicate vote evidence for the specified misbehaving validators. -func (a *AbciClient) RunBlockWithEvidence(tx *[]byte, misbehavingValidators []*types.Validator) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { +// It also produces the specified evidence for the specified misbehaving validators. +func (a *AbciClient) RunBlockWithEvidence(tx *[]byte, misbehavingValidators map[*types.Validator]MisbehaviourType) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer, misbehavingValidators) } +func (a *AbciClient) ConstructDuplicateVoteEvidence(v *types.Validator) (*types.DuplicateVoteEvidence, error) { + privVal := a.PrivValidators[v.Address.String()] + lastBlock := a.LastBlock + blockId, err := utils.GetBlockIdFromBlock(lastBlock) + if err != nil { + return nil, err + } + + lastState, err := a.Storage.GetState(lastBlock.Height) + if err != nil { + return nil, err + } + + // get the index of the validator in the last state + index, valInLastState := lastState.Validators.GetByAddress(v.Address) + + // produce vote A. + voteA := &cmttypes.Vote{ + ValidatorAddress: v.Address, + ValidatorIndex: int32(index), + Height: lastBlock.Height, + Round: 1, + Timestamp: time.Now().Add(a.timeOffset), + Type: cmttypes.PrecommitType, + BlockID: blockId.ToProto(), + } + + // produce vote B, which just has a different round. + voteB := &cmttypes.Vote{ + ValidatorAddress: v.Address, + ValidatorIndex: int32(index), + Height: lastBlock.Height, + Round: 2, // this is what differentiates the votes + Timestamp: time.Now().Add(a.timeOffset), + Type: cmttypes.PrecommitType, + BlockID: blockId.ToProto(), + } + + // sign the votes + privVal.SignVote(a.CurState.ChainID, voteA) + privVal.SignVote(a.CurState.ChainID, voteB) + + // votes need to pass validation rules + convertedVoteA, err := types.VoteFromProto(voteA) + err = convertedVoteA.ValidateBasic() + if err != nil { + a.Logger.Error("Error validating vote A", "error", err) + return nil, err + } + + convertedVoteB, err := types.VoteFromProto(voteB) + err = convertedVoteB.ValidateBasic() + if err != nil { + a.Logger.Error("Error validating vote B", "error", err) + return nil, err + } + + // build the actual evidence + evidence := types.DuplicateVoteEvidence{ + VoteA: convertedVoteA, + VoteB: convertedVoteB, + + TotalVotingPower: lastState.Validators.TotalVotingPower(), + ValidatorPower: valInLastState.VotingPower, + Timestamp: lastBlock.Time, + } + return &evidence, nil +} + +func (a *AbciClient) ConstructLightClientAttackEvidence( + v *types.Validator, + misbehaviourType MisbehaviourType, +) (*types.LightClientAttackEvidence, error) { + lastBlock := a.LastBlock + + lastState, err := a.Storage.GetState(lastBlock.Height) + if err != nil { + return nil, err + } + + // deepcopy the last block so we can modify it + cp, err := deepcopy.Anything(lastBlock) + if err != nil { + return nil, err + } + + // force the type conversion into a block + var conflictingBlock *types.Block + conflictingBlock = cp.(*types.Block) + + switch misbehaviourType { + case Lunatic: + // modify the app hash to be invalid + conflictingBlock.AppHash = []byte("some other app hash") + case Amnesia: + // TODO not sure how to handle this yet, just leave the block intact for now + case Equivocation: + // get another valid block by making it have a different time + conflictingBlock.Time = conflictingBlock.Time.Add(1 * time.Second) + default: + return nil, fmt.Errorf("unknown misbehaviour type %v for light client misbehaviour", misbehaviourType) + } + + // make the conflicting block into a light block + signedHeader := types.SignedHeader{ + Header: &conflictingBlock.Header, + Commit: a.LastCommit, + } + + conflictingLightBlock := types.LightBlock{ + SignedHeader: &signedHeader, + ValidatorSet: a.CurState.Validators, + } + + return &types.LightClientAttackEvidence{ + TotalVotingPower: lastState.Validators.TotalVotingPower(), + Timestamp: lastBlock.Time, + ByzantineValidators: []*types.Validator{v}, + CommonHeight: lastBlock.Height - 1, + ConflictingBlock: &conflictingLightBlock, + }, nil +} + // RunBlock runs a block with a specified transaction through the ABCI application. // It calls BeginBlock, DeliverTx, EndBlock, Commit and then // updates the state. @@ -599,7 +732,7 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( tx *[]byte, blockTime time.Time, proposer *types.Validator, - misbehavingValidators []*types.Validator, + misbehavingValidators map[*types.Validator]MisbehaviourType, ) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { // lock mutex to avoid running two blocks at the same time a.Logger.Debug("Locking mutex") @@ -636,79 +769,23 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( } evidences := make([]types.Evidence, 0) - for _, v := range misbehavingValidators { - privVal := a.PrivValidators[v.Address.String()] - // produce evidence of misbehaviour. - - // assemble a duplicate vote evidence for this validator, - // claiming it voted twice on the last block. - - lastBlock := a.LastBlock - blockId, err := utils.GetBlockIdFromBlock(lastBlock) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - lastState, err := a.Storage.GetState(lastBlock.Height) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - // get the index of the validator in the last state - index, valInLastState := lastState.Validators.GetByAddress(v.Address) - - // produce vote A. - voteA := &cmttypes.Vote{ - ValidatorAddress: v.Address, - ValidatorIndex: int32(index), - Height: lastBlock.Height, - Round: 1, - Timestamp: time.Now().Add(a.timeOffset), - Type: cmttypes.PrecommitType, - BlockID: blockId.ToProto(), - } - - // produce vote B, which just has a different round. - voteB := &cmttypes.Vote{ - ValidatorAddress: v.Address, - ValidatorIndex: int32(index), - Height: lastBlock.Height, - Round: 2, // this is what differentiates the votes - Timestamp: time.Now().Add(a.timeOffset), - Type: cmttypes.PrecommitType, - BlockID: blockId.ToProto(), - } - - // sign the votes - privVal.SignVote(a.CurState.ChainID, voteA) - privVal.SignVote(a.CurState.ChainID, voteB) - - // votes need to pass validation rules - convertedVoteA, err := types.VoteFromProto(voteA) - err = convertedVoteA.ValidateBasic() - if err != nil { - a.Logger.Error("Error validating vote A", "error", err) - return nil, nil, nil, nil, nil, err + for v, misbehaviourType := range misbehavingValidators { + // match the misbehaviour type to call the correct function + var evidence types.Evidence + var err error + if misbehaviourType == DuplicateVote { + // create double-sign evidence + evidence, err = a.ConstructDuplicateVoteEvidence(v) + } else { + // create light client attack evidence + evidence, err = a.ConstructLightClientAttackEvidence(v, misbehaviourType) } - convertedVoteB, err := types.VoteFromProto(voteB) - err = convertedVoteB.ValidateBasic() if err != nil { - a.Logger.Error("Error validating vote B", "error", err) return nil, nil, nil, nil, nil, err } - // build the actual evidence - evidence := types.DuplicateVoteEvidence{ - VoteA: convertedVoteA, - VoteB: convertedVoteB, - - TotalVotingPower: lastState.Validators.TotalVotingPower(), - ValidatorPower: valInLastState.VotingPower, - Timestamp: lastBlock.Time, - } - - evidences = append(evidences, &evidence) + evidences = append(evidences, evidence) } block := a.CurState.MakeBlock(a.CurState.LastBlockHeight+1, txs, a.LastCommit, evidences, proposerAddress) diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 96ed401..ca4c30c 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -53,10 +53,18 @@ var Routes = map[string]*rpc.RPCFunc{ "abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,prove"), // cometmock specific API - "advance_blocks": rpc.NewRPCFunc(AdvanceBlocks, "num_blocks"), - "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "private_key_address,status"), - "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration_in_seconds"), - "cause_double_sign": rpc.NewRPCFunc(CauseDoubleSign, "private_key_address"), + "advance_blocks": rpc.NewRPCFunc(AdvanceBlocks, "num_blocks"), + "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "private_key_address,status"), + "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration_in_seconds"), + "cause_double_sign": rpc.NewRPCFunc(CauseDoubleSign, "private_key_address"), + "cause_light_client_attack": rpc.NewRPCFunc(CauseLightClientAttack, "private_key_address"), +} + +type ResultCauseLightClientAttack struct{} + +func CauseLightClientAttack(ctx *rpctypes.Context, privateKeyAddress string) (*ResultCauseLightClientAttack, error) { + err := abci_client.GlobalClient.CauseLightClientAttack(privateKeyAddress) + return &ResultCauseLightClientAttack{}, err } type ResultCauseDoubleSign struct{} diff --git a/go.mod b/go.mod index afa1265..63fe9b6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/informalsystems/CometMock go 1.20 require ( + github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.7.0 ) diff --git a/go.sum b/go.sum index c7a78cd..b27f725 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= +github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From 772cc598059f9afc6d5d912de13227fc85f695ac Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 30 Aug 2023 15:40:58 +0200 Subject: [PATCH 07/11] Update call to light client evidence method (cherry picked from commit 4fe458e7526df6c33f15d0516917ebf4ec368110) --- cometmock/abci_client/client.go | 27 ++++++++++++++++++++++++++- cometmock/rpc_server/routes.go | 6 +++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 99ead85..c020af0 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -88,6 +88,31 @@ func (a *AbciClient) IncrementTimeOffset(additionalOffset time.Duration) error { return nil } +func (a *AbciClient) CauseLightClientAttack(address string, misbehaviourType string) error { + a.Logger.Info("Causing double sign", "address", address) + + validator, err := a.GetValidatorFromAddress(address) + if err != nil { + return err + } + + // get the misbehaviour type from the string + var misbehaviour MisbehaviourType + switch misbehaviourType { + case "Lunatic": + misbehaviour = Lunatic + case "Amnesia": + misbehaviour = Amnesia + case "Equivocation": + misbehaviour = Equivocation + default: + return fmt.Errorf("unknown misbehaviour type %s, possible types are: Equivocation, Lunatic, Amnesia", misbehaviourType) + } + + _, _, _, _, _, err = a.RunBlockWithEvidence(nil, map[*types.Validator]MisbehaviourType{validator: misbehaviour}) + return err +} + func (a *AbciClient) CauseDoubleSign(address string) error { a.Logger.Info("Causing double sign", "address", address) @@ -96,7 +121,7 @@ func (a *AbciClient) CauseDoubleSign(address string) error { return err } - _, _, _, _, _, err = a.RunBlockWithEvidence(nil, []*types.Validator{validator}) + _, _, _, _, _, err = a.RunBlockWithEvidence(nil, map[*types.Validator]MisbehaviourType{validator: DuplicateVote}) return err } diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index ca4c30c..5b83aec 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -57,13 +57,13 @@ var Routes = map[string]*rpc.RPCFunc{ "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "private_key_address,status"), "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration_in_seconds"), "cause_double_sign": rpc.NewRPCFunc(CauseDoubleSign, "private_key_address"), - "cause_light_client_attack": rpc.NewRPCFunc(CauseLightClientAttack, "private_key_address"), + "cause_light_client_attack": rpc.NewRPCFunc(CauseLightClientAttack, "private_key_address, misbehaviour_type"), } type ResultCauseLightClientAttack struct{} -func CauseLightClientAttack(ctx *rpctypes.Context, privateKeyAddress string) (*ResultCauseLightClientAttack, error) { - err := abci_client.GlobalClient.CauseLightClientAttack(privateKeyAddress) +func CauseLightClientAttack(ctx *rpctypes.Context, privateKeyAddress, misbehaviourType string) (*ResultCauseLightClientAttack, error) { + err := abci_client.GlobalClient.CauseLightClientAttack(privateKeyAddress, misbehaviourType) return &ResultCauseLightClientAttack{}, err } From 789937d38cf7eb49481c483641974f44c1cf2645 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 30 Aug 2023 15:47:36 +0200 Subject: [PATCH 08/11] Remove extra comma between arguments for light client attack (cherry picked from commit aa152a60d0e05a9933efc7d2c2850076e8cc1c1b) --- cometmock/rpc_server/routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 5b83aec..f91a6a9 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -57,7 +57,7 @@ var Routes = map[string]*rpc.RPCFunc{ "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "private_key_address,status"), "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration_in_seconds"), "cause_double_sign": rpc.NewRPCFunc(CauseDoubleSign, "private_key_address"), - "cause_light_client_attack": rpc.NewRPCFunc(CauseLightClientAttack, "private_key_address, misbehaviour_type"), + "cause_light_client_attack": rpc.NewRPCFunc(CauseLightClientAttack, "private_key_address,misbehaviour_type"), } type ResultCauseLightClientAttack struct{} From d4dca0ba208821621bdd02b0601a5e583b406da1 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 30 Aug 2023 16:53:13 +0200 Subject: [PATCH 09/11] Add double sign and light client attacks to README (cherry picked from commit bf336c7bf09eaee436cf5316a6b299a1f3055437) --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 0b0a271..88542e7 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,13 @@ Example usage: curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{"jsonrpc":"2.0","method":"advance_time","params":{"duration_in_seconds": "36000000"},"id":1}' 127.0.0.1:22331 ``` +* `cause_double_sign(private_key_address)`: Causes the validator with the given private key to double sign. This is done by signing two blocks with the same height. This will produce DuplicateVoteEvidence and propagate it to the app via ABCI. + +* `cause_light_client_attack(private_key_address, misbehaviour_type)`: Will LightClientAttackEvidence for the validator with the given private key. This will produce evidence in one of three different ways. Misbehaviour type can be: +* Equivocation: The evidence has a conflicting block that has the same height, but a non-deterministic field is different, e.g. time. +* Lunatic: The evidence has a conflicting block that differs in the app hash. +* Amnesia: The evidence has a conflicting block that is the same as the original block. + ## Limitations ### Not all CometBFT RPC endpoints are implemented From 8af4ece778443d0339544c5b55ee7bda7341d265 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 11 Sep 2023 14:35:01 +0200 Subject: [PATCH 10/11] Add testscript that starts a single chain (cherry picked from commit 0fb417959aeacb3afccadb446dec6a0589bb1c81) --- local-testnet-singlechain.sh | 208 +++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100755 local-testnet-singlechain.sh diff --git a/local-testnet-singlechain.sh b/local-testnet-singlechain.sh new file mode 100755 index 0000000..7232a35 --- /dev/null +++ b/local-testnet-singlechain.sh @@ -0,0 +1,208 @@ +#!/bin/bash +set -eux + +BINARY_NAME='simd' + +# User balance of stake tokens +USER_COINS="100000000000stake" +# Amount of stake tokens staked +STAKE="100000000stake" +# Node IP address +NODE_IP="127.0.0.1" + +# Home directory +HOME_DIR=$HOME + +# Validator moniker +MONIKERS=("coordinator" "alice" "bob") +LEAD_VALIDATOR_MONIKER="coordinator" + +PROV_NODES_ROOT_DIR=${HOME_DIR}/nodes/provider +CONS_NODES_ROOT_DIR=${HOME_DIR}/nodes/consumer + +# Base port. Ports assigned after these ports sequentially by nodes. +RPC_LADDR_BASEPORT=29170 +P2P_LADDR_BASEPORT=29180 +GRPC_LADDR_BASEPORT=29190 +NODE_ADDRESS_BASEPORT=29200 +PPROF_LADDR_BASEPORT=29210 +CLIENT_BASEPORT=29220 + +# keeps a comma separated list of node addresses for provider and consumer +PROVIDER_NODE_LISTEN_ADDR_STR="" +CONSUMER_NODE_LISTEN_ADDR_STR="" + +# Strings that keep the homes of provider nodes and homes of consumer nodes +PROV_NODES_HOME_STR="" +CONS_NODES_HOME_STR="" + +PROVIDER_COMETMOCK_ADDR=tcp://$NODE_IP:22331 +CONSUMER_COMETMOCK_ADDR=tcp://$NODE_IP:22332 + +# Clean start +pkill -f $BINARY_NAME &> /dev/null || true +pkill -f cometmock &> /dev/null || true +sleep 1 +rm -rf ${PROV_NODES_ROOT_DIR} +rm -rf ${CONS_NODES_ROOT_DIR} + +# Let lead validator create genesis file +LEAD_VALIDATOR_PROV_DIR=${PROV_NODES_ROOT_DIR}/provider-${LEAD_VALIDATOR_MONIKER} +LEAD_VALIDATOR_CONS_DIR=${CONS_NODES_ROOT_DIR}/consumer-${LEAD_VALIDATOR_MONIKER} +LEAD_PROV_KEY=${LEAD_VALIDATOR_MONIKER}-key +LEAD_PROV_LISTEN_ADDR=tcp://${NODE_IP}:${RPC_LADDR_BASEPORT} + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # Build genesis file and node directory structure + $BINARY_NAME init $MONIKER --chain-id provider --home ${PROV_NODE_DIR} + jq ".app_state.gov.params.voting_period = \"10s\" | .app_state.staking.params.unbonding_time = \"86400s\"" \ + ${PROV_NODE_DIR}/config/genesis.json > \ + ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json + + + sleep 1 + + # Create account keypair + $BINARY_NAME keys add $PROV_KEY --home ${PROV_NODE_DIR} --keyring-backend test --output json > ${PROV_NODE_DIR}/${PROV_KEY}.json 2>&1 + sleep 1 + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + # Add stake to user + PROV_ACCOUNT_ADDR=$(jq -r '.address' ${PROV_NODE_DIR}/${PROV_KEY}.json) + $BINARY_NAME genesis add-genesis-account $PROV_ACCOUNT_ADDR $USER_COINS --home ${PROV_NODE_DIR} --keyring-backend test + sleep 1 + + # copy genesis out, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json + fi + + PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + + # adjust configs of this node + sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${PROV_NODE_DIR}/config/config.toml + sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${PROV_NODE_DIR}/config/config.toml + + # make address book non-strict. necessary for this setup + sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${PROV_NODE_DIR}/config/config.toml + + # avoid port double binding + sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${PROV_NODE_DIR}/config/config.toml + + # allow duplicate IP addresses (all nodes are on the same machine) + sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${PROV_NODE_DIR}/config/config.toml +done + +for MONIKER in "${MONIKERS[@]}" +do + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json* ${PROV_NODE_DIR}/config/genesis.json + fi + + # Stake 1/1000 user's coins + $BINARY_NAME genesis gentx $PROV_KEY $STAKE --chain-id provider --home ${PROV_NODE_DIR} --keyring-backend test --moniker $MONIKER + sleep 1 + + # Copy gentxs to the lead validator for possible future collection. + # Obviously we don't need to copy the first validator's gentx to itself + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/gentx/* ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + fi +done + +# Collect genesis transactions with lead validator +$BINARY_NAME genesis collect-gentxs --home ${LEAD_VALIDATOR_PROV_DIR} --gentx-dir ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + +sleep 1 + + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + + PERSISTENT_PEERS="" + + for peer_index in "${!MONIKERS[@]}" + do + if [ $index == $peer_index ]; then + continue + fi + PEER_MONIKER=${MONIKERS[$peer_index]} + + PEER_PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${PEER_MONIKER} + + PEER_NODE_ID=$($BINARY_NAME tendermint show-node-id --home ${PEER_PROV_NODE_DIR}) + + PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $peer_index)) + PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" + done + + # remove trailing comma from persistent peers + PERSISTENT_PEERS=${PERSISTENT_PEERS:1} + + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${PROV_NODES_ROOT_DIR}/consumer-${MONIKER} + + # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + $index)) + NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + $index)) + + PROVIDER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$PROVIDER_NODE_LISTEN_ADDR_STR" + PROV_NODES_HOME_STR="${PROV_NODE_DIR},$PROV_NODES_HOME_STR" + + # Start gaia + $BINARY_NAME start \ + --home ${PROV_NODE_DIR} \ + --transport=grpc --with-tendermint=false \ + --p2p.persistent_peers ${PERSISTENT_PEERS} \ + --rpc.laddr tcp://${NODE_IP}:${RPC_LADDR_PORT} \ + --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ + --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ + --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ + --grpc-web.enable=false &> ${PROV_NODE_DIR}/logs & + + sleep 5 +done + +PROVIDER_NODE_LISTEN_ADDR_STR=${PROVIDER_NODE_LISTEN_ADDR_STR::${#PROVIDER_NODE_LISTEN_ADDR_STR}-1} +PROV_NODES_HOME_STR=${PROV_NODES_HOME_STR::${#PROV_NODES_HOME_STR}-1} + +echo "Testnet applications are set up! Run the following command to start CometMock:" +echo cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log + +sleep 5 \ No newline at end of file From 7143c76923e0fc5967aac50b840fe02af1331f22 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 11 Sep 2023 16:19:19 +0200 Subject: [PATCH 11/11] Make testnet script take the binary as input (cherry picked from commit e88df6d362542c03e4c96ef62d707c16f38eb936) --- local-testnet-singlechain.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/local-testnet-singlechain.sh b/local-testnet-singlechain.sh index 7232a35..21d2234 100755 --- a/local-testnet-singlechain.sh +++ b/local-testnet-singlechain.sh @@ -1,7 +1,7 @@ #!/bin/bash set -eux -BINARY_NAME='simd' +BINARY_NAME=$1 # User balance of stake tokens USER_COINS="100000000000stake" @@ -66,7 +66,7 @@ do # Build genesis file and node directory structure $BINARY_NAME init $MONIKER --chain-id provider --home ${PROV_NODE_DIR} - jq ".app_state.gov.params.voting_period = \"10s\" | .app_state.staking.params.unbonding_time = \"86400s\"" \ + jq ".app_state.gov.params.voting_period = \"100000s\" | .app_state.staking.params.unbonding_time = \"86400s\" | .app_state.slashing.params.signed_blocks_window=\"1000\" " \ ${PROV_NODE_DIR}/config/genesis.json > \ ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json @@ -203,6 +203,6 @@ PROVIDER_NODE_LISTEN_ADDR_STR=${PROVIDER_NODE_LISTEN_ADDR_STR::${#PROVIDER_NODE_ PROV_NODES_HOME_STR=${PROV_NODES_HOME_STR::${#PROV_NODES_HOME_STR}-1} echo "Testnet applications are set up! Run the following command to start CometMock:" -echo cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log +cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc sleep 5 \ No newline at end of file