From 4a715ac0ca1da92c4a1a81ed36ba80ff99f89305 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 12 Jun 2023 10:00:16 +0200 Subject: [PATCH 01/14] Remove double-comments --- cometmock/rpc_server/routes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 83beed3..5fcd41e 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -44,12 +44,12 @@ var Routes = map[string]*rpc.RPCFunc{ "tx_search": rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page,order_by"), "block_search": rpc.NewRPCFunc(BlockSearch, "query,page,per_page,order_by"), - // // tx broadcast API + // tx broadcast API "broadcast_tx_commit": rpc.NewRPCFunc(BroadcastTxCommit, "tx"), "broadcast_tx_sync": rpc.NewRPCFunc(BroadcastTxSync, "tx"), "broadcast_tx_async": rpc.NewRPCFunc(BroadcastTxAsync, "tx"), - // // abci API + // abci API "abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,prove"), } From b8df813bd4c0e8fe8351a9d766ec28925e382052 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 12 Jun 2023 18:51:33 +0200 Subject: [PATCH 02/14] Add logic for having nodes not sign --- cometmock/abci_client/client.go | 76 ++++++++++++++++++++++++++++++++- cometmock/main.go | 58 +++++-------------------- cometmock/rpc_server/routes.go | 41 ++++++++++++++++++ 3 files changed, 126 insertions(+), 49 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 240e6b7..1a71bb6 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -6,6 +6,7 @@ import ( "sync" "time" + db "github.com/cometbft/cometbft-db" abcitypes "github.com/cometbft/cometbft/abci/types" cryptoenc "github.com/cometbft/cometbft/crypto/encoding" "github.com/cometbft/cometbft/crypto/merkle" @@ -40,7 +41,7 @@ type AbciClient struct { LastBlock *types.Block LastCommit *types.Commit Storage storage.Storage - PrivValidators map[string]types.PrivValidator + PrivValidators map[string]types.PrivValidator // maps validator addresses to their priv validator structs IndexerService *txindex.IndexerService TxIndex *indexerkv.TxIndex BlockIndex *blockindexkv.BlockerIndexer @@ -49,6 +50,63 @@ type AbciClient struct { // can be used to check for nondeterminism in apps, but also slows down execution a bit, // though performance difference was not measured. ErrorOnUnequalResponses bool + + // validator addresses are mapped to false if they should not be signing, and to true if they should + signingStatus map[string]bool +} + +func CreateAndStartEventBus(logger cometlog.Logger) (*types.EventBus, error) { + eventBus := types.NewEventBus() + eventBus.SetLogger(logger.With("module", "events")) + if err := eventBus.Start(); err != nil { + return nil, err + } + return eventBus, nil +} + +func CreateAndStartIndexerService(eventBus *types.EventBus, logger cometlog.Logger) (*txindex.IndexerService, *indexerkv.TxIndex, *blockindexkv.BlockerIndexer, error) { + txIndexer := indexerkv.NewTxIndex(db.NewMemDB()) + blockIndexer := blockindexkv.New(db.NewMemDB()) + + indexerService := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus, false) + indexerService.SetLogger(logger.With("module", "txindex")) + + return indexerService, txIndexer, blockIndexer, indexerService.Start() +} + +func NewAbciClient(clients []AbciCounterpartyClient, logger cometlog.Logger, curState state.State, lastBlock *types.Block, lastCommit *types.Commit, storage storage.Storage, privValidators map[string]types.PrivValidator, errorOnUnequalResponses bool) *AbciClient { + signingStatus := make(map[string]bool) + for addr := range privValidators { + signingStatus[addr] = true + } + + eventBus, err := CreateAndStartEventBus(logger) + if err != nil { + logger.Error(err.Error()) + panic(err) + } + + indexerService, txIndex, blockIndex, err := CreateAndStartIndexerService(eventBus, logger) + if err != nil { + logger.Error(err.Error()) + panic(err) + } + + return &AbciClient{ + Clients: clients, + Logger: logger, + CurState: curState, + EventBus: *eventBus, + LastBlock: lastBlock, + LastCommit: lastCommit, + Storage: storage, + PrivValidators: privValidators, + IndexerService: indexerService, + TxIndex: txIndex, + BlockIndex: blockIndex, + ErrorOnUnequalResponses: errorOnUnequalResponses, + signingStatus: signingStatus, + } } func (a *AbciClient) RetryDisconnectedClients() { @@ -402,6 +460,17 @@ func (a *AbciClient) SendAbciQuery(data []byte, path string, height int64, prove return response.(*abcitypes.ResponseQuery), nil } +// RunEmptyBlocks runs a specified number of empty blocks through ABCI. +func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { + for i := 0; i < numBlocks; i++ { + _, _, _, _, _, err := a.RunBlock(nil, time.Now(), a.CurState.LastValidators.Proposer) + if err != nil { + return err + } + } + return nil +} + // RunBlock runs a block with a specified transaction through the ABCI application. // It calls BeginBlock, DeliverTx, EndBlock, Commit and then // updates the state. @@ -452,6 +521,11 @@ func (a *AbciClient) RunBlock(tx *[]byte, blockTime time.Time, proposer *types.V for index, val := range a.CurState.Validators.Validators { privVal := a.PrivValidators[val.Address.String()] + if !a.signingStatus[val.Address.String()] { + // validator should not sign this block + continue + } + // create and sign a precommit vote := &cmttypes.Vote{ ValidatorAddress: val.Address, diff --git a/cometmock/main.go b/cometmock/main.go index f25ddf8..8304ea7 100644 --- a/cometmock/main.go +++ b/cometmock/main.go @@ -5,29 +5,16 @@ import ( "strings" "time" - db "github.com/cometbft/cometbft-db" comet_abciclient "github.com/cometbft/cometbft/abci/client" cometlog "github.com/cometbft/cometbft/libs/log" "github.com/cometbft/cometbft/privval" "github.com/cometbft/cometbft/state" - blockindexkv "github.com/cometbft/cometbft/state/indexer/block/kv" - "github.com/cometbft/cometbft/state/txindex" - indexerkv "github.com/cometbft/cometbft/state/txindex/kv" "github.com/cometbft/cometbft/types" "github.com/p-offtermatt/CometMock/cometmock/abci_client" "github.com/p-offtermatt/CometMock/cometmock/rpc_server" "github.com/p-offtermatt/CometMock/cometmock/storage" ) -func CreateAndStartEventBus(logger cometlog.Logger) (*types.EventBus, error) { - eventBus := types.NewEventBus() - eventBus.SetLogger(logger.With("module", "events")) - if err := eventBus.Start(); err != nil { - return nil, err - } - return eventBus, nil -} - // GetMockPVsFromNodeHomes returns a list of MockPVs, created with the priv_validator_key's from the specified node homes // We use MockPV because they do not do sanity checks that would e.g. prevent double signing func GetMockPVsFromNodeHomes(nodeHomes []string) []types.PrivValidator { @@ -45,16 +32,6 @@ func GetMockPVsFromNodeHomes(nodeHomes []string) []types.PrivValidator { return mockPVs } -func CreateAndStartIndexerService(eventBus *types.EventBus, logger cometlog.Logger) (*txindex.IndexerService, *indexerkv.TxIndex, *blockindexkv.BlockerIndexer, error) { - txIndexer := indexerkv.NewTxIndex(db.NewMemDB()) - blockIndexer := blockindexkv.New(db.NewMemDB()) - - indexerService := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus, false) - indexerService.SetLogger(logger.With("module", "txindex")) - - return indexerService, txIndexer, blockIndexer, indexerService.Start() -} - func main() { logger := cometlog.NewTMLogger(cometlog.NewSyncWriter(os.Stdout)) @@ -111,18 +88,6 @@ func main() { } - eventBus, err := CreateAndStartEventBus(logger) - if err != nil { - logger.Error(err.Error()) - panic(err) - } - - indexerService, txIndex, blockIndex, err := CreateAndStartIndexerService(eventBus, logger) - if err != nil { - logger.Error(err.Error()) - panic(err) - } - for _, privVal := range privVals { pubkey, err := privVal.GetPubKey() if err != nil { @@ -134,19 +99,16 @@ func main() { privValsMap[addr.String()] = privVal } - abci_client.GlobalClient = &abci_client.AbciClient{ - Clients: clients, - Logger: logger, - CurState: curState, - ErrorOnUnequalResponses: true, - EventBus: *eventBus, - LastCommit: &types.Commit{}, - Storage: &storage.MapStorage{}, - PrivValidators: privValsMap, - IndexerService: indexerService, - TxIndex: txIndex, - BlockIndex: blockIndex, - } + abci_client.GlobalClient = abci_client.NewAbciClient( + clients, + logger, + curState, + &types.Block{}, + &types.Commit{}, + &storage.MapStorage{}, + privValsMap, + true, + ) // connect to clients abci_client.GlobalClient.RetryDisconnectedClients() diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 5fcd41e..949aef1 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -51,6 +51,47 @@ var Routes = map[string]*rpc.RPCFunc{ // abci API "abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,prove"), + + // cometmock specific API + "advance_blocks": rpc.NewRPCFunc(AdvanceBlocks, "num_blocks"), + "invoke_downtime": rpc.NewRPCFunc(InvokeDowntime, "val_cons_address,num_blocks"), + "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "val_cons_address,status"), +} + +type ResultSetSigningStatus struct{} + +func SetSigningStatus(ctx *rpctypes.Context, valConsAddress string, status string) (*ResultSetSigningStatus, error) { + if status != "down" && status != "up" { + return nil, errors.New("status must be either `up` to have the validator sign, or `down` to have the validator not sign") + } + + return &ResultSetSigningStatus{}, nil +} + +type ResultInvokeDowntime struct{} + +func InvokeDowntime(ctx *rpctypes.Context, valConsAddress string, numBlocks int) (*ResultInvokeDowntime, error) { + if numBlocks < 1 { + return nil, errors.New("num_blocks must be greater than 0") + } + + return &ResultInvokeDowntime{}, nil +} + +type ResultAdvanceBlocks struct{} + +// AdvanceBlocks advances the block height by numBlocks, running empty blocks. +// This API is specific to CometMock. +func AdvanceBlocks(ctx *rpctypes.Context, numBlocks int) (*ResultAdvanceBlocks, error) { + if numBlocks < 1 { + return nil, errors.New("num_blocks must be greater than 0") + } + + err := abci_client.GlobalClient.RunEmptyBlocks(numBlocks) + if err != nil { + return nil, err + } + return &ResultAdvanceBlocks{}, nil } // BlockSearch searches for a paginated set of blocks matching BeginBlock and From cc4c49ce5427ecee6ca4c970e8a0fababcd09f4a Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Tue, 13 Jun 2023 09:43:36 +0200 Subject: [PATCH 03/14] Add signing status to allow taking signers down --- cometmock/abci_client/client.go | 34 ++++++++++++++++++++++++++++----- cometmock/main.go | 6 +++++- cometmock/rpc_server/routes.go | 2 ++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 1a71bb6..6c1dd6a 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -28,7 +28,7 @@ var GlobalClient *AbciClient // store a mutex that allows only running one block at a time var blockMutex = sync.Mutex{} -var verbose = false +var verbose = true // AbciClient facilitates calls to the ABCI interface of multiple nodes. // It also tracks the current state and a common logger. @@ -55,6 +55,26 @@ type AbciClient struct { signingStatus map[string]bool } +func (a *AbciClient) GetSigningStatus(address string) (bool, error) { + status, ok := a.signingStatus[address] + if !ok { + return false, fmt.Errorf("address %s not found in signing status map, please double-check this is the key address of a validator key", address) + } + return status, nil +} + +func (a *AbciClient) SetSigningStatus(address string, status bool) error { + _, ok := a.signingStatus[address] + if !ok { + return fmt.Errorf("address %s not found in signing status map, please double-check this is the key address of a validator key", address) + } + a.signingStatus[address] = status + + a.Logger.Info("Set signing status", "address", address, "status", status) + + return nil +} + func CreateAndStartEventBus(logger cometlog.Logger) (*types.EventBus, error) { eventBus := types.NewEventBus() eventBus.SetLogger(logger.With("module", "events")) @@ -477,8 +497,12 @@ func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { // RunBlock is safe for use by multiple goroutines simultaneously. func (a *AbciClient) RunBlock(tx *[]byte, blockTime time.Time, proposer *types.Validator) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { // lock mutex to avoid running two blocks at the same time + a.Logger.Debug("Locking mutex") blockMutex.Lock() + defer blockMutex.Unlock() + defer a.Logger.Debug("Unlocking mutex") + a.Logger.Info("Running block") if verbose { a.Logger.Info("State at start of block", "state", a.CurState) @@ -537,7 +561,10 @@ func (a *AbciClient) RunBlock(tx *[]byte, blockTime time.Time, proposer *types.V BlockID: blockId.ToProto(), } - privVal.SignVote(a.CurState.ChainID, vote) + err = privVal.SignVote(a.CurState.ChainID, vote) + if err != nil { + return nil, nil, nil, nil, nil, err + } convertedVote, err := types.VoteFromProto(vote) if err != nil { @@ -647,9 +674,6 @@ func (a *AbciClient) RunBlock(tx *[]byte, blockTime time.Time, proposer *types.V } a.CurState.AppHash = resCommit.Data - // unlock mutex - blockMutex.Unlock() - return resBeginBlock, resCheckTx, resDeliverTx, resEndBlock, resCommit, nil } diff --git a/cometmock/main.go b/cometmock/main.go index 8304ea7..4028911 100644 --- a/cometmock/main.go +++ b/cometmock/main.go @@ -131,7 +131,11 @@ func main() { // produce a block every second for { - abci_client.GlobalClient.RunBlock(nil, time.Now(), abci_client.GlobalClient.CurState.LastValidators.Proposer) + _, _, _, _, _, err := abci_client.GlobalClient.RunBlock(nil, time.Now(), abci_client.GlobalClient.CurState.LastValidators.Proposer) + if err != nil { + logger.Error(err.Error()) + panic(err) + } time.Sleep(1 * time.Second) } } diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 949aef1..8f4af88 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -65,6 +65,8 @@ func SetSigningStatus(ctx *rpctypes.Context, valConsAddress string, status strin return nil, errors.New("status must be either `up` to have the validator sign, or `down` to have the validator not sign") } + abci_client.GlobalClient.SetSigningStatus(valConsAddress, status == "up") + return &ResultSetSigningStatus{}, nil } From 7449c57bb1252d677f7c8442daacbbdaf4e54018 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Tue, 13 Jun 2023 11:19:25 +0200 Subject: [PATCH 04/14] Use CommitSig.Absent instead of skipping CommitSigs --- cometmock/abci_client/client.go | 80 +++++++++++++++++++++------------ cometmock/rpc_server/routes.go | 4 +- 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 6c1dd6a..a855d3d 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -28,7 +28,7 @@ var GlobalClient *AbciClient // store a mutex that allows only running one block at a time var blockMutex = sync.Mutex{} -var verbose = true +var verbose = false // AbciClient facilitates calls to the ABCI interface of multiple nodes. // It also tracks the current state and a common logger. @@ -224,7 +224,7 @@ func (a *AbciClient) SendBeginBlock(block *types.Block) (*abcitypes.ResponseBegi a.Logger.Info("Sending BeginBlock to clients") } // build the BeginBlock request - beginBlockRequest := CreateBeginBlockRequest(&block.Header, block.LastCommit) + beginBlockRequest := a.CreateBeginBlockRequest(&block.Header, block.LastCommit) // send BeginBlock to all clients and collect the responses f := func(client AbciCounterpartyClient) (interface{}, error) { @@ -247,10 +247,33 @@ func (a *AbciClient) SendBeginBlock(block *types.Block) (*abcitypes.ResponseBegi return responses[0].(*abcitypes.ResponseBeginBlock), nil } -func CreateBeginBlockRequest(header *types.Header, lastCommit *types.Commit) *abcitypes.RequestBeginBlock { +func (a *AbciClient) CreateBeginBlockRequest(header *types.Header, lastCommit *types.Commit) *abcitypes.RequestBeginBlock { + commitSigs := lastCommit.Signatures + + // if this is the first block, LastCommitInfo.Votes will be empty, see https://github.com/cometbft/cometbft/blob/release/v0.34.24/state/execution.go#L342 + voteInfos := make([]abcitypes.VoteInfo, len(commitSigs)) + if lastCommit.Height != 0 { + for i := range commitSigs { + val := a.CurState.Validators.Validators[i] + byteAddress := val.Address.Bytes() + + abciVal := abcitypes.Validator{ + Address: byteAddress, + Power: val.VotingPower, + } + + signedLastBlock := !commitSigs[i].Absent() + + voteInfos[i] = abcitypes.VoteInfo{ + Validator: abciVal, + SignedLastBlock: signedLastBlock, + } + } + } + return &abcitypes.RequestBeginBlock{ // TODO: fill in Votes - LastCommitInfo: abcitypes.LastCommitInfo{Round: lastCommit.Round, Votes: []abcitypes.VoteInfo{}}, + LastCommitInfo: abcitypes.LastCommitInfo{Round: lastCommit.Round, Votes: voteInfos}, Header: *header.ToProto(), } } @@ -545,35 +568,34 @@ func (a *AbciClient) RunBlock(tx *[]byte, blockTime time.Time, proposer *types.V for index, val := range a.CurState.Validators.Validators { privVal := a.PrivValidators[val.Address.String()] - if !a.signingStatus[val.Address.String()] { - // validator should not sign this block - continue - } - - // create and sign a precommit - vote := &cmttypes.Vote{ - ValidatorAddress: val.Address, - ValidatorIndex: int32(index), - Height: block.Height, - Round: 1, - Timestamp: time.Now(), - Type: cmttypes.PrecommitType, - BlockID: blockId.ToProto(), - } + if a.signingStatus[val.Address.String()] { + // create and sign a precommit + vote := &cmttypes.Vote{ + ValidatorAddress: val.Address, + ValidatorIndex: int32(index), + Height: block.Height, + Round: 1, + Timestamp: time.Now(), + Type: cmttypes.PrecommitType, + BlockID: blockId.ToProto(), + } - err = privVal.SignVote(a.CurState.ChainID, vote) - if err != nil { - return nil, nil, nil, nil, nil, err - } + err = privVal.SignVote(a.CurState.ChainID, vote) + if err != nil { + return nil, nil, nil, nil, nil, err + } - convertedVote, err := types.VoteFromProto(vote) - if err != nil { - return nil, nil, nil, nil, nil, err - } + convertedVote, err := types.VoteFromProto(vote) + if err != nil { + return nil, nil, nil, nil, nil, err + } - commitSig := convertedVote.CommitSig() + commitSig := convertedVote.CommitSig() - commitSigs = append(commitSigs, commitSig) + commitSigs = append(commitSigs, commitSig) + } else { + commitSigs = append(commitSigs, types.NewCommitSigAbsent()) + } } a.LastCommit = types.NewCommit( diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 8f4af88..ac4c4a4 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -65,9 +65,9 @@ func SetSigningStatus(ctx *rpctypes.Context, valConsAddress string, status strin return nil, errors.New("status must be either `up` to have the validator sign, or `down` to have the validator not sign") } - abci_client.GlobalClient.SetSigningStatus(valConsAddress, status == "up") + err := abci_client.GlobalClient.SetSigningStatus(valConsAddress, status == "up") - return &ResultSetSigningStatus{}, nil + return &ResultSetSigningStatus{}, err } type ResultInvokeDowntime struct{} From 41467a84081896b69b3b46653ca6ba1672a903f5 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Tue, 13 Jun 2023 11:19:42 +0200 Subject: [PATCH 05/14] Add local testnet scripts --- local-testnet-debug.sh | 423 +++++++++++++++++++++++++++++++ local-testnet.sh | 547 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 970 insertions(+) create mode 100755 local-testnet-debug.sh create mode 100755 local-testnet.sh diff --git a/local-testnet-debug.sh b/local-testnet-debug.sh new file mode 100755 index 0000000..581bbc6 --- /dev/null +++ b/local-testnet-debug.sh @@ -0,0 +1,423 @@ +#!/bin/bash +set -eux + +# User balance of stake tokens +USER_COINS="100000000000stake" +# Amount of stake tokens staked +STAKE="100000000stake" +# Node IP address +NODE_IP="127.0.0.1" + +# Home directory +HOME_DIR=$HOME + +# Validator moniker +MONIKERS=("coordinator" "alice" "bob") +LEAD_VALIDATOR_MONIKER="coordinator" + +PROV_NODES_ROOT_DIR=${HOME_DIR}/nodes/provider +CONS_NODES_ROOT_DIR=${HOME_DIR}/nodes/consumer + +# Base port. Ports assigned after these ports sequentially by nodes. +RPC_LADDR_BASEPORT=29170 +P2P_LADDR_BASEPORT=29180 +GRPC_LADDR_BASEPORT=29190 +NODE_ADDRESS_BASEPORT=29200 +PPROF_LADDR_BASEPORT=29210 +CLIENT_BASEPORT=29220 + +# keeps a comma separated list of node addresses for provider and consumer +PROVIDER_NODE_LISTEN_ADDR_STR="" +CONSUMER_NODE_LISTEN_ADDR_STR="" + +PROVIDER_HOME_DIRS_STR="" +CONSUMER_HOME_DIRS_STR="" + +PROVIDER_COMETMOCK_ADDR=tcp://$NODE_IP:22331 +CONSUMER_COMETMOCK_ADDR=tcp://$NODE_IP:22332 + +# Clean start +pkill -f interchain-security-pd &> /dev/null || true +pkill -f cometmock &> /dev/null || true +sleep 1 +rm -rf ${PROV_NODES_ROOT_DIR} +rm -rf ${CONS_NODES_ROOT_DIR} + +# Let lead validator create genesis file +LEAD_VALIDATOR_PROV_DIR=${PROV_NODES_ROOT_DIR}/provider-${LEAD_VALIDATOR_MONIKER} +LEAD_VALIDATOR_CONS_DIR=${CONS_NODES_ROOT_DIR}/consumer-${LEAD_VALIDATOR_MONIKER} +LEAD_PROV_KEY=${LEAD_VALIDATOR_MONIKER}-key +LEAD_PROV_LISTEN_ADDR=tcp://${NODE_IP}:${RPC_LADDR_BASEPORT} + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # Build genesis file and node directory structure + interchain-security-pd init $MONIKER --chain-id provider --home ${PROV_NODE_DIR} + jq ".app_state.gov.voting_params.voting_period = \"10s\" | .app_state.staking.params.unbonding_time = \"86400s\"" \ + ${PROV_NODE_DIR}/config/genesis.json > \ + ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json + + + sleep 1 + + # Create account keypair + interchain-security-pd keys add $PROV_KEY --home ${PROV_NODE_DIR} --keyring-backend test --output json > ${PROV_NODE_DIR}/${PROV_KEY}.json 2>&1 + sleep 1 + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + # Add stake to user + PROV_ACCOUNT_ADDR=$(jq -r '.address' ${PROV_NODE_DIR}/${PROV_KEY}.json) + interchain-security-pd add-genesis-account $PROV_ACCOUNT_ADDR $USER_COINS --home ${PROV_NODE_DIR} --keyring-backend test + sleep 1 + + # copy genesis out, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json + fi + + PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + + # adjust configs of this node + sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${PROV_NODE_DIR}/config/config.toml + sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${PROV_NODE_DIR}/config/config.toml + + # make address book non-strict. necessary for this setup + sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${PROV_NODE_DIR}/config/config.toml + + # avoid port double binding + sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${PROV_NODE_DIR}/config/config.toml + + # allow duplicate IP addresses (all nodes are on the same machine) + sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${PROV_NODE_DIR}/config/config.toml +done + +for MONIKER in "${MONIKERS[@]}" +do + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json* ${PROV_NODE_DIR}/config/genesis.json + fi + + # Stake 1/1000 user's coins + interchain-security-pd gentx $PROV_KEY $STAKE --chain-id provider --home ${PROV_NODE_DIR} --keyring-backend test --moniker $MONIKER + sleep 1 + + # Copy gentxs to the lead validator for possible future collection. + # Obviously we don't need to copy the first validator's gentx to itself + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/gentx/* ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + fi +done + +# Collect genesis transactions with lead validator +interchain-security-pd collect-gentxs --home ${LEAD_VALIDATOR_PROV_DIR} --gentx-dir ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + +sleep 1 + + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + + PERSISTENT_PEERS="" + + for peer_index in "${!MONIKERS[@]}" + do + if [ $index == $peer_index ]; then + continue + fi + PEER_MONIKER=${MONIKERS[$peer_index]} + + PEER_PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${PEER_MONIKER} + + PEER_NODE_ID=$(interchain-security-pd tendermint show-node-id --home ${PEER_PROV_NODE_DIR}) + + PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $peer_index)) + PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" + done + + # remove trailing comma from persistent peers + PERSISTENT_PEERS=${PERSISTENT_PEERS:1} + + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${PROV_NODES_ROOT_DIR}/consumer-${MONIKER} + + # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + $index)) + NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + $index)) + + PROVIDER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$PROVIDER_NODE_LISTEN_ADDR_STR" + PROVIDER_HOME_DIRS_STR="$PROV_NODE_DIR,$PROVIDER_HOME_DIRS_STR" + + + # Start gaia + interchain-security-pd start \ + --home ${PROV_NODE_DIR} \ + --transport=grpc --with-tendermint=false \ + --p2p.persistent_peers ${PERSISTENT_PEERS} \ + --rpc.laddr tcp://${NODE_IP}:${RPC_LADDR_PORT} \ + --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ + --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ + --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ + --grpc-web.enable=false &> ${PROV_NODE_DIR}/logs & + + sleep 5 +done + +# remove trailing commas +PROVIDER_NODE_LISTEN_ADDR_STR=${PROVIDER_NODE_LISTEN_ADDR_STR::${#PROVIDER_NODE_LISTEN_ADDR_STR}-1} +PROVIDER_HOME_DIRS_STR=${PROVIDER_HOME_DIRS_STR::${#PROVIDER_HOME_DIRS_STR}-1} + +echo cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROVIDER_HOME_DIRS_STR &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log & + +echo "Start cometmock, then press any key to continue" +read -n 1 -s key + +# Build consumer chain proposal file +tee ${LEAD_VALIDATOR_PROV_DIR}/consumer-proposal.json< /dev/null || true +sleep 1 +rm -rf ${CONS_NODES_ROOT_DIR} + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + # validator key + PROV_KEY=${MONIKER}-key + + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # Build genesis file and node directory structure + interchain-security-cd init $MONIKER --chain-id consumer --home ${CONS_NODE_DIR} + + sleep 1 + + # Create account keypair + interchain-security-cd keys add $PROV_KEY --home ${CONS_NODE_DIR} --keyring-backend test --output json > ${CONS_NODE_DIR}/${PROV_KEY}.json 2>&1 + sleep 1 + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json ${CONS_NODE_DIR}/config/genesis.json + fi + + # Add stake to user + CONS_ACCOUNT_ADDR=$(jq -r '.address' ${CONS_NODE_DIR}/${PROV_KEY}.json) + interchain-security-cd add-genesis-account $CONS_ACCOUNT_ADDR $USER_COINS --home ${CONS_NODE_DIR} + sleep 4 + + ### this probably doesnt have to be done for each node + # Add consumer genesis states to genesis file + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) + RPC_LADDR=tcp://${NODE_IP}:${RPC_LADDR_PORT} + interchain-security-pd query provider consumer-genesis consumer --home ${PROV_NODE_DIR} --node $PROVIDER_COMETMOCK_ADDR -o json > consumer_gen.json + jq -s '.[0].app_state.ccvconsumer = .[1] | .[0]' ${CONS_NODE_DIR}/config/genesis.json consumer_gen.json > ${CONS_NODE_DIR}/edited_genesis.json \ + && mv ${CONS_NODE_DIR}/edited_genesis.json ${CONS_NODE_DIR}/config/genesis.json + rm consumer_gen.json + ### + + # copy genesis out, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${CONS_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json + fi + + PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + + # adjust configs of this node + sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${CONS_NODE_DIR}/config/config.toml + sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${CONS_NODE_DIR}/config/config.toml + + # make address book non-strict. necessary for this setup + sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${CONS_NODE_DIR}/config/config.toml + + # avoid port double binding + sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${CONS_NODE_DIR}/config/config.toml + + # allow duplicate IP addresses (all nodes are on the same machine) + sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${CONS_NODE_DIR}/config/config.toml + + # Create validator states + echo '{"height": "0","round": 0,"step": 0}' > ${CONS_NODE_DIR}/data/priv_validator_state.json + + # Copy validator key files + cp ${PROV_NODE_DIR}/config/priv_validator_key.json ${CONS_NODE_DIR}/config/priv_validator_key.json + cp ${PROV_NODE_DIR}/config/node_key.json ${CONS_NODE_DIR}/config/node_key.json + + # Set default client port + CLIENT_PORT=$(($CLIENT_BASEPORT + ${#MONIKERS[@]} + $index)) + sed -i -r "/node =/ s/= .*/= \"tcp:\/\/${NODE_IP}:${CLIENT_PORT}\"/" ${CONS_NODE_DIR}/config/client.toml + +done + +sleep 1 + + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + + PERSISTENT_PEERS="" + + for peer_index in "${!MONIKERS[@]}" + do + if [ $index == $peer_index ]; then + continue + fi + PEER_MONIKER=${MONIKERS[$peer_index]} + + PEER_CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${PEER_MONIKER} + + PEER_NODE_ID=$(interchain-security-pd tendermint show-node-id --home ${PEER_CONS_NODE_DIR}) + + PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + ${#MONIKERS[@]} + $peer_index)) + PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" + done + + # remove trailing comma from persistent peers + PERSISTENT_PEERS=${PERSISTENT_PEERS:1} + + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json ${CONS_NODE_DIR}/config/genesis.json + fi + + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + ${#MONIKERS[@]} + $index)) + + CONSUMER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$CONSUMER_NODE_LISTEN_ADDR_STR" + CONSUMER_HOME_DIRS_STR="$CONS_NODE_DIR,$CONSUMER_HOME_DIRS_STR" + + # Start gaia + interchain-security-cd start \ + --home ${CONS_NODE_DIR} \ + --transport=grpc --with-tendermint=false \ + --p2p.persistent_peers ${PERSISTENT_PEERS} \ + --rpc.laddr tcp://${NODE_IP}:${RPC_LADDR_PORT} \ + --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ + --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ + --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ + --grpc-web.enable=false &> ${CONS_NODE_DIR}/logs & + + sleep 1 +done + +# remove trailing commas +CONSUMER_NODE_LISTEN_ADDR_STR=${CONSUMER_NODE_LISTEN_ADDR_STR::${#CONSUMER_NODE_LISTEN_ADDR_STR}-1} +CONSUMER_HOME_DIRS_STR=${CONSUMER_HOME_DIRS_STR::${#CONSUMER_HOME_DIRS_STR}-1} + +echo cometmock $CONSUMER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json $CONSUMER_COMETMOCK_ADDR $CONSUMER_HOME_DIRS_STR &> ${LEAD_VALIDATOR_CONS_DIR}/cometmock_log & +echo "Start cometmock, then press any key to continue" +read -n 1 -s key + +sleep 5 + +rm -r ~/.relayer + +# initialize gorelayer +rly config init + +# add chains +rly chains add --file go_rly_provider.json provider +rly chains add --file go_rly_consumer.json consumer + +# gorelayer +rly keys delete consumer default -y || true +rly keys delete provider default -y || true + +# take keys from provider and consumer and add them to gorelayer +rly keys restore provider default "$(cat ${LEAD_VALIDATOR_PROV_DIR}/${LEAD_VALIDATOR_MONIKER}-key.json | jq -r '.mnemonic')" +rly keys restore consumer default "$(cat ${LEAD_VALIDATOR_CONS_DIR}/${LEAD_VALIDATOR_MONIKER}-key.json | jq -r '.mnemonic')" + +rly paths new consumer provider testpath diff --git a/local-testnet.sh b/local-testnet.sh new file mode 100755 index 0000000..5ad19bc --- /dev/null +++ b/local-testnet.sh @@ -0,0 +1,547 @@ +#!/bin/bash +set -eux + +# User balance of stake tokens +USER_COINS="100000000000stake" +# Amount of stake tokens staked +STAKE="100000000stake" +# Node IP address +NODE_IP="127.0.0.1" + +# Home directory +HOME_DIR=$HOME + +# Validator moniker +MONIKERS=("coordinator" "alice" "bob") +LEAD_VALIDATOR_MONIKER="coordinator" + +PROV_NODES_ROOT_DIR=${HOME_DIR}/nodes/provider +CONS_NODES_ROOT_DIR=${HOME_DIR}/nodes/consumer + +# Base port. Ports assigned after these ports sequentially by nodes. +RPC_LADDR_BASEPORT=29170 +P2P_LADDR_BASEPORT=29180 +GRPC_LADDR_BASEPORT=29190 +NODE_ADDRESS_BASEPORT=29200 +PPROF_LADDR_BASEPORT=29210 +CLIENT_BASEPORT=29220 + +# keeps a comma separated list of node addresses for provider and consumer +PROVIDER_NODE_LISTEN_ADDR_STR="" +CONSUMER_NODE_LISTEN_ADDR_STR="" + +# Strings that keep the homes of provider nodes and homes of consumer nodes +PROV_NODES_HOME_STR="" +CONS_NODES_HOME_STR="" + +PROVIDER_COMETMOCK_ADDR=tcp://$NODE_IP:22331 +CONSUMER_COMETMOCK_ADDR=tcp://$NODE_IP:22332 + +# Clean start +pkill -f interchain-security-pd &> /dev/null || true +pkill -f cometmock &> /dev/null || true +sleep 1 +rm -rf ${PROV_NODES_ROOT_DIR} +rm -rf ${CONS_NODES_ROOT_DIR} + +# Let lead validator create genesis file +LEAD_VALIDATOR_PROV_DIR=${PROV_NODES_ROOT_DIR}/provider-${LEAD_VALIDATOR_MONIKER} +LEAD_VALIDATOR_CONS_DIR=${CONS_NODES_ROOT_DIR}/consumer-${LEAD_VALIDATOR_MONIKER} +LEAD_PROV_KEY=${LEAD_VALIDATOR_MONIKER}-key +LEAD_PROV_LISTEN_ADDR=tcp://${NODE_IP}:${RPC_LADDR_BASEPORT} + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # Build genesis file and node directory structure + interchain-security-pd init $MONIKER --chain-id provider --home ${PROV_NODE_DIR} + jq ".app_state.gov.voting_params.voting_period = \"10s\" | .app_state.staking.params.unbonding_time = \"86400s\"" \ + ${PROV_NODE_DIR}/config/genesis.json > \ + ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json + + + sleep 1 + + # Create account keypair + interchain-security-pd keys add $PROV_KEY --home ${PROV_NODE_DIR} --keyring-backend test --output json > ${PROV_NODE_DIR}/${PROV_KEY}.json 2>&1 + sleep 1 + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + # Add stake to user + PROV_ACCOUNT_ADDR=$(jq -r '.address' ${PROV_NODE_DIR}/${PROV_KEY}.json) + interchain-security-pd add-genesis-account $PROV_ACCOUNT_ADDR $USER_COINS --home ${PROV_NODE_DIR} --keyring-backend test + sleep 1 + + # copy genesis out, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json + fi + + PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + + # adjust configs of this node + sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${PROV_NODE_DIR}/config/config.toml + sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${PROV_NODE_DIR}/config/config.toml + + # make address book non-strict. necessary for this setup + sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${PROV_NODE_DIR}/config/config.toml + + # avoid port double binding + sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${PROV_NODE_DIR}/config/config.toml + + # allow duplicate IP addresses (all nodes are on the same machine) + sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${PROV_NODE_DIR}/config/config.toml +done + +for MONIKER in "${MONIKERS[@]}" +do + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json* ${PROV_NODE_DIR}/config/genesis.json + fi + + # Stake 1/1000 user's coins + interchain-security-pd gentx $PROV_KEY $STAKE --chain-id provider --home ${PROV_NODE_DIR} --keyring-backend test --moniker $MONIKER + sleep 1 + + # Copy gentxs to the lead validator for possible future collection. + # Obviously we don't need to copy the first validator's gentx to itself + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/gentx/* ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + fi +done + +# Collect genesis transactions with lead validator +interchain-security-pd collect-gentxs --home ${LEAD_VALIDATOR_PROV_DIR} --gentx-dir ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + +sleep 1 + + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + + PERSISTENT_PEERS="" + + for peer_index in "${!MONIKERS[@]}" + do + if [ $index == $peer_index ]; then + continue + fi + PEER_MONIKER=${MONIKERS[$peer_index]} + + PEER_PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${PEER_MONIKER} + + PEER_NODE_ID=$(interchain-security-pd tendermint show-node-id --home ${PEER_PROV_NODE_DIR}) + + PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $peer_index)) + PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" + done + + # remove trailing comma from persistent peers + PERSISTENT_PEERS=${PERSISTENT_PEERS:1} + + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${PROV_NODES_ROOT_DIR}/consumer-${MONIKER} + + # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + $index)) + NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + $index)) + + PROVIDER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$PROVIDER_NODE_LISTEN_ADDR_STR" + PROV_NODES_HOME_STR="${PROV_NODE_DIR},$PROV_NODES_HOME_STR" + + # Start gaia + interchain-security-pd start \ + --home ${PROV_NODE_DIR} \ + --transport=grpc --with-tendermint=false \ + --p2p.persistent_peers ${PERSISTENT_PEERS} \ + --rpc.laddr tcp://${NODE_IP}:${RPC_LADDR_PORT} \ + --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ + --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ + --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ + --grpc-web.enable=false &> ${PROV_NODE_DIR}/logs & + + sleep 5 +done + +PROVIDER_NODE_LISTEN_ADDR_STR=${PROVIDER_NODE_LISTEN_ADDR_STR::${#PROVIDER_NODE_LISTEN_ADDR_STR}-1} +PROV_NODES_HOME_STR=${PROV_NODES_HOME_STR::${#PROV_NODES_HOME_STR}-1} + +cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log & + +sleep 5 + +# Build consumer chain proposal file +tee ${LEAD_VALIDATOR_PROV_DIR}/consumer-proposal.json< /dev/null || true +sleep 1 +rm -rf ${CONS_NODES_ROOT_DIR} + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + # validator key + PROV_KEY=${MONIKER}-key + + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # Build genesis file and node directory structure + interchain-security-cd init $MONIKER --chain-id consumer --home ${CONS_NODE_DIR} + + sleep 1 + + # Create account keypair + interchain-security-cd keys add $PROV_KEY --home ${CONS_NODE_DIR} --keyring-backend test --output json > ${CONS_NODE_DIR}/${PROV_KEY}.json 2>&1 + sleep 1 + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json ${CONS_NODE_DIR}/config/genesis.json + fi + + # Add stake to user + CONS_ACCOUNT_ADDR=$(jq -r '.address' ${CONS_NODE_DIR}/${PROV_KEY}.json) + interchain-security-cd add-genesis-account $CONS_ACCOUNT_ADDR $USER_COINS --home ${CONS_NODE_DIR} + sleep 10 + + ### this probably doesnt have to be done for each node + # Add consumer genesis states to genesis file + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) + RPC_LADDR=tcp://${NODE_IP}:${RPC_LADDR_PORT} + interchain-security-pd query provider consumer-genesis consumer --home ${PROV_NODE_DIR} --node $PROVIDER_COMETMOCK_ADDR -o json > consumer_gen.json + jq -s '.[0].app_state.ccvconsumer = .[1] | .[0]' ${CONS_NODE_DIR}/config/genesis.json consumer_gen.json > ${CONS_NODE_DIR}/edited_genesis.json \ + && mv ${CONS_NODE_DIR}/edited_genesis.json ${CONS_NODE_DIR}/config/genesis.json + rm consumer_gen.json + ### + + # copy genesis out, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${CONS_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json + fi + + PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + + # adjust configs of this node + sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${CONS_NODE_DIR}/config/config.toml + sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${CONS_NODE_DIR}/config/config.toml + + # make address book non-strict. necessary for this setup + sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${CONS_NODE_DIR}/config/config.toml + + # avoid port double binding + sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${CONS_NODE_DIR}/config/config.toml + + # allow duplicate IP addresses (all nodes are on the same machine) + sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${CONS_NODE_DIR}/config/config.toml + + # Create validator states + echo '{"height": "0","round": 0,"step": 0}' > ${CONS_NODE_DIR}/data/priv_validator_state.json + + # Copy validator key files + cp ${PROV_NODE_DIR}/config/priv_validator_key.json ${CONS_NODE_DIR}/config/priv_validator_key.json + cp ${PROV_NODE_DIR}/config/node_key.json ${CONS_NODE_DIR}/config/node_key.json + + # Set default client port + CLIENT_PORT=$(($CLIENT_BASEPORT + ${#MONIKERS[@]} + $index)) + sed -i -r "/node =/ s/= .*/= \"tcp:\/\/${NODE_IP}:${CLIENT_PORT}\"/" ${CONS_NODE_DIR}/config/client.toml + +done + +sleep 1 + + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + + PERSISTENT_PEERS="" + + for peer_index in "${!MONIKERS[@]}" + do + if [ $index == $peer_index ]; then + continue + fi + PEER_MONIKER=${MONIKERS[$peer_index]} + + PEER_CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${PEER_MONIKER} + + PEER_NODE_ID=$(interchain-security-pd tendermint show-node-id --home ${PEER_CONS_NODE_DIR}) + + PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + ${#MONIKERS[@]} + $peer_index)) + PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" + done + + # remove trailing comma from persistent peers + PERSISTENT_PEERS=${PERSISTENT_PEERS:1} + + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json ${CONS_NODE_DIR}/config/genesis.json + fi + + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + ${#MONIKERS[@]} + $index)) + NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + ${#MONIKERS[@]} + $index)) + + CONSUMER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$CONSUMER_NODE_LISTEN_ADDR_STR" + # Add the home directory to the CONS_NODES_HOME_STR + CONS_NODES_HOME_STR="${CONS_NODE_DIR},${CONS_NODES_HOME_STR}" + + # Start gaia + interchain-security-cd start \ + --home ${CONS_NODE_DIR} \ + --transport=grpc --with-tendermint=false \ + --p2p.persistent_peers ${PERSISTENT_PEERS} \ + --rpc.laddr tcp://${NODE_IP}:${RPC_LADDR_PORT} \ + --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ + --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ + --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ + --grpc-web.enable=false &> ${CONS_NODE_DIR}/logs & + + sleep 6 +done + +# remove trailing comma from consumer node listen addr str +CONSUMER_NODE_LISTEN_ADDR_STR=${CONSUMER_NODE_LISTEN_ADDR_STR::${#CONSUMER_NODE_LISTEN_ADDR_STR}-1} +CONS_NODES_HOME_STR=${CONS_NODES_HOME_STR::${#CONS_NODES_HOME_STR}-1} + +cometmock $CONSUMER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_CONS_DIR}/config/genesis.json $CONSUMER_COMETMOCK_ADDR $CONS_NODES_HOME_STR &> ${LEAD_VALIDATOR_CONS_DIR}/cometmock_log & + +sleep 5 +# # Setup Hermes in packet relayer mode +# pkill -f hermes 2> /dev/null || true + +# tee ~/.hermes/config.toml< ~/.hermes/logs & + +# interchain-security-pd q tendermint-validator-set --home ${PROV_NODE_DIR} +# interchain-security-cd q tendermint-validator-set --home ${CONS_NODE_DIR} + +# DELEGATIONS=$(interchain-security-pd q staking delegations $PROV_ACCOUNT_ADDR --home ${PROV_NODE_DIR} -o json) + +# OPERATOR_ADDR=$(echo $DELEGATIONS | jq -r '.delegation_responses[0].delegation.validator_address') + +# interchain-security-pd tx staking delegate $OPERATOR_ADDR 1000000stake \ +# --from $PROV_KEY \ +# --keyring-backend test \ +# --home ${PROV_NODE_DIR} \ +# --chain-id provider \ +# -y -b block + +# sleep 13 + +# interchain-security-pd q tendermint-validator-set --home ${PROV_NODE_DIR} +# interchain-security-cd q tendermint-validator-set --home ${CONS_NODE_DIR} + + +# # sleep 5 + +# # tee ${PROV_NODE_DIR}/stop-consumer-proposal.json< Date: Tue, 13 Jun 2023 13:30:44 +0200 Subject: [PATCH 06/14] Propagate timestamp to RunBlock --- cometmock/abci_client/client.go | 17 ++- cometmock/main.go | 4 +- cometmock/rpc_server/routes.go | 23 ++- cometmock/test_utils/helpers.go | 248 -------------------------------- 4 files changed, 27 insertions(+), 265 deletions(-) delete mode 100644 cometmock/test_utils/helpers.go diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index a855d3d..3d30df0 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -53,6 +53,11 @@ type AbciClient struct { // validator addresses are mapped to false if they should not be signing, and to true if they should signingStatus map[string]bool + + // time offset. whenever we qury the time, we add this offset to it + // this means after modifying this, blocks will have the timestamp offset by this value. + // this will look to the app like one block took a long time to be produced. + timeOffset time.Duration } func (a *AbciClient) GetSigningStatus(address string) (bool, error) { @@ -506,7 +511,7 @@ func (a *AbciClient) SendAbciQuery(data []byte, path string, height int64, prove // RunEmptyBlocks runs a specified number of empty blocks through ABCI. func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { for i := 0; i < numBlocks; i++ { - _, _, _, _, _, err := a.RunBlock(nil, time.Now(), a.CurState.LastValidators.Proposer) + _, _, _, _, _, err := a.RunBlock(nil) if err != nil { return err } @@ -514,11 +519,17 @@ func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { return nil } +// RunBlock runs a block with a specified transaction through the ABCI application. +// It calls RunBlockWithTimeAndProposer with the current time and the LastValidators.Proposer. +func (a *AbciClient) RunBlock(tx *[]byte) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { + return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer) +} + // RunBlock runs a block with a specified transaction through the ABCI application. // It calls BeginBlock, DeliverTx, EndBlock, Commit and then // updates the state. // RunBlock is safe for use by multiple goroutines simultaneously. -func (a *AbciClient) RunBlock(tx *[]byte, blockTime time.Time, proposer *types.Validator) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { +func (a *AbciClient) RunBlockWithTimeAndProposer(tx *[]byte, blockTime time.Time, proposer *types.Validator) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { // lock mutex to avoid running two blocks at the same time a.Logger.Debug("Locking mutex") blockMutex.Lock() @@ -575,7 +586,7 @@ func (a *AbciClient) RunBlock(tx *[]byte, blockTime time.Time, proposer *types.V ValidatorIndex: int32(index), Height: block.Height, Round: 1, - Timestamp: time.Now(), + Timestamp: time.Now().Add(a.timeOffset), Type: cmttypes.PrecommitType, BlockID: blockId.ToProto(), } diff --git a/cometmock/main.go b/cometmock/main.go index 4028911..ed37a4c 100644 --- a/cometmock/main.go +++ b/cometmock/main.go @@ -121,7 +121,7 @@ func main() { } // run an empty block - _, _, _, _, _, err = abci_client.GlobalClient.RunBlock(nil, time.Now(), abci_client.GlobalClient.CurState.LastValidators.Proposer) + _, _, _, _, _, err = abci_client.GlobalClient.RunBlock(nil) if err != nil { logger.Error(err.Error()) panic(err) @@ -131,7 +131,7 @@ func main() { // produce a block every second for { - _, _, _, _, _, err := abci_client.GlobalClient.RunBlock(nil, time.Now(), abci_client.GlobalClient.CurState.LastValidators.Proposer) + _, _, _, _, _, err := abci_client.GlobalClient.RunBlock(nil) if err != nil { logger.Error(err.Error()) panic(err) diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index ac4c4a4..58ba950 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -54,8 +54,17 @@ var Routes = map[string]*rpc.RPCFunc{ // cometmock specific API "advance_blocks": rpc.NewRPCFunc(AdvanceBlocks, "num_blocks"), - "invoke_downtime": rpc.NewRPCFunc(InvokeDowntime, "val_cons_address,num_blocks"), "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "val_cons_address,status"), + "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration"), +} + +type ResultAdvanceTime struct{} + +// AdvanceTime advances the block time by the given duration. +// This API is specific to CometMock. +func AdvanceTime(ctx *rpctypes.Context, duration time.Duration) (*ResultAdvanceTime, error) { + // TODO: Implement the AdvanceTime function + return &ResultAdvanceTime{}, nil } type ResultSetSigningStatus struct{} @@ -70,16 +79,6 @@ func SetSigningStatus(ctx *rpctypes.Context, valConsAddress string, status strin return &ResultSetSigningStatus{}, err } -type ResultInvokeDowntime struct{} - -func InvokeDowntime(ctx *rpctypes.Context, valConsAddress string, numBlocks int) (*ResultInvokeDowntime, error) { - if numBlocks < 1 { - return nil, errors.New("num_blocks must be greater than 0") - } - - return &ResultInvokeDowntime{}, nil -} - type ResultAdvanceBlocks struct{} // AdvanceBlocks advances the block height by numBlocks, running empty blocks. @@ -462,7 +461,7 @@ func BroadcastTx(tx *types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { byteTx := []byte(*tx) - _, responseCheckTx, responseDeliverTx, _, _, err := abci_client.GlobalClient.RunBlock(&byteTx, time.Now(), abci_client.GlobalClient.CurState.LastValidators.Proposer) + _, responseCheckTx, responseDeliverTx, _, _, err := abci_client.GlobalClient.RunBlock(&byteTx) if err != nil { return nil, err } diff --git a/cometmock/test_utils/helpers.go b/cometmock/test_utils/helpers.go deleted file mode 100644 index fa3140b..0000000 --- a/cometmock/test_utils/helpers.go +++ /dev/null @@ -1,248 +0,0 @@ -package test_utils - -import ( - "time" - - "github.com/cometbft/cometbft/crypto" - "github.com/cometbft/cometbft/crypto/ed25519" - "github.com/cometbft/cometbft/crypto/tmhash" - cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" - cmtversion "github.com/cometbft/cometbft/proto/tendermint/version" - "github.com/cometbft/cometbft/types" - cmttime "github.com/cometbft/cometbft/types/time" - "github.com/cometbft/cometbft/version" -) - -// file is adapted from https://github.com/cometbft/cometbft/blob/9267594e0a17c01cc4a97b399ada5eaa8a734db5/light/helpers_test.go#L16 - -// privKeys is a helper type for testing. -// -// It lets us simulate signing with many keys. The main use case is to create -// a set, and call GenSignedHeader to get properly signed header for testing. -// -// You can set different weights of validators each time you call ToValidators, -// and can optionally extend the validator set later with Extend. -type privKeys []crypto.PrivKey - -// genPrivKeys produces an array of private keys to generate commits. -func GenPrivKeys(n int) privKeys { - res := make(privKeys, n) - for i := range res { - res[i] = ed25519.GenPrivKey() - } - return res -} - -// // Change replaces the key at index i. -// func (pkz privKeys) Change(i int) privKeys { -// res := make(privKeys, len(pkz)) -// copy(res, pkz) -// res[i] = ed25519.GenPrivKey() -// return res -// } - -// Extend adds n more keys (to remove, just take a slice). -func (pkz privKeys) Extend(n int) privKeys { - extra := GenPrivKeys(n) - return append(pkz, extra...) -} - -// // GenSecpPrivKeys produces an array of secp256k1 private keys to generate commits. -// func GenSecpPrivKeys(n int) privKeys { -// res := make(privKeys, n) -// for i := range res { -// res[i] = secp256k1.GenPrivKey() -// } -// return res -// } - -// // ExtendSecp adds n more secp256k1 keys (to remove, just take a slice). -// func (pkz privKeys) ExtendSecp(n int) privKeys { -// extra := GenSecpPrivKeys(n) -// return append(pkz, extra...) -// } - -// ToValidators produces a valset from the set of keys. -// The first key has weight `init` and it increases by `inc` every step -// so we can have all the same weight, or a simple linear distribution -// (should be enough for testing). -func (pkz privKeys) ToValidators(init, inc int64) *types.ValidatorSet { - res := make([]*types.Validator, len(pkz)) - for i, k := range pkz { - res[i] = types.NewValidator(k.PubKey(), init+int64(i)*inc) - } - return types.NewValidatorSet(res) -} - -// signHeader properly signs the header with all keys from first to last exclusive. -func (pkz privKeys) signHeader(header *types.Header, valSet *types.ValidatorSet, first, last int) *types.Commit { - commitSigs := make([]types.CommitSig, len(pkz)) - for i := 0; i < len(pkz); i++ { - commitSigs[i] = types.NewCommitSigAbsent() - } - - blockID := types.BlockID{ - Hash: header.Hash(), - PartSetHeader: types.PartSetHeader{Total: 1, Hash: crypto.CRandBytes(32)}, - } - - // Fill in the votes we want. - for i := first; i < last && i < len(pkz); i++ { - vote := makeVote(header, valSet, pkz[i], blockID) - commitSigs[vote.ValidatorIndex] = vote.CommitSig() - } - - return types.NewCommit(header.Height, 1, blockID, commitSigs) -} - -func makeVote(header *types.Header, valset *types.ValidatorSet, - key crypto.PrivKey, blockID types.BlockID, -) *types.Vote { - addr := key.PubKey().Address() - idx, _ := valset.GetByAddress(addr) - vote := &types.Vote{ - ValidatorAddress: addr, - ValidatorIndex: idx, - Height: header.Height, - Round: 1, - Timestamp: cmttime.Now(), - Type: cmtproto.PrecommitType, - BlockID: blockID, - } - - v := vote.ToProto() - // Sign it - signBytes := types.VoteSignBytes(header.ChainID, v) - sig, err := key.Sign(signBytes) - if err != nil { - panic(err) - } - - vote.Signature = sig - - return vote -} - -func genHeader(chainID string, height int64, bTime time.Time, txs types.Txs, - valset, nextValset *types.ValidatorSet, appHash, consHash, resHash []byte, -) *types.Header { - return &types.Header{ - Version: cmtversion.Consensus{Block: version.BlockProtocol, App: 0}, - ChainID: chainID, - Height: height, - Time: bTime, - // LastBlockID - // LastCommitHash - ValidatorsHash: valset.Hash(), - NextValidatorsHash: nextValset.Hash(), - DataHash: txs.Hash(), - AppHash: appHash, - ConsensusHash: consHash, - LastResultsHash: resHash, - ProposerAddress: valset.Validators[0].Address, - } -} - -// GenSignedHeader calls genHeader and signHeader and combines them into a SignedHeader. -func (pkz privKeys) GenSignedHeader(chainID string, height int64, bTime time.Time, txs types.Txs, - valset, nextValset *types.ValidatorSet, appHash, consHash, resHash []byte, first, last int, -) *types.SignedHeader { - header := genHeader(chainID, height, bTime, txs, valset, nextValset, appHash, consHash, resHash) - return &types.SignedHeader{ - Header: header, - Commit: pkz.signHeader(header, valset, first, last), - } -} - -// GenSignedHeaderLastBlockID calls genHeader and signHeader and combines them into a SignedHeader. -func (pkz privKeys) GenSignedHeaderLastBlockID(chainID string, height int64, bTime time.Time, txs types.Txs, - valset, nextValset *types.ValidatorSet, appHash, consHash, resHash []byte, first, last int, - lastBlockID types.BlockID, -) *types.SignedHeader { - header := genHeader(chainID, height, bTime, txs, valset, nextValset, appHash, consHash, resHash) - header.LastBlockID = lastBlockID - return &types.SignedHeader{ - Header: header, - Commit: pkz.signHeader(header, valset, first, last), - } -} - -func (pkz privKeys) ChangeKeys(delta int) privKeys { - newKeys := pkz[delta:] - return newKeys.Extend(delta) -} - -// Generates the header and validator set to create a full entire mock node with blocks to height ( -// blockSize) and with variation in validator sets. BlockIntervals are in per minute. -// NOTE: Expected to have a large validator set size ~ 100 validators. -func genMockNodeWithKeys( - chainID string, - blockSize int64, - valSize int, - valVariation float32, - bTime time.Time) ( - map[int64]*types.SignedHeader, - map[int64]*types.ValidatorSet, - map[int64]privKeys, -) { - var ( - headers = make(map[int64]*types.SignedHeader, blockSize) - valset = make(map[int64]*types.ValidatorSet, blockSize+1) - keymap = make(map[int64]privKeys, blockSize+1) - keys = GenPrivKeys(valSize) - totalVariation = valVariation - valVariationInt int - newKeys privKeys - ) - - valVariationInt = int(totalVariation) - totalVariation = -float32(valVariationInt) - newKeys = keys.ChangeKeys(valVariationInt) - keymap[1] = keys - keymap[2] = newKeys - - // genesis header and vals - lastHeader := keys.GenSignedHeader(chainID, 1, bTime.Add(1*time.Minute), nil, - keys.ToValidators(2, 0), newKeys.ToValidators(2, 0), hash("app_hash"), hash("cons_hash"), - hash("results_hash"), 0, len(keys)) - currentHeader := lastHeader - headers[1] = currentHeader - valset[1] = keys.ToValidators(2, 0) - keys = newKeys - - for height := int64(2); height <= blockSize; height++ { - totalVariation += valVariation - valVariationInt = int(totalVariation) - totalVariation = -float32(valVariationInt) - newKeys = keys.ChangeKeys(valVariationInt) - currentHeader = keys.GenSignedHeaderLastBlockID(chainID, height, bTime.Add(time.Duration(height)*time.Minute), - nil, - keys.ToValidators(2, 0), newKeys.ToValidators(2, 0), hash("app_hash"), hash("cons_hash"), - hash("results_hash"), 0, len(keys), types.BlockID{Hash: lastHeader.Hash()}) - headers[height] = currentHeader - valset[height] = keys.ToValidators(2, 0) - lastHeader = currentHeader - keys = newKeys - keymap[height+1] = keys - } - - return headers, valset, keymap -} - -func genMockNode( - chainID string, - blockSize int64, - valSize int, - valVariation float32, - bTime time.Time) ( - string, - map[int64]*types.SignedHeader, - map[int64]*types.ValidatorSet, -) { - headers, valset, _ := genMockNodeWithKeys(chainID, blockSize, valSize, valVariation, bTime) - return chainID, headers, valset -} - -func hash(s string) []byte { - return tmhash.Sum([]byte(s)) -} From 03b86f727415823a2ee1a5a5e922eb9971866133 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 14 Jun 2023 08:37:22 +0200 Subject: [PATCH 07/14] Fix Dockerfile: leading / before file locations --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c426fb..b6449bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV GOFLAGS="-buildvcs=false" # cache gomodules for cometmock ADD ./go.mod /go.mod -ADD ./go.sum go.sum +ADD ./go.sum /go.sum RUN go mod download # Add CometMock and install it From 7c8fff59b5da6257b8b5004da2f501281e583367 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 14 Jun 2023 08:37:51 +0200 Subject: [PATCH 08/14] Add time manipulation --- cometmock/abci_client/client.go | 14 ++++++++++++++ cometmock/rpc_server/routes.go | 16 +++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 3d30df0..ed78888 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -60,6 +60,20 @@ type AbciClient struct { timeOffset time.Duration } +func (a *AbciClient) GetTimeOffset() time.Duration { + return a.timeOffset +} + +func (a *AbciClient) IncrementTimeOffset(additionalOffset time.Duration) error { + if additionalOffset < 0 { + a.Logger.Error("time offset cannot be decremented, please provide a positive offset") + return fmt.Errorf("time offset cannot be decremented, please provide a positive offset") + } + a.Logger.Debug("Incrementing time offset", "additionalOffset", additionalOffset.String()) + a.timeOffset = a.timeOffset + additionalOffset + return nil +} + func (a *AbciClient) GetSigningStatus(address string) (bool, error) { status, ok := a.signingStatus[address] if !ok { diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 58ba950..d8790bc 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -55,16 +55,22 @@ var Routes = map[string]*rpc.RPCFunc{ // cometmock specific API "advance_blocks": rpc.NewRPCFunc(AdvanceBlocks, "num_blocks"), "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "val_cons_address,status"), - "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration"), + "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration_in_seconds"), } -type ResultAdvanceTime struct{} +type ResultAdvanceTime struct { + NewTime time.Time `json:"new_time"` +} // AdvanceTime advances the block time by the given duration. // This API is specific to CometMock. -func AdvanceTime(ctx *rpctypes.Context, duration time.Duration) (*ResultAdvanceTime, error) { - // TODO: Implement the AdvanceTime function - return &ResultAdvanceTime{}, nil +func AdvanceTime(ctx *rpctypes.Context, duration_in_seconds time.Duration) (*ResultAdvanceTime, error) { + if duration_in_seconds < 0 { + return nil, errors.New("duration to advance time by must be greater than 0") + } + + abci_client.GlobalClient.IncrementTimeOffset(duration_in_seconds * time.Second) + return &ResultAdvanceTime{time.Now().Add(abci_client.GlobalClient.GetTimeOffset())}, nil } type ResultSetSigningStatus struct{} From 2303177e8a7c3e4d773897c9c12826cc61c85ff5 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Wed, 14 Jun 2023 12:12:05 +0200 Subject: [PATCH 09/14] Use PrivKeyAddress instead of ValidatorAddress --- cometmock/rpc_server/routes.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index d8790bc..11c26d0 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -54,7 +54,7 @@ var Routes = map[string]*rpc.RPCFunc{ // cometmock specific API "advance_blocks": rpc.NewRPCFunc(AdvanceBlocks, "num_blocks"), - "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "val_cons_address,status"), + "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "privateKeyAddress,status"), "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration_in_seconds"), } @@ -75,12 +75,12 @@ func AdvanceTime(ctx *rpctypes.Context, duration_in_seconds time.Duration) (*Res type ResultSetSigningStatus struct{} -func SetSigningStatus(ctx *rpctypes.Context, valConsAddress string, status string) (*ResultSetSigningStatus, error) { +func SetSigningStatus(ctx *rpctypes.Context, privateKeyAddress string, status string) (*ResultSetSigningStatus, error) { if status != "down" && status != "up" { return nil, errors.New("status must be either `up` to have the validator sign, or `down` to have the validator not sign") } - err := abci_client.GlobalClient.SetSigningStatus(valConsAddress, status == "up") + err := abci_client.GlobalClient.SetSigningStatus(privateKeyAddress, status == "up") return &ResultSetSigningStatus{}, err } From 4713c1c6ee44412a1cd9417389252b79e183d753 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Thu, 15 Jun 2023 09:53:31 +0200 Subject: [PATCH 10/14] Add mutexes for storage --- cometmock/storage/storage.go | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/cometmock/storage/storage.go b/cometmock/storage/storage.go index 38ab2e7..b12f8f8 100644 --- a/cometmock/storage/storage.go +++ b/cometmock/storage/storage.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "sync" protostate "github.com/cometbft/cometbft/proto/tendermint/state" cometstate "github.com/cometbft/cometbft/state" @@ -39,16 +40,22 @@ type Storage interface { // MapStorage is a simple in-memory implementation of Storage. type MapStorage struct { - blocks map[int64]*types.Block - commits map[int64]*types.Commit - states map[int64]*cometstate.State - responses map[int64]*protostate.ABCIResponses + blocks map[int64]*types.Block + blocksMutex sync.RWMutex + commits map[int64]*types.Commit + commitMutex sync.RWMutex + states map[int64]*cometstate.State + statesMutex sync.RWMutex + responses map[int64]*protostate.ABCIResponses + responsesMutex sync.RWMutex } // ensure MapStorage implements Storage var _ Storage = (*MapStorage)(nil) func (m *MapStorage) InsertBlock(height int64, block *types.Block) error { + m.blocksMutex.Lock() + defer m.blocksMutex.Unlock() if m.blocks == nil { m.blocks = make(map[int64]*types.Block) } @@ -57,6 +64,9 @@ func (m *MapStorage) InsertBlock(height int64, block *types.Block) error { } func (m *MapStorage) GetBlock(height int64) (*types.Block, error) { + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + if m.blocks == nil { m.blocks = make(map[int64]*types.Block) } @@ -67,6 +77,9 @@ func (m *MapStorage) GetBlock(height int64) (*types.Block, error) { } func (m *MapStorage) InsertCommit(height int64, commit *types.Commit) error { + m.commitMutex.Lock() + defer m.commitMutex.Unlock() + if m.commits == nil { m.commits = make(map[int64]*types.Commit) } @@ -76,6 +89,9 @@ func (m *MapStorage) InsertCommit(height int64, commit *types.Commit) error { } func (m *MapStorage) GetCommit(height int64) (*types.Commit, error) { + m.commitMutex.RLock() + defer m.commitMutex.RUnlock() + if m.commits == nil { m.commits = make(map[int64]*types.Commit) } @@ -87,6 +103,9 @@ func (m *MapStorage) GetCommit(height int64) (*types.Commit, error) { } func (m *MapStorage) InsertState(height int64, state *cometstate.State) error { + m.statesMutex.Lock() + defer m.statesMutex.Unlock() + if m.states == nil { m.states = make(map[int64]*cometstate.State) } @@ -96,6 +115,9 @@ func (m *MapStorage) InsertState(height int64, state *cometstate.State) error { } func (m *MapStorage) GetState(height int64) (*cometstate.State, error) { + m.statesMutex.RLock() + defer m.statesMutex.RUnlock() + if m.states == nil { m.states = make(map[int64]*cometstate.State) } @@ -107,6 +129,9 @@ func (m *MapStorage) GetState(height int64) (*cometstate.State, error) { } func (m *MapStorage) InsertResponses(height int64, responses *protostate.ABCIResponses) error { + m.responsesMutex.Lock() + defer m.responsesMutex.Unlock() + if m.responses == nil { m.responses = make(map[int64]*protostate.ABCIResponses) } @@ -116,6 +141,9 @@ func (m *MapStorage) InsertResponses(height int64, responses *protostate.ABCIRes } func (m *MapStorage) GetResponses(height int64) (*protostate.ABCIResponses, error) { + m.responsesMutex.RLock() + defer m.responsesMutex.RUnlock() + if m.responses == nil { m.responses = make(map[int64]*protostate.ABCIResponses) } From 730fe6adb8992d03104e49fc6d7810646fd805ec Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Thu, 15 Jun 2023 09:59:12 +0200 Subject: [PATCH 11/14] Add mutexes to concurrently accessed maps --- cometmock/abci_client/client.go | 16 ++++++++++++++-- cometmock/storage/storage.go | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index ed78888..7fc567a 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -52,7 +52,8 @@ type AbciClient struct { ErrorOnUnequalResponses bool // validator addresses are mapped to false if they should not be signing, and to true if they should - signingStatus map[string]bool + signingStatus map[string]bool + signingStatusMutex sync.RWMutex // time offset. whenever we qury the time, we add this offset to it // this means after modifying this, blocks will have the timestamp offset by this value. @@ -75,6 +76,9 @@ func (a *AbciClient) IncrementTimeOffset(additionalOffset time.Duration) error { } func (a *AbciClient) GetSigningStatus(address string) (bool, error) { + a.signingStatusMutex.RLock() + defer a.signingStatusMutex.RUnlock() + status, ok := a.signingStatus[address] if !ok { return false, fmt.Errorf("address %s not found in signing status map, please double-check this is the key address of a validator key", address) @@ -83,6 +87,9 @@ func (a *AbciClient) GetSigningStatus(address string) (bool, error) { } func (a *AbciClient) SetSigningStatus(address string, status bool) error { + a.signingStatusMutex.Lock() + defer a.signingStatusMutex.Unlock() + _, ok := a.signingStatus[address] if !ok { return fmt.Errorf("address %s not found in signing status map, please double-check this is the key address of a validator key", address) @@ -593,7 +600,12 @@ func (a *AbciClient) RunBlockWithTimeAndProposer(tx *[]byte, blockTime time.Time for index, val := range a.CurState.Validators.Validators { privVal := a.PrivValidators[val.Address.String()] - if a.signingStatus[val.Address.String()] { + shouldSign, err := a.GetSigningStatus(val.Address.String()) + if err != nil { + return nil, nil, nil, nil, nil, err + } + + if shouldSign { // create and sign a precommit vote := &cmttypes.Vote{ ValidatorAddress: val.Address, diff --git a/cometmock/storage/storage.go b/cometmock/storage/storage.go index b12f8f8..2151d22 100644 --- a/cometmock/storage/storage.go +++ b/cometmock/storage/storage.go @@ -10,6 +10,7 @@ import ( ) // Storage is an interface for storing blocks, commits and states by height. +// All methods are thread-safe. type Storage interface { // InsertBlock inserts a block at a given height. // If there is already a block at that height, it should be overwritten. From 3f50bf465c402f5a8b05a50b2e100c094757780d Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Thu, 15 Jun 2023 14:51:06 +0200 Subject: [PATCH 12/14] Return signing status map for rpc call --- cometmock/abci_client/client.go | 13 +++++++++++++ cometmock/rpc_server/routes.go | 10 +++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 7fc567a..1384419 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -75,6 +75,19 @@ func (a *AbciClient) IncrementTimeOffset(additionalOffset time.Duration) error { return nil } +// GetSigningStatusMap gets a copy of the signing status map that can be used for reading. +func (a *AbciClient) GetSigningStatusMap() map[string]bool { + a.signingStatusMutex.RLock() + defer a.signingStatusMutex.RUnlock() + + statusMap := make(map[string]bool, len(a.signingStatus)) + for k, v := range a.signingStatus { + statusMap[k] = v + } + return statusMap +} + +// GetSigningStatus gets the signing status of the given address. func (a *AbciClient) GetSigningStatus(address string) (bool, error) { a.signingStatusMutex.RLock() defer a.signingStatusMutex.RUnlock() diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 11c26d0..92a35e8 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -54,7 +54,7 @@ var Routes = map[string]*rpc.RPCFunc{ // cometmock specific API "advance_blocks": rpc.NewRPCFunc(AdvanceBlocks, "num_blocks"), - "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "privateKeyAddress,status"), + "set_signing_status": rpc.NewRPCFunc(SetSigningStatus, "private_key_address,status"), "advance_time": rpc.NewRPCFunc(AdvanceTime, "duration_in_seconds"), } @@ -73,7 +73,9 @@ func AdvanceTime(ctx *rpctypes.Context, duration_in_seconds time.Duration) (*Res return &ResultAdvanceTime{time.Now().Add(abci_client.GlobalClient.GetTimeOffset())}, nil } -type ResultSetSigningStatus struct{} +type ResultSetSigningStatus struct { + NewSigningStatusMap map[string]bool `json:"new_signing_status_map"` +} func SetSigningStatus(ctx *rpctypes.Context, privateKeyAddress string, status string) (*ResultSetSigningStatus, error) { if status != "down" && status != "up" { @@ -82,7 +84,9 @@ func SetSigningStatus(ctx *rpctypes.Context, privateKeyAddress string, status st err := abci_client.GlobalClient.SetSigningStatus(privateKeyAddress, status == "up") - return &ResultSetSigningStatus{}, err + return &ResultSetSigningStatus{ + NewSigningStatusMap: abci_client.GlobalClient.GetSigningStatusMap(), + }, err } type ResultAdvanceBlocks struct{} From f6c898d05b1bd1fb18bf5d507eec4e01b48e2a7a Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 16 Jun 2023 13:50:41 +0200 Subject: [PATCH 13/14] Add sleep before consumer start --- local-testnet-debug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-testnet-debug.sh b/local-testnet-debug.sh index 581bbc6..4ebd867 100755 --- a/local-testnet-debug.sh +++ b/local-testnet-debug.sh @@ -247,7 +247,7 @@ do interchain-security-pd tx gov vote 1 yes --from $PROV_KEY --chain-id provider --home ${PROV_NODE_DIR} --node $PROVIDER_COMETMOCK_ADDR -b block -y --keyring-backend test done -# sleep 3 +sleep 3 # # ## CONSUMER CHAIN ## From 9f431396edf52e128c9701fa2e0ae0a755c9c8d0 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 16 Jun 2023 14:33:24 +0200 Subject: [PATCH 14/14] Last block signer info needs to take last block val set --- cometmock/abci_client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 1384419..25701d0 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -293,7 +293,7 @@ func (a *AbciClient) CreateBeginBlockRequest(header *types.Header, lastCommit *t voteInfos := make([]abcitypes.VoteInfo, len(commitSigs)) if lastCommit.Height != 0 { for i := range commitSigs { - val := a.CurState.Validators.Validators[i] + val := a.CurState.LastValidators.Validators[i] byteAddress := val.Address.Bytes() abciVal := abcitypes.Validator{