From fbc390c39c28940efea9fef392704991ce2f6ac7 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 25 May 2020 16:39:44 +0900 Subject: [PATCH 01/38] refactor: rename VotingPower to StakingPower --- abci/types/types.pb.go | 2 +- blockchain/v0/reactor_test.go | 2 +- blockchain/v1/reactor_test.go | 2 +- blockchain/v2/reactor_test.go | 2 +- consensus/common_test.go | 2 +- consensus/state.go | 6 +- evidence/pool.go | 2 +- evidence/pool_test.go | 2 +- rpc/client/rpc_test.go | 2 +- rpc/core/status.go | 10 ++-- rpc/core/types/responses.go | 6 +- state/execution_test.go | 4 +- state/helpers_test.go | 2 +- state/state.go | 4 +- state/state_test.go | 32 +++++----- types/protobuf.go | 10 ++-- types/validator.go | 26 ++++----- types/validator_set.go | 106 +++++++++++++++++----------------- types/validator_set_test.go | 88 ++++++++++++++-------------- types/vote_set.go | 26 ++++----- types/voter_set.go | 28 ++++----- types/voter_set_test.go | 12 ++-- 22 files changed, 188 insertions(+), 188 deletions(-) diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go index 05f7d5063..c01f052e0 100644 --- a/abci/types/types.pb.go +++ b/abci/types/types.pb.go @@ -15525,7 +15525,7 @@ func (m *Evidence) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TotalVotingPower", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TotalStakingPower", wireType) } m.TotalVotingPower = 0 for shift := uint(0); ; shift += 7 { diff --git a/blockchain/v0/reactor_test.go b/blockchain/v0/reactor_test.go index 4c5bb394a..314a2bfb9 100644 --- a/blockchain/v0/reactor_test.go +++ b/blockchain/v0/reactor_test.go @@ -32,7 +32,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G val, privVal := types.RandValidator(randPower, minPower) validators[i] = types.GenesisValidator{ PubKey: val.PubKey, - Power: val.VotingPower, + Power: val.StakingPower, } privValidators[i] = privVal } diff --git a/blockchain/v1/reactor_test.go b/blockchain/v1/reactor_test.go index da4d6064a..f88af51a3 100644 --- a/blockchain/v1/reactor_test.go +++ b/blockchain/v1/reactor_test.go @@ -32,7 +32,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G val, privVal := types.RandValidator(randPower, minPower) validators[i] = types.GenesisValidator{ PubKey: val.PubKey, - Power: val.VotingPower, + Power: val.StakingPower, } privValidators[i] = privVal } diff --git a/blockchain/v2/reactor_test.go b/blockchain/v2/reactor_test.go index 88ca0e1de..b1d2ef09b 100644 --- a/blockchain/v2/reactor_test.go +++ b/blockchain/v2/reactor_test.go @@ -445,7 +445,7 @@ func randGenesisDoc(chainID string, numValidators int, randPower bool, minPower val, privVal := types.RandValidator(randPower, minPower) validators[i] = types.GenesisValidator{ PubKey: val.PubKey, - Power: val.VotingPower, + Power: val.StakingPower, } privValidators[i] = privVal } diff --git a/consensus/common_test.go b/consensus/common_test.go index 9c079f178..fb3a890ea 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -778,7 +778,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G val, privVal := types.RandValidator(randPower, minPower) validators[i] = types.GenesisValidator{ PubKey: val.PubKey, - Power: val.VotingPower, + Power: val.StakingPower, } privValidators[i] = privVal } diff --git a/consensus/state.go b/consensus/state.go index 23ba3e477..d04ff43e8 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1491,14 +1491,14 @@ func (cs *State) recordMetrics(height int64, block *types.Block) { commitSig := block.LastCommit.Signatures[i] if commitSig.Absent() { missingVoters++ - missingVotersPower += val.VotingPower + missingVotersPower += val.StakingPower } if cs.privValidator != nil && bytes.Equal(val.Address, cs.privValidator.GetPubKey().Address()) { label := []string{ "validator_address", val.Address.String(), } - cs.metrics.VoterPower.With(label...).Set(float64(val.VotingPower)) + cs.metrics.VoterPower.With(label...).Set(float64(val.StakingPower)) if commitSig.ForBlock() { cs.metrics.VoterLastSignedHeight.With(label...).Set(float64(height)) } else { @@ -1514,7 +1514,7 @@ func (cs *State) recordMetrics(height int64, block *types.Block) { byzantineVotersPower := int64(0) for _, ev := range block.Evidence.Evidence { if _, val := cs.Voters.GetByAddress(ev.Address()); val != nil { - byzantineVotersPower += val.VotingPower + byzantineVotersPower += val.StakingPower } } cs.metrics.ByzantineVotersPower.Set(float64(byzantineVotersPower)) diff --git a/evidence/pool.go b/evidence/pool.go index d2e998f10..4ef6425ab 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -108,7 +108,7 @@ func (evpool *Pool) AddEvidence(evidence types.Evidence) (err error) { // TODO: something better ? valSet, _, _ := sm.LoadValidators(evpool.stateDB, evidence.Height()) _, val := valSet.GetByAddress(evidence.Address()) - priority := val.VotingPower + priority := val.StakingPower added := evpool.store.AddNewEvidence(evidence, priority) if !added { diff --git a/evidence/pool_test.go b/evidence/pool_test.go index 7224f9d17..22c631250 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -26,7 +26,7 @@ func initializeValidatorState(valAddr []byte, height int64) dbm.DB { // create validator set and state vals := []*types.Validator{ - {Address: valAddr, VotingPower: 1}, + {Address: valAddr, StakingPower: 1}, } state := sm.State{ LastBlockHeight: 0, diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 6b506fbb5..51b5fb86b 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -177,7 +177,7 @@ func TestGenesisAndValidators(t *testing.T) { val := vals.Voters[0] // make sure the current set is also the genesis set - assert.Equal(t, gval.Power, val.VotingPower) + assert.Equal(t, gval.Power, val.StakingPower) assert.Equal(t, gval.PubKey, val.PubKey) } } diff --git a/rpc/core/status.go b/rpc/core/status.go index 9d4fb201e..3cf5664c1 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -38,9 +38,9 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) { latestBlockTime := time.Unix(0, latestBlockTimeNano) - var votingPower int64 + var stakingPower int64 if val := validatorAtHeight(latestHeight); val != nil { - votingPower = val.VotingPower + stakingPower = val.StakingPower } result := &ctypes.ResultStatus{ @@ -53,9 +53,9 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) { CatchingUp: consensusReactor.FastSync(), }, ValidatorInfo: ctypes.ValidatorInfo{ - Address: pubKey.Address(), - PubKey: pubKey, - VotingPower: votingPower, + Address: pubKey.Address(), + PubKey: pubKey, + StakingPower: stakingPower, }, } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 5d6201a25..94657348c 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -70,9 +70,9 @@ type SyncInfo struct { // Info about the node's validator type ValidatorInfo struct { - Address bytes.HexBytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - VotingPower int64 `json:"voting_power"` + Address bytes.HexBytes `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + StakingPower int64 `json:"staking_power"` } // Node Status diff --git a/state/execution_test.go b/state/execution_test.go index 9f8bfd504..94151cfd4 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -310,7 +310,7 @@ func TestUpdateValidators(t *testing.T) { assert.NoError(t, err) require.Equal(t, tc.resultingSet.Size(), tc.currentSet.Size()) - assert.Equal(t, tc.resultingSet.TotalVotingPower(), tc.currentSet.TotalVotingPower()) + assert.Equal(t, tc.resultingSet.TotalStakingPower(), tc.currentSet.TotalStakingPower()) assert.Equal(t, tc.resultingSet.Validators[0].Address, tc.currentSet.Validators[0].Address) if tc.resultingSet.Size() > 1 { @@ -379,7 +379,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) { require.True(t, ok, "Expected event of type EventDataValidatorSetUpdates, got %T", msg.Data()) if assert.NotEmpty(t, event.ValidatorUpdates) { assert.Equal(t, pubkey, event.ValidatorUpdates[0].PubKey) - assert.EqualValues(t, 10, event.ValidatorUpdates[0].VotingPower) + assert.EqualValues(t, 10, event.ValidatorUpdates[0].StakingPower) } case <-updatesSub.Cancelled(): t.Fatalf("updatesSub was cancelled (reason: %v)", updatesSub.Err()) diff --git a/state/helpers_test.go b/state/helpers_test.go index e38038954..d83b5a56e 100644 --- a/state/helpers_test.go +++ b/state/helpers_test.go @@ -221,7 +221,7 @@ func makeHeaderPartsResponsesValPowerChange( // If the pubkey is new, remove the old and add the new. _, val := state.NextValidators.GetByIndex(0) - if val.VotingPower != power { + if val.StakingPower != power { abciResponses.EndBlock = &abci.ResponseEndBlock{ ValidatorUpdates: []abci.ValidatorUpdate{ types.TM2PB.NewValidatorUpdate(val.PubKey, power), diff --git a/state/state.go b/state/state.go index d4315c40f..45c66f19d 100644 --- a/state/state.go +++ b/state/state.go @@ -192,8 +192,8 @@ func MedianTime(commit *types.Commit, voters *types.VoterSet) time.Time { _, validator := voters.GetByAddress(commitSig.ValidatorAddress) // If there's no condition, TestValidateBlockCommit panics; not needed normally. if validator != nil { - totalVotingPower += validator.VotingPower - weightedTimes[i] = tmtime.NewWeightedTime(commitSig.Timestamp, validator.VotingPower) + totalVotingPower += validator.StakingPower + weightedTimes[i] = tmtime.NewWeightedTime(commitSig.Timestamp, validator.StakingPower) } } diff --git a/state/state_test.go b/state/state_test.go index f3b180b2f..dee9a4d6e 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -224,7 +224,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { highestHeight := changeHeights[N-1] + 5 changeIndex := 0 _, val := state.Validators.GetByIndex(0) - power := val.VotingPower + power := val.StakingPower var err error var validatorUpdates []*types.Validator for i := int64(1); i < highestHeight; i++ { @@ -246,7 +246,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { // On each height change, increment the power by one. testCases := make([]int64, highestHeight) changeIndex = 0 - power = val.VotingPower + power = val.StakingPower for i := int64(1); i < highestHeight+1; i++ { // We get to the height after a change height use the next pubkey (note // our counter starts at 0 this time). @@ -263,7 +263,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { assert.Equal(t, v.Size(), 1, "validator set size is greater than 1: %d", v.Size()) _, val := v.GetByIndex(0) - assert.Equal(t, val.VotingPower, power, fmt.Sprintf(`unexpected powerat + assert.Equal(t, val.StakingPower, power, fmt.Sprintf(`unexpected powerat height %d`, i)) } } @@ -377,7 +377,7 @@ func testProposerFreq(t *testing.T, caseNum int, valSet *types.ValidatorSet) { // assert frequencies match expected (max off by 1) for i, freq := range freqs { _, val := valSet.GetByIndex(i) - expectFreq := int(val.VotingPower) * runMult + expectFreq := int(val.StakingPower) * runMult gotFreq := freq abs := int(math.Abs(float64(expectFreq - gotFreq))) @@ -401,7 +401,7 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { defer tearDown(t) val1VotingPower := int64(10) val1PubKey := ed25519.GenPrivKey().PubKey() - val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, VotingPower: val1VotingPower} + val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, StakingPower: val1VotingPower} state.Validators = types.NewValidatorSet([]*types.Validator{val1}) state.NextValidators = state.Validators @@ -459,7 +459,7 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { assert.Equal(t, wantVal2Prio, addedVal2.ProposerPriority) // Updating a validator does not reset the ProposerPriority to zero: - // 1. Add - Val2 VotingPower change to 1 => + // 1. Add - Val2 StakingPower change to 1 => updatedVotingPowVal2 := int64(1) updateVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: updatedVotingPowVal2} validatorUpdates, err = types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{updateVal}) @@ -482,7 +482,7 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { wantVal2Prio = prevVal2.ProposerPriority // scale to diffMax = 22 = 2 * tvp, diff=39-(-38)=77 // new totalPower - totalPower := updatedVal1.VotingPower + updatedVal2.VotingPower + totalPower := updatedVal1.StakingPower + updatedVal2.StakingPower dist := wantVal2Prio - wantVal1Prio // ratio := (dist + 2*totalPower - 1) / 2*totalPower = 98/22 = 4 ratio := (dist + 2*totalPower - 1) / (2 * totalPower) @@ -494,9 +494,9 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { // 4. IncrementProposerPriority() -> // v1(10):-9+10, v2(1):9+1 -> v2 proposer so subsract tvp(11) // v1(10):1, v2(1):-1 - wantVal2Prio += updatedVal2.VotingPower // 10 -> prop - wantVal1Prio += updatedVal1.VotingPower // 1 - wantVal2Prio -= totalPower // -1 + wantVal2Prio += updatedVal2.StakingPower // 10 -> prop + wantVal1Prio += updatedVal1.StakingPower // 1 + wantVal2Prio -= totalPower // -1 assert.Equal(t, wantVal2Prio, updatedVal2.ProposerPriority) assert.Equal(t, wantVal1Prio, updatedVal1.ProposerPriority) @@ -512,7 +512,7 @@ func TestProposerPriorityProposerAlternates(t *testing.T) { defer tearDown(t) val1VotingPower := int64(10) val1PubKey := ed25519.GenPrivKey().PubKey() - val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, VotingPower: val1VotingPower} + val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, StakingPower: val1VotingPower} // reset state validators to above validator state.Validators = types.NewValidatorSet([]*types.Validator{val1}) @@ -679,13 +679,13 @@ func TestLargeGenesisValidator(t *testing.T) { tearDown, _, state := setupTestCase(t) defer tearDown(t) - genesisVotingPower := types.MaxTotalVotingPower / 1000 + genesisVotingPower := types.MaxTotalStakingPower / 1000 genesisPubKey := ed25519.GenPrivKey().PubKey() // fmt.Println("genesis addr: ", genesisPubKey.Address()) genesisVal := &types.Validator{ - Address: genesisPubKey.Address(), - PubKey: genesisPubKey, - VotingPower: genesisVotingPower, + Address: genesisPubKey.Address(), + PubKey: genesisPubKey, + StakingPower: genesisVotingPower, } // reset state validators to above validator state.Validators = types.NewValidatorSet([]*types.Validator{genesisVal}) @@ -708,7 +708,7 @@ func TestLargeGenesisValidator(t *testing.T) { updatedState, err := sm.UpdateState(oldState, blockID, &block.Header, abciResponses, validatorUpdates) require.NoError(t, err) - // no changes in voting power (ProposerPrio += VotingPower == Voting in 1st round; than shiftByAvg == 0, + // no changes in voting power (ProposerPrio += StakingPower == Voting in 1st round; than shiftByAvg == 0, // than -Total == -Voting) // -> no change in ProposerPrio (stays zero): assert.EqualValues(t, oldState.NextValidators, updatedState.NextValidators) diff --git a/types/protobuf.go b/types/protobuf.go index 5ff70936f..2cf53225b 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -71,7 +71,7 @@ func (tm2pb) Header(header *Header) abci.Header { func (tm2pb) Validator(val *Validator) abci.Validator { return abci.Validator{ Address: val.PubKey.Address(), - Power: val.VotingPower, + Power: val.StakingPower, } } @@ -93,7 +93,7 @@ func (tm2pb) PartSetHeader(header PartSetHeader) abci.PartSetHeader { func (tm2pb) ValidatorUpdate(val *Validator) abci.ValidatorUpdate { return abci.ValidatorUpdate{ PubKey: TM2PB.PubKey(val.PubKey), - Power: val.VotingPower, + Power: val.StakingPower, } } @@ -149,8 +149,8 @@ func (tm2pb) ConsensusParams(params *ConsensusParams) *abci.ConsensusParams { // ABCI Evidence includes information from the past that's not included in the evidence itself // so Evidence types stays compact. // XXX: panics on nil or unknown pubkey type -func (tm2pb) Evidence(ev Evidence, valSet *VoterSet, evTime time.Time) abci.Evidence { - _, val := valSet.GetByAddress(ev.Address()) +func (tm2pb) Evidence(ev Evidence, voterSet *VoterSet, evTime time.Time) abci.Evidence { + _, val := voterSet.GetByAddress(ev.Address()) if val == nil { // should already have checked this panic(val) @@ -173,7 +173,7 @@ func (tm2pb) Evidence(ev Evidence, valSet *VoterSet, evTime time.Time) abci.Evid Validator: TM2PB.Validator(val), Height: ev.Height(), Time: evTime, - TotalVotingPower: valSet.TotalVotingPower(), + TotalVotingPower: voterSet.TotalVotingPower(), } } diff --git a/types/validator.go b/types/validator.go index c3cadc4d3..5d99a342d 100644 --- a/types/validator.go +++ b/types/validator.go @@ -13,18 +13,18 @@ import ( // NOTE: The ProposerPriority is not included in Validator.Hash(); // make sure to update that method if changes are made here type Validator struct { - Address Address `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - VotingPower int64 `json:"voting_power"` + Address Address `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + StakingPower int64 `json:"staking_power"` ProposerPriority int64 `json:"proposer_priority"` } -func NewValidator(pubKey crypto.PubKey, votingPower int64) *Validator { +func NewValidator(pubKey crypto.PubKey, stakingPower int64) *Validator { return &Validator{ Address: pubKey.Address(), PubKey: pubKey, - VotingPower: votingPower, + StakingPower: stakingPower, ProposerPriority: 0, } } @@ -66,7 +66,7 @@ func (v *Validator) String() string { return fmt.Sprintf("Validator{%v %v VP:%v A:%v}", v.Address, v.PubKey, - v.VotingPower, + v.StakingPower, v.ProposerPriority) } @@ -74,7 +74,7 @@ func (v *Validator) String() string { func ValidatorListString(vals []*Validator) string { chunks := make([]string, len(vals)) for i, val := range vals { - chunks[i] = fmt.Sprintf("%s:%d", val.Address, val.VotingPower) + chunks[i] = fmt.Sprintf("%s:%d", val.Address, val.StakingPower) } return strings.Join(chunks, ",") @@ -86,11 +86,11 @@ func ValidatorListString(vals []*Validator) string { // which changes every round. func (v *Validator) Bytes() []byte { return cdcEncode(struct { - PubKey crypto.PubKey - VotingPower int64 + PubKey crypto.PubKey + StakingPower int64 }{ v.PubKey, - v.VotingPower, + v.StakingPower, }) } @@ -101,11 +101,11 @@ func (v *Validator) Bytes() []byte { // UNSTABLE func RandValidator(randPower bool, minPower int64) (*Validator, PrivValidator) { privVal := NewMockPV() - votePower := minPower + stakingPower := minPower if randPower { - votePower += int64(tmrand.Uint32()) + stakingPower += int64(tmrand.Uint32()) } pubKey := privVal.GetPubKey() - val := NewValidator(pubKey, votePower) + val := NewValidator(pubKey, stakingPower) return val, privVal } diff --git a/types/validator_set.go b/types/validator_set.go index a902348fa..4185379bc 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -14,14 +14,14 @@ import ( ) const ( - // MaxTotalVotingPower - the maximum allowed total voting power. + // MaxTotalStakingPower - the maximum allowed total voting power. // It needs to be sufficiently small to, in all cases: // 1. prevent clipping in incrementProposerPriority() // 2. let (diff+diffMax-1) not overflow in IncrementProposerPriority() // (Proof of 1 is tricky, left to the reader). // It could be higher, but this is sufficiently large for our purposes, // and leaves room for defensive purposes. - MaxTotalVotingPower = int64(math.MaxInt64) / 8 + MaxTotalStakingPower = int64(math.MaxInt64) / 8 // PriorityWindowSizeFactor - is a constant that when multiplied with the total voting power gives // the maximum allowed distance between validator priorities. @@ -43,7 +43,7 @@ type ValidatorSet struct { Validators []*Validator `json:"validators"` // cached (unexported) - totalVotingPower int64 + totalStakingPower int64 } // NewValidatorSet initializes a VoterSet by copying over the @@ -75,7 +75,7 @@ func (vals *ValidatorSet) CopyIncrementProposerPriority(times int) *ValidatorSet return copy } -// TODO The current random selection by VRF uses VotingPower, so the processing on ProposerPriority can be removed, +// TODO The current random selection by VRF uses StakingPower, so the processing on ProposerPriority can be removed, // TODO but it remains for later verification of random selection based on ProposerPriority. // IncrementProposerPriority increments ProposerPriority of each validator and updates the // proposer. Panics if validator set is empty. @@ -90,8 +90,8 @@ func (vals *ValidatorSet) IncrementProposerPriority(times int) { // Cap the difference between priorities to be proportional to 2*totalPower by // re-normalizing priorities, i.e., rescale all priorities by multiplying with: - // 2*totalVotingPower/(maxPriority - minPriority) - diffMax := PriorityWindowSizeFactor * vals.TotalVotingPower() + // 2*totalStakingPower/(maxPriority - minPriority) + diffMax := PriorityWindowSizeFactor * vals.TotalStakingPower() vals.RescalePriorities(diffMax) vals.shiftByAvgProposerPriority() @@ -129,13 +129,13 @@ func (vals *ValidatorSet) RescalePriorities(diffMax int64) { func (vals *ValidatorSet) incrementProposerPriority() *Validator { for _, val := range vals.Validators { // Check for overflow for sum. - newPrio := safeAddClip(val.ProposerPriority, val.VotingPower) + newPrio := safeAddClip(val.ProposerPriority, val.StakingPower) val.ProposerPriority = newPrio } // Decrement the validator with most ProposerPriority. mostest := vals.getValWithMostPriority() // Mind the underflow. - mostest.ProposerPriority = safeSubClip(mostest.ProposerPriority, vals.TotalVotingPower()) + mostest.ProposerPriority = safeSubClip(mostest.ProposerPriority, vals.TotalStakingPower()) return mostest } @@ -211,8 +211,8 @@ func validatorListCopy(valsList []*Validator) []*Validator { // Copy each validator into a new VoterSet. func (vals *ValidatorSet) Copy() *ValidatorSet { return &ValidatorSet{ - Validators: validatorListCopy(vals.Validators), - totalVotingPower: vals.totalVotingPower, + Validators: validatorListCopy(vals.Validators), + totalStakingPower: vals.totalStakingPower, } } @@ -253,32 +253,32 @@ func (vals *ValidatorSet) Size() int { return len(vals.Validators) } -// Forces recalculation of the set's total voting power. -// Panics if total voting power is bigger than MaxTotalVotingPower. -func (vals *ValidatorSet) updateTotalVotingPower() { +// Forces recalculation of the set's total staking power. +// Panics if total voting power is bigger than MaxTotalStakingPower. +func (vals *ValidatorSet) updateTotalStakingPower() { sum := int64(0) for _, val := range vals.Validators { // mind overflow - sum = safeAddClip(sum, val.VotingPower) - if sum > MaxTotalVotingPower { + sum = safeAddClip(sum, val.StakingPower) + if sum > MaxTotalStakingPower { panic(fmt.Sprintf( "Total voting power should be guarded to not exceed %v; got: %v", - MaxTotalVotingPower, + MaxTotalStakingPower, sum)) } } - vals.totalVotingPower = sum + vals.totalStakingPower = sum } -// TotalVotingPower returns the sum of the voting powers of all validators. +// TotalStakingPower returns the sum of the voting powers of all validators. // It recomputes the total voting power if required. -func (vals *ValidatorSet) TotalVotingPower() int64 { - if vals.totalVotingPower == 0 { - vals.updateTotalVotingPower() +func (vals *ValidatorSet) TotalStakingPower() int64 { + if vals.totalStakingPower == 0 { + vals.updateTotalStakingPower() } - return vals.totalVotingPower + return vals.totalStakingPower } // Hash returns the Merkle root hash build using validators (as leaves) in the @@ -329,14 +329,14 @@ func processChanges(origChanges []*Validator) (updates, removals []*Validator, e } switch { - case valUpdate.VotingPower < 0: - err = fmt.Errorf("voting power can't be negative: %d", valUpdate.VotingPower) + case valUpdate.StakingPower < 0: + err = fmt.Errorf("voting power can't be negative: %d", valUpdate.StakingPower) return nil, nil, err - case valUpdate.VotingPower > MaxTotalVotingPower: + case valUpdate.StakingPower > MaxTotalStakingPower: err = fmt.Errorf("to prevent clipping/overflow, voting power can't be higher than %d, got %d", - MaxTotalVotingPower, valUpdate.VotingPower) + MaxTotalStakingPower, valUpdate.StakingPower) return nil, nil, err - case valUpdate.VotingPower == 0: + case valUpdate.StakingPower == 0: removals = append(removals, valUpdate) default: updates = append(updates, valUpdate) @@ -359,7 +359,7 @@ func processChanges(origChanges []*Validator) (updates, removals []*Validator, e // // Returns: // tvpAfterUpdatesBeforeRemovals - the new total voting power if these updates would be applied without the removals. -// Note that this will be < 2 * MaxTotalVotingPower in case high power validators are removed and +// Note that this will be < 2 * MaxTotalStakingPower in case high power validators are removed and // validators are added/ updated with high power values. // // err - non-nil if the maximum allowed total voting power would be exceeded @@ -371,9 +371,9 @@ func verifyUpdates( delta := func(update *Validator, vals *ValidatorSet) int64 { _, val := vals.GetByAddress(update.Address) if val != nil { - return update.VotingPower - val.VotingPower + return update.StakingPower - val.StakingPower } - return update.VotingPower + return update.StakingPower } updatesCopy := validatorListCopy(updates) @@ -381,13 +381,13 @@ func verifyUpdates( return delta(updatesCopy[i], vals) < delta(updatesCopy[j], vals) }) - tvpAfterRemovals := vals.TotalVotingPower() - removedPower + tvpAfterRemovals := vals.TotalStakingPower() - removedPower for _, upd := range updatesCopy { tvpAfterRemovals += delta(upd, vals) - if tvpAfterRemovals > MaxTotalVotingPower { + if tvpAfterRemovals > MaxTotalStakingPower { err = fmt.Errorf( "failed to add/update validator %v, total voting power would exceed the max allowed %v", - upd.Address, MaxTotalVotingPower) + upd.Address, MaxTotalStakingPower) return 0, err } } @@ -405,31 +405,31 @@ func numNewValidators(updates []*Validator, vals *ValidatorSet) int { } // computeNewPriorities computes the proposer priority for the validators not present in the set based on -// 'updatedTotalVotingPower'. +// 'updatedTotalStakingPower'. // Leaves unchanged the priorities of validators that are changed. // // 'updates' parameter must be a list of unique validators to be added or updated. // -// 'updatedTotalVotingPower' is the total voting power of a set where all updates would be applied but -// not the removals. It must be < 2*MaxTotalVotingPower and may be close to this limit if close to -// MaxTotalVotingPower will be removed. This is still safe from overflow since MaxTotalVotingPower is maxInt64/8. +// 'updatedTotalStakingPower' is the total voting power of a set where all updates would be applied but +// not the removals. It must be < 2*MaxTotalStakingPower and may be close to this limit if close to +// MaxTotalStakingPower will be removed. This is still safe from overflow since MaxTotalStakingPower is maxInt64/8. // // No changes are made to the validator set 'vals'. -func computeNewPriorities(updates []*Validator, vals *ValidatorSet, updatedTotalVotingPower int64) { +func computeNewPriorities(updates []*Validator, vals *ValidatorSet, updatedTotalStakingPower int64) { for _, valUpdate := range updates { address := valUpdate.Address _, val := vals.GetByAddress(address) if val == nil { // add val - // Set ProposerPriority to -C*totalVotingPower (with C ~= 1.125) to make sure validators can't + // Set ProposerPriority to -C*totalStakingPower (with C ~= 1.125) to make sure validators can't // un-bond and then re-bond to reset their (potentially previously negative) ProposerPriority to zero. // - // Contract: updatedVotingPower < 2 * MaxTotalVotingPower to ensure ProposerPriority does + // Contract: updatedStakingPower < 2 * MaxTotalStakingPower to ensure ProposerPriority does // not exceed the bounds of int64. // - // Compute ProposerPriority = -1.125*totalVotingPower == -(updatedVotingPower + (updatedVotingPower >> 3)). - valUpdate.ProposerPriority = -(updatedTotalVotingPower + (updatedTotalVotingPower >> 3)) + // Compute ProposerPriority = -1.125*totalStakingPower == -(updatedStakingPower + (updatedStakingPower >> 3)). + valUpdate.ProposerPriority = -(updatedTotalStakingPower + (updatedTotalStakingPower >> 3)) } else { valUpdate.ProposerPriority = val.ProposerPriority } @@ -479,21 +479,21 @@ func (vals *ValidatorSet) applyUpdates(updates []*Validator) { // Checks that the validators to be removed are part of the validator set. // No changes are made to the validator set 'vals'. -func verifyRemovals(deletes []*Validator, vals *ValidatorSet) (votingPower int64, err error) { +func verifyRemovals(deletes []*Validator, vals *ValidatorSet) (staingPower int64, err error) { - removedVotingPower := int64(0) + removedStakingPower := int64(0) for _, valUpdate := range deletes { address := valUpdate.Address _, val := vals.GetByAddress(address) if val == nil { - return removedVotingPower, fmt.Errorf("failed to find validator %X to remove", address) + return removedStakingPower, fmt.Errorf("failed to find validator %X to remove", address) } - removedVotingPower += val.VotingPower + removedStakingPower += val.StakingPower } if len(deletes) > len(vals.Validators) { panic("more deletes than validators") } - return removedVotingPower, nil + return removedStakingPower, nil } // Removes the validators specified in 'deletes' from validator set 'vals'. @@ -552,14 +552,14 @@ func (vals *ValidatorSet) updateWithChangeSet(changes []*Validator, allowDeletes // Verify that applying the 'deletes' against 'vals' will not result in error. // Get the voting power that is going to be removed. - removedVotingPower, err := verifyRemovals(deletes, vals) + removedStakingPower, err := verifyRemovals(deletes, vals) if err != nil { return err } // Verify that applying the 'updates' against 'vals' will not result in error. - // Get the updated total voting power before removal. Note that this is < 2 * MaxTotalVotingPower - tvpAfterUpdatesBeforeRemovals, err := verifyUpdates(updates, vals, removedVotingPower) + // Get the updated total voting power before removal. Note that this is < 2 * MaxTotalStakingPower + tvpAfterUpdatesBeforeRemovals, err := verifyUpdates(updates, vals, removedStakingPower) if err != nil { return err } @@ -571,10 +571,10 @@ func (vals *ValidatorSet) updateWithChangeSet(changes []*Validator, allowDeletes vals.applyUpdates(updates) vals.applyRemovals(deletes) - vals.updateTotalVotingPower() // will panic if total voting power > MaxTotalVotingPower + vals.updateTotalStakingPower() // will panic if total voting power > MaxTotalStakingPower // Scale and center. - vals.RescalePriorities(PriorityWindowSizeFactor * vals.TotalVotingPower()) + vals.RescalePriorities(PriorityWindowSizeFactor * vals.TotalStakingPower()) vals.shiftByAvgProposerPriority() return nil @@ -605,7 +605,7 @@ func (vals *ValidatorSet) SelectProposer(proofHash []byte, height int64, round i for i, val := range vals.Validators { candidates[i] = &candidate{idx: i, val: val} } - samples := tmrand.RandomSamplingWithPriority(seed, candidates, 1, uint64(vals.TotalVotingPower())) + samples := tmrand.RandomSamplingWithPriority(seed, candidates, 1, uint64(vals.TotalStakingPower())) proposerIdx := samples[0].(*candidate).idx return vals.Validators[proposerIdx] } diff --git a/types/validator_set_test.go b/types/validator_set_test.go index d868fce41..af90c9d55 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -45,11 +45,11 @@ func TestValidatorSetBasic(t *testing.T) { assert.Nil(t, addr) assert.Nil(t, val) assert.Zero(t, vset.Size()) - assert.Equal(t, int64(0), vset.TotalVotingPower()) + assert.Equal(t, int64(0), vset.TotalStakingPower()) assert.Nil(t, vset.Hash()) // add - val = randValidator(vset.TotalVotingPower()) + val = randValidator(vset.TotalStakingPower()) assert.NoError(t, vset.UpdateWithChangeSet([]*Validator{val})) assert.True(t, vset.HasAddress(val.Address)) @@ -58,17 +58,17 @@ func TestValidatorSetBasic(t *testing.T) { addr, _ = vset.GetByIndex(0) assert.Equal(t, []byte(val.Address), addr) assert.Equal(t, 1, vset.Size()) - assert.Equal(t, val.VotingPower, vset.TotalVotingPower()) + assert.Equal(t, val.StakingPower, vset.TotalStakingPower()) assert.NotNil(t, vset.Hash()) assert.NotPanics(t, func() { vset.IncrementProposerPriority(1) }) assert.Equal(t, val.Address, vset.SelectProposer([]byte{}, 1, 0).Address) // update - val = randValidator(vset.TotalVotingPower()) + val = randValidator(vset.TotalStakingPower()) assert.NoError(t, vset.UpdateWithChangeSet([]*Validator{val})) _, val = vset.GetByAddress(val.Address) - val.VotingPower += 100 + val.StakingPower += 100 proposerPriority := val.ProposerPriority val.ProposerPriority = 0 @@ -157,7 +157,7 @@ func verifyWinningRate(t *testing.T, vals *ValidatorSet, tries int, error float6 } for i := 0; i < len(actual); i++ { - expected := float64(vals.Validators[i].VotingPower) / float64(vals.TotalVotingPower()) + expected := float64(vals.Validators[i].StakingPower) / float64(vals.TotalStakingPower()) if math.Abs(expected-actual[i]) > expected*error { t.Errorf("The winning rate is too far off from expected: %f ∉ %f±%f", actual[i], expected, expected*error) @@ -255,7 +255,7 @@ func TestProposerSelection2(t *testing.T) { } func newValidator(address []byte, power int64) *Validator { - return &Validator{Address: address, VotingPower: power, PubKey: randPubKey()} + return &Validator{Address: address, StakingPower: power, PubKey: randPubKey()} } func randPubKey() crypto.PubKey { @@ -272,10 +272,10 @@ func max(a, b int64) int64 { } func randValidator(totalVotingPower int64) *Validator { - // this modulo limits the ProposerPriority/VotingPower to stay in the - // bounds of MaxTotalVotingPower minus the already existing voting power: - val := NewValidator(randPubKey(), max(int64(tmrand.Uint64()%uint64((MaxTotalVotingPower-totalVotingPower))), 1)) - val.ProposerPriority = tmrand.Int64() % (MaxTotalVotingPower - totalVotingPower) + // this modulo limits the ProposerPriority/StakingPower to stay in the + // bounds of MaxTotalStakingPower minus the already existing voting power: + val := NewValidator(randPubKey(), max(int64(tmrand.Uint64()%uint64((MaxTotalStakingPower-totalVotingPower))), 1)) + val.ProposerPriority = tmrand.Int64() % (MaxTotalStakingPower - totalVotingPower) return val } @@ -284,7 +284,7 @@ func randValidatorSet(numValidators int) *ValidatorSet { totalVotingPower := int64(0) for i := 0; i < numValidators; i++ { validators[i] = randValidator(totalVotingPower) - totalVotingPower += validators[i].VotingPower + totalVotingPower += validators[i].StakingPower } return NewValidatorSet(validators) } @@ -326,13 +326,13 @@ func (vals *ValidatorSet) fromBytes(b []byte) { //------------------------------------------------------------------- func TestValidatorSetTotalVotingPowerPanicsOnOverflow(t *testing.T) { - // NewValidatorSet calls IncrementProposerPriority which calls TotalVotingPower() + // NewValidatorSet calls IncrementProposerPriority which calls TotalStakingPower() // which should panic on overflows: shouldPanic := func() { NewValidatorSet([]*Validator{ - {Address: []byte("a"), VotingPower: math.MaxInt64, ProposerPriority: 0}, - {Address: []byte("b"), VotingPower: math.MaxInt64, ProposerPriority: 0}, - {Address: []byte("c"), VotingPower: math.MaxInt64, ProposerPriority: 0}, + {Address: []byte("a"), StakingPower: math.MaxInt64, ProposerPriority: 0}, + {Address: []byte("b"), StakingPower: math.MaxInt64, ProposerPriority: 0}, + {Address: []byte("c"), StakingPower: math.MaxInt64, ProposerPriority: 0}, }) } @@ -426,9 +426,9 @@ func TestAveragingInIncrementProposerPriorityWithVotingPower(t *testing.T) { total := vp0 + vp1 + vp2 avg := (vp0 + vp1 + vp2 - total) / 3 vals := ValidatorSet{Validators: []*Validator{ - {Address: []byte{0}, ProposerPriority: 0, VotingPower: vp0}, - {Address: []byte{1}, ProposerPriority: 0, VotingPower: vp1}, - {Address: []byte{2}, ProposerPriority: 0, VotingPower: vp2}}} + {Address: []byte{0}, ProposerPriority: 0, StakingPower: vp0}, + {Address: []byte{1}, ProposerPriority: 0, StakingPower: vp1}, + {Address: []byte{2}, ProposerPriority: 0, StakingPower: vp2}}} tcs := []struct { vals *ValidatorSet wantProposerPrioritys []int64 @@ -439,7 +439,7 @@ func TestAveragingInIncrementProposerPriorityWithVotingPower(t *testing.T) { 0: { vals.Copy(), []int64{ - // Acumm+VotingPower-Avg: + // Acumm+StakingPower-Avg: 0 + vp0 - total - avg, // mostest will be subtracted by total voting power (12) 0 + vp1, 0 + vp2}, @@ -473,7 +473,7 @@ func TestAveragingInIncrementProposerPriorityWithVotingPower(t *testing.T) { vals.Copy(), []int64{ 0 + 4*(vp0-total) + vp0, // 4 iters was mostest - 0 + 5*vp1 - total, // now this val is mostest for the 1st time (hence -12==totalVotingPower) + 0 + 5*vp1 - total, // now this val is mostest for the 1st time (hence -12==totalStakingPower) 0 + 5*vp2}, 5, vals.Validators[2]}, @@ -727,11 +727,11 @@ func verifyValidatorSet(t *testing.T, valSet *ValidatorSet) { assert.Equal(t, len(valSet.Validators), cap(valSet.Validators)) // verify that the set's total voting power has been updated - tvp := valSet.totalVotingPower - valSet.updateTotalVotingPower() - expectedTvp := valSet.TotalVotingPower() + tvp := valSet.totalStakingPower + valSet.updateTotalStakingPower() + expectedTvp := valSet.TotalStakingPower() assert.Equal(t, expectedTvp, tvp, - "expected TVP %d. Got %d, valSet=%s", expectedTvp, tvp, valSet) + "expected TVP %d. Got %d, voterSet=%s", expectedTvp, tvp, valSet) // verify that validator priorities are centered valsCount := int64(len(valSet.Validators)) @@ -749,7 +749,7 @@ func toTestValList(valList []*Validator) []testVal { testList := make([]testVal, len(valList)) for i, val := range valList { testList[i].name = string(val.Address) - testList[i].power = val.VotingPower + testList[i].power = val.StakingPower } return testList } @@ -841,7 +841,7 @@ func TestValSetUpdatesDuplicateEntries(t *testing.T) { } func TestValSetUpdatesOverflows(t *testing.T) { - maxVP := MaxTotalVotingPower + maxVP := MaxTotalStakingPower testCases := []valSetErrTestCase{ { // single update leading to overflow testValSet(2, 10), @@ -949,7 +949,7 @@ func TestValSetUpdatesBasicTestsExecute(t *testing.T) { // is changed in the list of validators previously passed as parameter to UpdateWithChangeSet. // this is to make sure copies of the validators are made by UpdateWithChangeSet. if len(valList) > 0 { - valList[0].VotingPower++ + valList[0].StakingPower++ assert.Equal(t, toTestValList(valListCopy), toTestValList(valSet.Validators), "test %v", i) } @@ -1245,39 +1245,39 @@ func TestValSetUpdateOverflowRelated(t *testing.T) { testCases := []testVSetCfg{ { name: "1 no false overflow error messages for updates", - startVals: []testVal{{"v1", 1}, {"v2", MaxTotalVotingPower - 1}}, - updatedVals: []testVal{{"v1", MaxTotalVotingPower - 1}, {"v2", 1}}, - expectedVals: []testVal{{"v1", MaxTotalVotingPower - 1}, {"v2", 1}}, + startVals: []testVal{{"v1", 1}, {"v2", MaxTotalStakingPower - 1}}, + updatedVals: []testVal{{"v1", MaxTotalStakingPower - 1}, {"v2", 1}}, + expectedVals: []testVal{{"v1", MaxTotalStakingPower - 1}, {"v2", 1}}, wantErr: false, }, { // this test shows that it is important to apply the updates in the order of the change in power // i.e. apply first updates with decreases in power, v2 change in this case. name: "2 no false overflow error messages for updates", - startVals: []testVal{{"v1", 1}, {"v2", MaxTotalVotingPower - 1}}, - updatedVals: []testVal{{"v1", MaxTotalVotingPower/2 - 1}, {"v2", MaxTotalVotingPower / 2}}, - expectedVals: []testVal{{"v1", MaxTotalVotingPower/2 - 1}, {"v2", MaxTotalVotingPower / 2}}, + startVals: []testVal{{"v1", 1}, {"v2", MaxTotalStakingPower - 1}}, + updatedVals: []testVal{{"v1", MaxTotalStakingPower/2 - 1}, {"v2", MaxTotalStakingPower / 2}}, + expectedVals: []testVal{{"v1", MaxTotalStakingPower/2 - 1}, {"v2", MaxTotalStakingPower / 2}}, wantErr: false, }, { name: "3 no false overflow error messages for deletes", - startVals: []testVal{{"v1", MaxTotalVotingPower - 2}, {"v2", 1}, {"v3", 1}}, + startVals: []testVal{{"v1", MaxTotalStakingPower - 2}, {"v2", 1}, {"v3", 1}}, deletedVals: []testVal{{"v1", 0}}, - addedVals: []testVal{{"v4", MaxTotalVotingPower - 2}}, - expectedVals: []testVal{{"v2", 1}, {"v3", 1}, {"v4", MaxTotalVotingPower - 2}}, + addedVals: []testVal{{"v4", MaxTotalStakingPower - 2}}, + expectedVals: []testVal{{"v2", 1}, {"v3", 1}, {"v4", MaxTotalStakingPower - 2}}, wantErr: false, }, { name: "4 no false overflow error messages for adds, updates and deletes", startVals: []testVal{ - {"v1", MaxTotalVotingPower / 4}, {"v2", MaxTotalVotingPower / 4}, - {"v3", MaxTotalVotingPower / 4}, {"v4", MaxTotalVotingPower / 4}}, + {"v1", MaxTotalStakingPower / 4}, {"v2", MaxTotalStakingPower / 4}, + {"v3", MaxTotalStakingPower / 4}, {"v4", MaxTotalStakingPower / 4}}, deletedVals: []testVal{{"v2", 0}}, updatedVals: []testVal{ - {"v1", MaxTotalVotingPower/2 - 2}, {"v3", MaxTotalVotingPower/2 - 3}, {"v4", 2}}, + {"v1", MaxTotalStakingPower/2 - 2}, {"v3", MaxTotalStakingPower/2 - 3}, {"v4", 2}}, addedVals: []testVal{{"v5", 3}}, expectedVals: []testVal{ - {"v1", MaxTotalVotingPower/2 - 2}, {"v3", MaxTotalVotingPower/2 - 3}, {"v4", 2}, {"v5", 3}}, + {"v1", MaxTotalStakingPower/2 - 2}, {"v3", MaxTotalStakingPower/2 - 3}, {"v4", 2}, {"v5", 3}}, wantErr: false, }, { @@ -1286,9 +1286,9 @@ func TestValSetUpdateOverflowRelated(t *testing.T) { {"v1", 1}, {"v2", 1}, {"v3", 1}, {"v4", 1}, {"v5", 1}, {"v6", 1}, {"v7", 1}, {"v8", 1}, {"v9", 1}}, updatedVals: []testVal{ - {"v1", MaxTotalVotingPower}, {"v2", MaxTotalVotingPower}, {"v3", MaxTotalVotingPower}, - {"v4", MaxTotalVotingPower}, {"v5", MaxTotalVotingPower}, {"v6", MaxTotalVotingPower}, - {"v7", MaxTotalVotingPower}, {"v8", MaxTotalVotingPower}, {"v9", 8}}, + {"v1", MaxTotalStakingPower}, {"v2", MaxTotalStakingPower}, {"v3", MaxTotalStakingPower}, + {"v4", MaxTotalStakingPower}, {"v5", MaxTotalStakingPower}, {"v6", MaxTotalStakingPower}, + {"v7", MaxTotalStakingPower}, {"v8", MaxTotalStakingPower}, {"v9", 8}}, expectedVals: []testVal{ {"v1", 1}, {"v2", 1}, {"v3", 1}, {"v4", 1}, {"v5", 1}, {"v6", 1}, {"v7", 1}, {"v8", 1}, {"v9", 1}}, diff --git a/types/vote_set.go b/types/vote_set.go index 121dad601..902e4efcd 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -63,7 +63,7 @@ type VoteSet struct { height int64 round int signedMsgType SignedMsgType - valSet *VoterSet + voterSet *VoterSet mtx sync.Mutex votesBitArray *bits.BitArray @@ -84,7 +84,7 @@ func NewVoteSet(chainID string, height int64, round int, signedMsgType SignedMsg height: height, round: round, signedMsgType: signedMsgType, - valSet: voterSet, + voterSet: voterSet, votesBitArray: bits.NewBitArray(voterSet.Size()), votes: make([]*Vote, voterSet.Size()), sum: 0, @@ -127,7 +127,7 @@ func (voteSet *VoteSet) Size() int { if voteSet == nil { return 0 } - return voteSet.valSet.Size() + return voteSet.voterSet.Size() } // Returns added=true if vote is valid and new. @@ -175,10 +175,10 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { } // Ensure that signer is a validator. - lookupAddr, val := voteSet.valSet.GetByIndex(valIndex) + lookupAddr, val := voteSet.voterSet.GetByIndex(valIndex) if val == nil { return false, errors.Wrapf(ErrVoteInvalidValidatorIndex, - "Cannot find voter %d in valSet of size %d", valIndex, voteSet.valSet.Size()) + "Cannot find voter %d in voterSet of size %d", valIndex, voteSet.voterSet.Size()) } // Ensure that the signer has the right address. @@ -203,7 +203,7 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { } // Add vote and get conflicting vote if any. - added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.VotingPower) + added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.StakingPower) if conflicting != nil { return added, NewConflictingVoteError(val, conflicting, vote) } @@ -269,14 +269,14 @@ func (voteSet *VoteSet) addVerifiedVote( } // ... and there's no conflicting vote. // Start tracking this blockKey - votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) + votesByBlock = newBlockVotes(false, voteSet.voterSet.Size()) voteSet.votesByBlock[blockKey] = votesByBlock // We'll add the vote in a bit. } // Before adding to votesByBlock, see if we'll exceed quorum origSum := votesByBlock.sum - quorum := voteSet.valSet.TotalVotingPower()*2/3 + 1 + quorum := voteSet.voterSet.TotalVotingPower()*2/3 + 1 // Add vote to votesByBlock votesByBlock.addVerifiedVote(vote, votingPower) @@ -332,7 +332,7 @@ func (voteSet *VoteSet) SetPeerMaj23(peerID P2PID, blockID BlockID) error { votesByBlock.peerMaj23 = true // No need to copy votes, already there. } else { - votesByBlock = newBlockVotes(true, voteSet.valSet.Size()) + votesByBlock = newBlockVotes(true, voteSet.voterSet.Size()) voteSet.votesByBlock[blockKey] = votesByBlock // No need to copy votes, no votes to copy over. } @@ -379,7 +379,7 @@ func (voteSet *VoteSet) GetByAddress(address []byte) *Vote { } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - valIndex, val := voteSet.valSet.GetByAddress(address) + valIndex, val := voteSet.voterSet.GetByAddress(address) if val == nil { panic("GetByAddress(address) returned nil") } @@ -414,13 +414,13 @@ func (voteSet *VoteSet) HasTwoThirdsAny() bool { } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - return voteSet.sum > voteSet.valSet.TotalVotingPower()*2/3 + return voteSet.sum > voteSet.voterSet.TotalVotingPower()*2/3 } func (voteSet *VoteSet) HasAll() bool { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - return voteSet.sum == voteSet.valSet.TotalVotingPower() + return voteSet.sum == voteSet.voterSet.TotalVotingPower() } // If there was a +2/3 majority for blockID, return blockID and true. @@ -539,7 +539,7 @@ func (voteSet *VoteSet) StringShort() string { // return the power voted, the total, and the fraction func (voteSet *VoteSet) sumTotalFrac() (int64, int64, float64) { - voted, total := voteSet.sum, voteSet.valSet.TotalVotingPower() + voted, total := voteSet.sum, voteSet.voterSet.TotalVotingPower() fracVoted := float64(voted) / float64(total) return voted, total, fracVoted } diff --git a/types/voter_set.go b/types/voter_set.go index 9898321be..482839957 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -90,16 +90,16 @@ func (voters *VoterSet) Copy() *VoterSet { } // Forces recalculation of the set's total voting power. -// Panics if total voting power is bigger than MaxTotalVotingPower. +// Panics if total voting power is bigger than MaxTotalStakingPower. func (voters *VoterSet) updateTotalVotingPower() { sum := int64(0) for _, val := range voters.Voters { // mind overflow - sum = safeAddClip(sum, val.VotingPower) - if sum > MaxTotalVotingPower { + sum = safeAddClip(sum, val.StakingPower) + if sum > MaxTotalStakingPower { panic(fmt.Sprintf( "Total voting power should be guarded to not exceed %v; got: %v", - MaxTotalVotingPower, + MaxTotalStakingPower, sum)) } } @@ -156,7 +156,7 @@ func (voters *VoterSet) VerifyCommit(chainID string, blockID BlockID, } // Good! if blockID.Equals(commitSig.BlockID(commit.BlockID)) { - talliedVotingPower += val.VotingPower + talliedVotingPower += val.StakingPower } // else { // It's OK that the BlockID doesn't match. We include stray @@ -235,7 +235,7 @@ func (voters *VoterSet) VerifyFutureCommit(newSet *VoterSet, chainID string, } // Good! if blockID.Equals(commitSig.BlockID(commit.BlockID)) { - oldVotingPower += val.VotingPower + oldVotingPower += val.StakingPower } // else { // It's OK that the BlockID doesn't match. We include stray @@ -296,7 +296,7 @@ func (voters *VoterSet) VerifyCommitTrusting(chainID string, blockID BlockID, // Good! if blockID.Equals(commitSig.BlockID(commit.BlockID)) { - talliedVotingPower += val.VotingPower + talliedVotingPower += val.StakingPower } // else { // It's OK that the BlockID doesn't match. We include stray @@ -395,7 +395,7 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { for i, val := range validators.Validators { candidates[i] = &candidate{idx: i, win: 0, val: val} } - totalSampling := tmrand.RandomSamplingToMax(seed, candidates, MaxVoters, uint64(validators.TotalVotingPower())) + totalSampling := tmrand.RandomSamplingToMax(seed, candidates, MaxVoters, uint64(validators.TotalStakingPower())) voters := 0 for _, candi := range candidates { if candi.(*candidate).win > 0 { @@ -409,9 +409,9 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { if candi.(*candidate).win > 0 { vals[index] = &Validator{Address: candi.(*candidate).val.Address, PubKey: candi.(*candidate).val.PubKey, - // VotingPower = TotalVotingPower * win / totalSampling : can be overflow - VotingPower: validators.TotalVotingPower()/int64(totalSampling)*int64(candi.(*candidate).win) + - int64(math.Ceil(float64(validators.TotalVotingPower()%int64(totalSampling))/float64(int64(totalSampling))* + // StakingPower = TotalStakingPower * win / totalSampling : can be overflow + StakingPower: validators.TotalStakingPower()/int64(totalSampling)*int64(candi.(*candidate).win) + + int64(math.Ceil(float64(validators.TotalStakingPower()%int64(totalSampling))/float64(int64(totalSampling))* float64(candi.(*candidate).win)))} index++ } @@ -432,11 +432,11 @@ type candidate struct { } func (c *candidate) Priority() uint64 { - // TODO Is it possible to have a negative VotingPower? - if c.val.VotingPower < 0 { + // TODO Is it possible to have a negative StakingPower? + if c.val.StakingPower < 0 { return 0 } - return uint64(c.val.VotingPower) + return uint64(c.val.StakingPower) } func (c *candidate) LessThan(other tmrand.Candidate) bool { diff --git a/types/voter_set_test.go b/types/voter_set_test.go index 0359540a6..7b6fc5621 100644 --- a/types/voter_set_test.go +++ b/types/voter_set_test.go @@ -14,14 +14,14 @@ func TestSelectVoter(t *testing.T) { valSet := randValidatorSet(30) for i := 0; i < 10000; i++ { voterSet := SelectVoter(valSet, []byte{byte(i)}) - assert.True(t, math.Abs(float64(valSet.TotalVotingPower()-voterSet.TotalVotingPower())) <= 10) + assert.True(t, math.Abs(float64(valSet.TotalStakingPower()-voterSet.TotalVotingPower())) <= 10) } } func toGenesisValidators(vals []*Validator) []GenesisValidator { genVals := make([]GenesisValidator, len(vals)) for i, val := range vals { - genVals[i] = GenesisValidator{Address: val.Address, PubKey: val.PubKey, Power: val.VotingPower, Name: "name"} + genVals[i] = GenesisValidator{Address: val.Address, PubKey: val.PubKey, Power: val.StakingPower, Name: "name"} } return genVals } @@ -65,7 +65,7 @@ func findLargestVotingPowerGap(t *testing.T, loopCount int, minMaxRate int, maxV for i := 0; i < loopCount; i++ { voterSet := SelectVoter(valSet, hash) for _, voter := range voterSet.Voters { - accumulation[voter.Address.String()] += voter.VotingPower + accumulation[voter.Address.String()] += voter.StakingPower } proposer := valSet.SelectProposer(hash, int64(i), 0) message := MakeRoundHash(hash, int64(i), 0) @@ -76,8 +76,8 @@ func findLargestVotingPowerGap(t *testing.T, loopCount int, minMaxRate int, maxV largestGap := float64(0) for _, val := range valSet.Validators { acc := accumulation[val.Address.String()] / int64(loopCount) - if math.Abs(float64(val.VotingPower-acc))/float64(val.VotingPower) > largestGap { - largestGap = math.Abs(float64(val.VotingPower-acc)) / float64(val.VotingPower) + if math.Abs(float64(val.StakingPower-acc))/float64(val.StakingPower) > largestGap { + largestGap = math.Abs(float64(val.StakingPower-acc)) / float64(val.StakingPower) } } t.Logf("<< min power=100, max power=%d, actual average voters=%d, max voters=%d >> largest gap: %f", @@ -99,7 +99,7 @@ func TestSelectVoterMaxVarious(t *testing.T) { MaxVoters = voters valSet, _ := randValidatorSetWithMinMax(validators, 100, 100*int64(minMaxRate)) voterSet := SelectVoter(valSet, []byte{byte(hash)}) - assert.True(t, int(math.Abs(float64(valSet.TotalVotingPower()-voterSet.TotalVotingPower()))) <= voters) + assert.True(t, int(math.Abs(float64(valSet.TotalStakingPower()-voterSet.TotalVotingPower()))) <= voters) if voterSet.Size() < MaxVoters { t.Logf("Cannot elect voters up to MaxVoters: validators=%d, MaxVoters=%d, actual voters=%d", validators, voters, voterSet.Size()) From 9b8d758d6c4f7cd2fbcea952ce4501c25ed5ca10 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 25 May 2020 19:53:10 +0900 Subject: [PATCH 02/38] fix: separate StakingPower and VotingPower --- consensus/reactor_test.go | 38 +++++++++++------------ lite/dbprovider.go | 8 ++--- state/state_test.go | 60 ++++++++++++++++++------------------- types/validator.go | 6 ++++ types/validator_set_test.go | 18 +++++------ types/voter_set.go | 59 +++++++++++++++++++++++++++++------- types/voter_set_test.go | 10 +++---- 7 files changed, 121 insertions(+), 78 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 076d6ee0b..ad52f27cf 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -333,7 +333,7 @@ func TestReactorRecordsVotesAndBlockParts(t *testing.T) { //------------------------------------------------------------- // ensure we can make blocks despite cycling a validator set -func TestReactorVotingPowerChange(t *testing.T) { +func TestReactorStakingPowerChange(t *testing.T) { nVals := 4 logger := log.TestingLogger() css, cleanup := randConsensusNet( @@ -363,48 +363,48 @@ func TestReactorVotingPowerChange(t *testing.T) { val1PubKey := css[0].privValidator.GetPubKey() val1PubKeyABCI := types.TM2PB.PubKey(val1PubKey) updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25) - previousTotalVotingPower := css[0].GetRoundState().LastVoters.TotalVotingPower() + previousTotalStakingPower := css[0].GetRoundState().LastVoters.TotalStakingPower() waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) - if css[0].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { + if css[0].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { t.Fatalf( - "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, - css[0].GetRoundState().LastVoters.TotalVotingPower()) + "expected staking power to change (before: %d, after: %d)", + previousTotalStakingPower, + css[0].GetRoundState().LastVoters.TotalStakingPower()) } updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2) - previousTotalVotingPower = css[0].GetRoundState().LastVoters.TotalVotingPower() + previousTotalStakingPower = css[0].GetRoundState().LastVoters.TotalStakingPower() waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) - if css[0].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { + if css[0].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { t.Fatalf( "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, - css[0].GetRoundState().LastVoters.TotalVotingPower()) + previousTotalStakingPower, + css[0].GetRoundState().LastVoters.TotalStakingPower()) } updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26) - previousTotalVotingPower = css[0].GetRoundState().LastVoters.TotalVotingPower() + previousTotalStakingPower = css[0].GetRoundState().LastVoters.TotalStakingPower() waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) - if css[0].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { + if css[0].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { t.Fatalf( "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, - css[0].GetRoundState().LastVoters.TotalVotingPower()) + previousTotalStakingPower, + css[0].GetRoundState().LastVoters.TotalStakingPower()) } } @@ -469,18 +469,18 @@ func TestReactorValidatorSetChanges(t *testing.T) { updateValidatorPubKey1 := css[nVals].privValidator.GetPubKey() updatePubKey1ABCI := types.TM2PB.PubKey(updateValidatorPubKey1) updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) - previousTotalVotingPower := css[nVals].GetRoundState().LastVoters.TotalVotingPower() + previousTotalStakingPower := css[nVals].GetRoundState().LastVoters.TotalStakingPower() waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1) waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1) waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css) waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css) - if css[nVals].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { + if css[nVals].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { t.Errorf( - "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, - css[nVals].GetRoundState().LastVoters.TotalVotingPower()) + "expected staking power to change (before: %d, after: %d)", + previousTotalStakingPower, + css[nVals].GetRoundState().LastVoters.TotalStakingPower()) } //--------------------------------------------------------------------------- diff --git a/lite/dbprovider.go b/lite/dbprovider.go index 6a12c3772..ec9d2cff5 100644 --- a/lite/dbprovider.go +++ b/lite/dbprovider.go @@ -152,7 +152,7 @@ func (dbp *DBProvider) VoterSet(chainID string, height int64) (valset *types.Vot return dbp.getVoterSet(chainID, height) } -func (dbp *DBProvider) getVoterSet(chainID string, height int64) (valset *types.VoterSet, err error) { +func (dbp *DBProvider) getVoterSet(chainID string, height int64) (voterSet *types.VoterSet, err error) { vsBz, err := dbp.db.Get(voterSetKey(chainID, height)) if err != nil { return nil, err @@ -161,7 +161,7 @@ func (dbp *DBProvider) getVoterSet(chainID string, height int64) (valset *types. err = lerr.ErrUnknownValidators(chainID, height) return } - err = dbp.cdc.UnmarshalBinaryLengthPrefixed(vsBz, &valset) + err = dbp.cdc.UnmarshalBinaryLengthPrefixed(vsBz, &voterSet) if err != nil { return } @@ -169,8 +169,8 @@ func (dbp *DBProvider) getVoterSet(chainID string, height int64) (valset *types. // To test deep equality. This makes it easier to test for e.g. voterSet // equivalence using assert.Equal (tests for deep equality) in our tests, // which also tests for unexported/private field equivalence. - valset.TotalVotingPower() - + voterSet.TotalVotingPower() + voterSet.TotalStakingPower() return } diff --git a/state/state_test.go b/state/state_test.go index dee9a4d6e..8e4d6e9c4 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -361,7 +361,7 @@ func genValSetWithPowers(powers []int64) *types.ValidatorSet { func testProposerFreq(t *testing.T, caseNum int, valSet *types.ValidatorSet) { voterSet := types.ToVoterAll(valSet) N := voterSet.Size() - totalPower := voterSet.TotalVotingPower() + totalPower := voterSet.TotalStakingPower() // run the proposer selection and track frequencies runMult := 1 @@ -399,9 +399,9 @@ func testProposerFreq(t *testing.T, caseNum int, valSet *types.ValidatorSet) { func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { tearDown, _, state := setupTestCase(t) defer tearDown(t) - val1VotingPower := int64(10) + val1StakingPower := int64(10) val1PubKey := ed25519.GenPrivKey().PubKey() - val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, StakingPower: val1VotingPower} + val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, StakingPower: val1StakingPower} state.Validators = types.NewValidatorSet([]*types.Validator{val1}) state.NextValidators = state.Validators @@ -418,14 +418,14 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { require.NoError(t, err) updatedState, err := sm.UpdateState(state, blockID, &block.Header, abciResponses, validatorUpdates) assert.NoError(t, err) - curTotal := val1VotingPower + curTotal := val1StakingPower // one increment step and one validator: 0 + power - total_power == 0 - assert.Equal(t, 0+val1VotingPower-curTotal, updatedState.NextValidators.Validators[0].ProposerPriority) + assert.Equal(t, 0+val1StakingPower-curTotal, updatedState.NextValidators.Validators[0].ProposerPriority) // add a validator val2PubKey := ed25519.GenPrivKey().PubKey() - val2VotingPower := int64(100) - updateAddVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: val2VotingPower} + val2StakingPower := int64(100) + updateAddVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: val2StakingPower} validatorUpdates, err = types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{updateAddVal}) assert.NoError(t, err) updatedState2, err := sm.UpdateState(updatedState, blockID, &block.Header, abciResponses, validatorUpdates) @@ -440,7 +440,7 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { // Steps from adding new validator: // 0 - val1 prio is 0, TVP after add: wantVal1Prio := int64(0) - totalPowerAfter := val1VotingPower + val2VotingPower + totalPowerAfter := val1StakingPower + val2StakingPower // 1. Add - Val2 should be initially added with (-123) => wantVal2Prio := -(totalPowerAfter + (totalPowerAfter >> 3)) // 2. Scale - noop @@ -451,9 +451,9 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { wantVal1Prio -= avg.Int64() // 62 // 4. Steps from IncrementProposerPriority - wantVal1Prio += val1VotingPower // 72 - wantVal2Prio += val2VotingPower // 39 - wantVal1Prio -= totalPowerAfter // -38 as val1 is proposer + wantVal1Prio += val1StakingPower // 72 + wantVal2Prio += val2StakingPower // 39 + wantVal1Prio -= totalPowerAfter // -38 as val1 is proposer assert.Equal(t, wantVal1Prio, updatedVal1.ProposerPriority) assert.Equal(t, wantVal2Prio, addedVal2.ProposerPriority) @@ -466,7 +466,7 @@ func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { assert.NoError(t, err) // this will cause the diff of priorities (77) - // to be larger than threshold == 2*totalVotingPower (22): + // to be larger than threshold == 2*totalStakingPower (22): updatedState3, err := sm.UpdateState(updatedState2, blockID, &block.Header, abciResponses, validatorUpdates) assert.NoError(t, err) @@ -510,9 +510,9 @@ func TestProposerPriorityProposerAlternates(t *testing.T) { // have the same voting power (and the 2nd was added later). tearDown, _, state := setupTestCase(t) defer tearDown(t) - val1VotingPower := int64(10) + val1StakingPower := int64(10) val1PubKey := ed25519.GenPrivKey().PubKey() - val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, StakingPower: val1VotingPower} + val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, StakingPower: val1StakingPower} // reset state validators to above validator state.Validators = types.NewValidatorSet([]*types.Validator{val1}) @@ -534,14 +534,14 @@ func TestProposerPriorityProposerAlternates(t *testing.T) { assert.NoError(t, err) // 0 + 10 (initial prio) - 10 (avg) - 10 (mostest - total) = -10 - totalPower := val1VotingPower - wantVal1Prio := 0 + val1VotingPower - totalPower + totalPower := val1StakingPower + wantVal1Prio := 0 + val1StakingPower - totalPower assert.Equal(t, wantVal1Prio, updatedState.NextValidators.Validators[0].ProposerPriority) assert.Equal(t, val1PubKey.Address(), updatedState.NextValidators.Validators[0].Address) // add a validator with the same voting power as the first val2PubKey := ed25519.GenPrivKey().PubKey() - updateAddVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: val1VotingPower} + updateAddVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: val1StakingPower} validatorUpdates, err = types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{updateAddVal}) assert.NoError(t, err) @@ -562,8 +562,8 @@ func TestProposerPriorityProposerAlternates(t *testing.T) { _, updatedVal2 := updatedState2.NextValidators.GetByAddress(val2PubKey.Address()) // 1. Add - val2VotingPower := val1VotingPower - totalPower = val1VotingPower + val2VotingPower // 20 + val2StakingPower := val1StakingPower + totalPower = val1StakingPower + val2StakingPower // 20 v2PrioWhenAddedVal2 := -(totalPower + (totalPower >> 3)) // -22 // 2. Scale - noop // 3. Center @@ -572,9 +572,9 @@ func TestProposerPriorityProposerAlternates(t *testing.T) { expectedVal2Prio := v2PrioWhenAddedVal2 - avg.Int64() // -11 expectedVal1Prio := oldVal1.ProposerPriority - avg.Int64() // 11 // 4. Increment - expectedVal2Prio += val2VotingPower // -11 + 10 = -1 - expectedVal1Prio += val1VotingPower // 11 + 10 == 21 - expectedVal1Prio -= totalPower // 1, val1 proposer + expectedVal2Prio += val2StakingPower // -11 + 10 = -1 + expectedVal1Prio += val1StakingPower // 11 + 10 == 21 + expectedVal1Prio -= totalPower // 1, val1 proposer assert.EqualValues(t, expectedVal1Prio, updatedVal1.ProposerPriority) assert.EqualValues( @@ -602,9 +602,9 @@ func TestProposerPriorityProposerAlternates(t *testing.T) { // check if expected proposer prio is matched: // Increment - expectedVal2Prio2 := expectedVal2Prio + val2VotingPower // -1 + 10 = 9 - expectedVal1Prio2 := expectedVal1Prio + val1VotingPower // 1 + 10 == 11 - expectedVal1Prio2 -= totalPower // -9, val1 proposer + expectedVal2Prio2 := expectedVal2Prio + val2StakingPower // -1 + 10 = 9 + expectedVal1Prio2 := expectedVal1Prio + val1StakingPower // 1 + 10 == 11 + expectedVal1Prio2 -= totalPower // -9, val1 proposer assert.EqualValues( t, @@ -679,13 +679,13 @@ func TestLargeGenesisValidator(t *testing.T) { tearDown, _, state := setupTestCase(t) defer tearDown(t) - genesisVotingPower := types.MaxTotalStakingPower / 1000 + genesisStakingPower := types.MaxTotalStakingPower / 1000 genesisPubKey := ed25519.GenPrivKey().PubKey() // fmt.Println("genesis addr: ", genesisPubKey.Address()) genesisVal := &types.Validator{ Address: genesisPubKey.Address(), PubKey: genesisPubKey, - StakingPower: genesisVotingPower, + StakingPower: genesisStakingPower, } // reset state validators to above validator state.Validators = types.NewValidatorSet([]*types.Validator{genesisVal}) @@ -723,8 +723,8 @@ func TestLargeGenesisValidator(t *testing.T) { // see how long it takes until the effect wears off and both begin to alternate // see: https://github.com/tendermint/tendermint/issues/2960 firstAddedValPubKey := ed25519.GenPrivKey().PubKey() - firstAddedValVotingPower := int64(10) - firstAddedVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(firstAddedValPubKey), Power: firstAddedValVotingPower} + firstAddedValStakingPower := int64(10) + firstAddedVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(firstAddedValPubKey), Power: firstAddedValStakingPower} validatorUpdates, err := types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{firstAddedVal}) assert.NoError(t, err) abciResponses := &sm.ABCIResponses{ @@ -768,7 +768,7 @@ func TestLargeGenesisValidator(t *testing.T) { for i := 0; i < 10; i++ { addedPubKey := ed25519.GenPrivKey().PubKey() - addedVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(addedPubKey), Power: firstAddedValVotingPower} + addedVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(addedPubKey), Power: firstAddedValStakingPower} validatorUpdates, err := types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{addedVal}) assert.NoError(t, err) diff --git a/types/validator.go b/types/validator.go index 5d99a342d..644df13f6 100644 --- a/types/validator.go +++ b/types/validator.go @@ -12,11 +12,16 @@ import ( // Volatile state for each Validator // NOTE: The ProposerPriority is not included in Validator.Hash(); // make sure to update that method if changes are made here +// StakingPower is the potential voting power proportional to the amount of stake, +// and VotingPower is the actual voting power granted by the election process. +// StakingPower is durable and can be changed by staking txs. +// VotingPower is volatile and can be changed at every height. type Validator struct { Address Address `json:"address"` PubKey crypto.PubKey `json:"pub_key"` StakingPower int64 `json:"staking_power"` + VotingPower int64 `json:"voting_power"` ProposerPriority int64 `json:"proposer_priority"` } @@ -25,6 +30,7 @@ func NewValidator(pubKey crypto.PubKey, stakingPower int64) *Validator { Address: pubKey.Address(), PubKey: pubKey, StakingPower: stakingPower, + VotingPower: 0, ProposerPriority: 0, } } diff --git a/types/validator_set_test.go b/types/validator_set_test.go index af90c9d55..724e1b7fe 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -271,20 +271,20 @@ func max(a, b int64) int64 { return b } -func randValidator(totalVotingPower int64) *Validator { +func randValidator(totalStakingPower int64) *Validator { // this modulo limits the ProposerPriority/StakingPower to stay in the // bounds of MaxTotalStakingPower minus the already existing voting power: - val := NewValidator(randPubKey(), max(int64(tmrand.Uint64()%uint64((MaxTotalStakingPower-totalVotingPower))), 1)) - val.ProposerPriority = tmrand.Int64() % (MaxTotalStakingPower - totalVotingPower) + val := NewValidator(randPubKey(), max(int64(tmrand.Uint64()%uint64((MaxTotalStakingPower-totalStakingPower))), 1)) + val.ProposerPriority = tmrand.Int64() % (MaxTotalStakingPower - totalStakingPower) return val } func randValidatorSet(numValidators int) *ValidatorSet { validators := make([]*Validator, numValidators) - totalVotingPower := int64(0) + totalStakingPower := int64(0) for i := 0; i < numValidators; i++ { - validators[i] = randValidator(totalVotingPower) - totalVotingPower += validators[i].StakingPower + validators[i] = randValidator(totalStakingPower) + totalStakingPower += validators[i].StakingPower } return NewValidatorSet(validators) } @@ -325,7 +325,7 @@ func (vals *ValidatorSet) fromBytes(b []byte) { //------------------------------------------------------------------- -func TestValidatorSetTotalVotingPowerPanicsOnOverflow(t *testing.T) { +func TestValidatorSetTotalStakingPowerPanicsOnOverflow(t *testing.T) { // NewValidatorSet calls IncrementProposerPriority which calls TotalStakingPower() // which should panic on overflows: shouldPanic := func() { @@ -416,7 +416,7 @@ func TestAveragingInIncrementProposerPriority(t *testing.T) { } } -func TestAveragingInIncrementProposerPriorityWithVotingPower(t *testing.T) { +func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { // Other than TestAveragingInIncrementProposerPriority this is a more complete test showing // how each ProposerPriority changes in relation to the validator's voting power respectively. // average is zero in each round: @@ -573,7 +573,7 @@ func TestValidatorSetVerifyCommit(t *testing.T) { privKey := ed25519.GenPrivKey() pubKey := privKey.PubKey() v1 := NewValidator(pubKey, 1000) - vset := NewVoterSet([]*Validator{v1}) + vset := ToVoterAll(NewValidatorSet([]*Validator{v1})) // good var ( diff --git a/types/voter_set.go b/types/voter_set.go index 482839957..cb010fdc3 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -23,12 +23,14 @@ type VoterSet struct { Voters []*Validator `json:"voters"` // cached (unexported) - totalVotingPower int64 + totalVotingPower int64 + totalStakingPower int64 } func NewVoterSet(valz []*Validator) *VoterSet { sort.Sort(ValidatorsByAddress(valz)) vals := &VoterSet{Voters: copyValidatorListShallow(valz), totalVotingPower: 0} + vals.updateTotalStakingPower() vals.updateTotalVotingPower() return vals } @@ -95,7 +97,7 @@ func (voters *VoterSet) updateTotalVotingPower() { sum := int64(0) for _, val := range voters.Voters { // mind overflow - sum = safeAddClip(sum, val.StakingPower) + sum = safeAddClip(sum, val.VotingPower) if sum > MaxTotalStakingPower { panic(fmt.Sprintf( "Total voting power should be guarded to not exceed %v; got: %v", @@ -107,6 +109,22 @@ func (voters *VoterSet) updateTotalVotingPower() { voters.totalVotingPower = sum } +func (voters *VoterSet) updateTotalStakingPower() { + sum := int64(0) + for _, val := range voters.Voters { + // mind overflow + sum = safeAddClip(sum, val.StakingPower) + if sum > MaxTotalStakingPower { + panic(fmt.Sprintf( + "Total voting power should be guarded to not exceed %v; got: %v", + MaxTotalStakingPower, + sum)) + } + } + + voters.totalStakingPower = sum +} + func (voters *VoterSet) TotalVotingPower() int64 { if voters.totalVotingPower == 0 { voters.updateTotalVotingPower() @@ -114,6 +132,13 @@ func (voters *VoterSet) TotalVotingPower() int64 { return voters.totalVotingPower } +func (voters *VoterSet) TotalStakingPower() int64 { + if voters.totalStakingPower == 0 { + voters.updateTotalStakingPower() + } + return voters.totalStakingPower +} + // Hash returns the Merkle root hash build using validators (as leaves) in the // set. func (voters *VoterSet) Hash() []byte { @@ -156,7 +181,7 @@ func (voters *VoterSet) VerifyCommit(chainID string, blockID BlockID, } // Good! if blockID.Equals(commitSig.BlockID(commit.BlockID)) { - talliedVotingPower += val.StakingPower + talliedVotingPower += val.VotingPower } // else { // It's OK that the BlockID doesn't match. We include stray @@ -235,7 +260,7 @@ func (voters *VoterSet) VerifyFutureCommit(newSet *VoterSet, chainID string, } // Good! if blockID.Equals(commitSig.BlockID(commit.BlockID)) { - oldVotingPower += val.StakingPower + oldVotingPower += val.VotingPower } // else { // It's OK that the BlockID doesn't match. We include stray @@ -296,7 +321,7 @@ func (voters *VoterSet) VerifyCommitTrusting(chainID string, blockID BlockID, // Good! if blockID.Equals(commitSig.BlockID(commit.BlockID)) { - talliedVotingPower += val.StakingPower + talliedVotingPower += val.VotingPower } // else { // It's OK that the BlockID doesn't match. We include stray @@ -385,9 +410,7 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { // TODO: decide MaxVoters; make it to config if len(proofHash) == 0 || validators.Size() <= MaxVoters { // height 1 has voter set that is same to validator set - result := &VoterSet{Voters: copyValidatorListShallow(validators.Validators), totalVotingPower: 0} - result.updateTotalVotingPower() - return result + return ToVoterAll(validators) } seed := hashToSeed(proofHash) @@ -408,9 +431,10 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { for _, candi := range candidates { if candi.(*candidate).win > 0 { vals[index] = &Validator{Address: candi.(*candidate).val.Address, - PubKey: candi.(*candidate).val.PubKey, + PubKey: candi.(*candidate).val.PubKey, + StakingPower: candi.(*candidate).val.StakingPower, // StakingPower = TotalStakingPower * win / totalSampling : can be overflow - StakingPower: validators.TotalStakingPower()/int64(totalSampling)*int64(candi.(*candidate).win) + + VotingPower: validators.TotalStakingPower()/int64(totalSampling)*int64(candi.(*candidate).win) + int64(math.Ceil(float64(validators.TotalStakingPower()%int64(totalSampling))/float64(int64(totalSampling))* float64(candi.(*candidate).win)))} index++ @@ -421,7 +445,20 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { // This should be used in only test func ToVoterAll(validators *ValidatorSet) *VoterSet { - return NewVoterSet(validators.Validators) + newVoters := make([]*Validator, validators.Size()) + for i, val := range validators.Validators { + newVoters[i] = &Validator{ + Address: val.Address, + PubKey: val.PubKey, + StakingPower: val.StakingPower, + VotingPower: val.StakingPower, + ProposerPriority: val.ProposerPriority, + } + } + voters := NewVoterSet(newVoters) + voters.updateTotalVotingPower() + voters.updateTotalStakingPower() + return voters } // candidate save simple validator data for selecting proposer diff --git a/types/voter_set_test.go b/types/voter_set_test.go index 7b6fc5621..0d769a07d 100644 --- a/types/voter_set_test.go +++ b/types/voter_set_test.go @@ -41,17 +41,17 @@ The result when we set LoopCount to 10000 << min power=100, max power=100000000, actual average voters=20, max voters=20 >> largest gap: 0.076547 << min power=100, max power=100000000, actual average voters=29, max voters=29 >> largest gap: 0.147867 */ -func TestSelectVoterReasonableVotingPower(t *testing.T) { +func TestSelectVoterReasonableStakingPower(t *testing.T) { // Raise LoopCount to get smaller gap over 10000. But large LoopCount takes a lot of time const LoopCount = 100 for minMaxRate := 1; minMaxRate <= 1000000; minMaxRate *= 100 { - findLargestVotingPowerGap(t, LoopCount, minMaxRate, 10) - findLargestVotingPowerGap(t, LoopCount, minMaxRate, 20) - findLargestVotingPowerGap(t, LoopCount, minMaxRate, 29) + findLargestStakingPowerGap(t, LoopCount, minMaxRate, 10) + findLargestStakingPowerGap(t, LoopCount, minMaxRate, 20) + findLargestStakingPowerGap(t, LoopCount, minMaxRate, 29) } } -func findLargestVotingPowerGap(t *testing.T, loopCount int, minMaxRate int, maxVoters int) { +func findLargestStakingPowerGap(t *testing.T, loopCount int, minMaxRate int, maxVoters int) { valSet, privMap := randValidatorSetWithMinMax(30, 100, 100*int64(minMaxRate)) genDoc := &GenesisDoc{ GenesisTime: tmtime.Now(), From d68124d03995ec257df684538d74bee2554771a9 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Fri, 29 May 2020 16:38:57 +0900 Subject: [PATCH 03/38] feat: implement RandomSamplingWithoutReplacement --- libs/rand/sampling.go | 77 +++++++++++------ libs/rand/sampling_test.go | 166 ++++++++++++++++++++++++++++--------- types/validator.go | 20 +++++ types/validator_set.go | 5 +- types/voter_set.go | 72 ++++------------ types/voter_set_test.go | 19 +++-- 6 files changed, 231 insertions(+), 128 deletions(-) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 8aaf411c7..4d94bcb77 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -2,6 +2,7 @@ package rand import ( "fmt" + "math" s "sort" ) @@ -9,7 +10,7 @@ import ( type Candidate interface { Priority() uint64 LessThan(other Candidate) bool - IncreaseWin() + Reward(rewards uint64) } const uint64Mask = uint64(0x7FFFFFFFFFFFFFFF) @@ -66,34 +67,47 @@ func RandomSamplingWithPriority( totalPriority, actualTotalPriority, seed, sampleSize, undrawn, undrawn, thresholds[undrawn], len(candidates))) } -const MaxSamplingLoopTry = 1000 +func moveWinnerToLast(candidates []Candidate, winner int) { + winnerCandidate := candidates[winner] + copy(candidates[winner:], candidates[winner+1:]) + candidates[len(candidates)-1] = winnerCandidate +} + +// `RandomSamplingWithoutReplacement` elects winners among candidates without replacement so it updates rewards of winners. +// This function continues to elect winners until the both of two conditions(samplingThreshold, priorityRateThreshold) are met. +func RandomSamplingWithoutReplacement( + seed uint64, candidates []Candidate, samplingThreshold int, priorityRateThreshold float64, rewardUnit uint64) ( + winners []Candidate) { -// `RandomSamplingToMax` elects voters among candidates so it updates wins of candidates -// Voters can be elected by a maximum `limitCandidates`. -// However, if the likely candidates are less than the `limitCandidates`, -// the number of voters may be less than the `limitCandidates`. -// This is to prevent falling into an infinite loop. -func RandomSamplingToMax( - seed uint64, candidates []Candidate, limitCandidates int, totalPriority uint64) uint64 { + if len(candidates) < samplingThreshold { + panic(fmt.Sprintf("The number of candidates(%d) cannot be less samplingThreshold %d", + len(candidates), samplingThreshold)) + } - if len(candidates) < limitCandidates { - panic("The number of candidates cannot be less limitCandidate") + if priorityRateThreshold > 1 { + panic(fmt.Sprintf("priorityRateThreshold cannot be greater than 1.0: %f", priorityRateThreshold)) } + totalPriority := sumTotalPriority(candidates) + priorityThreshold := uint64(math.Ceil(float64(totalPriority) * priorityRateThreshold)) + if priorityThreshold > totalPriority { + // This can be possible because of float64's precision when priorityRateThreshold is 1 and totalPriority is very big + priorityThreshold = totalPriority + } candidates = sort(candidates) - totalSampling := uint64(0) - winCandidates := make(map[Candidate]bool) - for len(winCandidates) < limitCandidates && totalSampling < MaxSamplingLoopTry { - threshold := uint64(float64(nextRandom(&seed)&uint64Mask) / float64(uint64Mask+1) * float64(totalPriority)) + winnersPriority := uint64(0) + losersPriorities := make([]uint64, len(candidates)) + winnerNum := 0 + for winnerNum < samplingThreshold || winnersPriority < priorityThreshold { + threshold := uint64(float64(nextRandom(&seed)&uint64Mask) / float64(uint64Mask+1) * float64(totalPriority-winnersPriority)) cumulativePriority := uint64(0) found := false - for _, candidate := range candidates { + for i, candidate := range candidates[:len(candidates)-winnerNum] { if threshold < cumulativePriority+candidate.Priority() { - if !winCandidates[candidate] { - winCandidates[candidate] = true - } - candidate.IncreaseWin() - totalSampling++ + moveWinnerToLast(candidates, i) + winnersPriority += candidate.Priority() + losersPriorities[winnerNum] = totalPriority - winnersPriority + winnerNum++ found = true break } @@ -101,15 +115,30 @@ func RandomSamplingToMax( } if !found { - panic(fmt.Sprintf("Cannot find random sample. totalPriority may be wrong: totalPriority=%d, "+ - "actualTotalPriority=%d, threshold=%d", totalPriority, sumTotalPriority(candidates), threshold)) + panic(fmt.Sprintf("Cannot find random sample. winnerNum=%d, samplingThreshold=%d, "+ + "winnersPriority=%d, priorityThreshold=%d, totalPriority=%d, threshold=%d", + winnerNum, samplingThreshold, winnersPriority, priorityThreshold, totalPriority, threshold)) } } - return totalSampling + + compensationRewardProportions := make([]float64, winnerNum) + for i := winnerNum - 2; i >= 0; i-- { // last winner doesn't get compensation reward + compensationRewardProportions[i] = compensationRewardProportions[i+1] + 1/float64(losersPriorities[i]) + } + winners = candidates[len(candidates)-winnerNum:] + for i, winner := range winners { + // TODO protect overflow and verify the accuracy of the calculations. + // voter.Priority()*rewardUnit can be overflow, so we multiply voter.Priority() and rewardProportion at first + winner.Reward(rewardUnit + uint64((float64(winner.Priority())*compensationRewardProportions[i])*float64(rewardUnit))) + } + return } func sumTotalPriority(candidates []Candidate) (sum uint64) { for _, candi := range candidates { + if candi.Priority() == 0 { + panic("candidate(%d) priority must not be 0") + } sum += candi.Priority() } return diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 3ed78e0dc..30929e942 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -5,16 +5,18 @@ import ( "math" s "sort" "testing" + + "github.com/stretchr/testify/assert" ) type Element struct { - ID uint32 - Win uint64 - Weight uint64 + id uint32 + reward uint64 + weight uint64 } func (e *Element) Priority() uint64 { - return e.Weight + return e.weight } func (e *Element) LessThan(other Candidate) bool { @@ -22,11 +24,11 @@ func (e *Element) LessThan(other Candidate) bool { if !ok { panic("incompatible type") } - return e.ID < o.ID + return e.id < o.id } -func (e *Element) IncreaseWin() { - e.Win++ +func (e *Element) Reward(reward uint64) { + e.reward += reward } func TestRandomSamplingWithPriority(t *testing.T) { @@ -52,7 +54,7 @@ func TestRandomSamplingWithPriority(t *testing.T) { for i := 0; i < 100000; i++ { elected = RandomSamplingWithPriority(uint64(i), candidates, 10, uint64(len(candidates))) for _, e := range elected { - counts[e.(*Element).ID]++ + counts[e.(*Element).id]++ } } expected := float64(1) / float64(100) @@ -89,52 +91,136 @@ func TestRandomSamplingPanicCase(t *testing.T) { } } -func numberOfWinnersAndWins(candidate []Candidate) (winners uint64, totalWins uint64) { +func resetReward(candidate []Candidate) { for _, c := range candidate { - if c.(*Element).Win > 0 { - winners++ - totalWins += c.(*Element).Win - } + c.(*Element).reward = 0 } - return } -func TestRandomSamplingToMax(t *testing.T) { - candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i) }) - voters1 := RandomSamplingToMax(0, candidates1, 10, sumTotalPriority(candidates1)) - winners, totalWins := numberOfWinnersAndWins(candidates1) - if winners != 10 { - t.Errorf(fmt.Sprintf("unexpected sample size: %d", winners)) +func totalReward(candidate []Candidate) uint64 { + total := uint64(0) + for _, candi := range candidate { + total += candi.(*Element).reward } - if voters1 != totalWins { - t.Errorf(fmt.Sprintf("unexpected totalWins: %d", voters1)) + return total +} + +func TestRandomSamplingWithoutReplacement1Candidate(t *testing.T) { + candidates := newCandidates(1, func(i int) uint64 { return uint64(1000 * (i + 1)) }) + + winners := RandomSamplingWithoutReplacement(0, candidates, 1, 1, 1000) + assert.True(t, len(winners) == 1) + assert.True(t, candidates[0] == winners[0]) + assert.True(t, winners[0].(*Element).reward == 1000) + resetReward(candidates) + + winners2 := RandomSamplingWithoutReplacement(0, candidates, 1, 0.5, 1000) + assert.True(t, len(winners2) == 1) + assert.True(t, candidates[0] == winners2[0]) + assert.True(t, winners2[0].(*Element).reward == 1000) + resetReward(candidates) + + winners3 := RandomSamplingWithoutReplacement(0, candidates, 0, 0.5, 1000) + assert.True(t, len(winners3) == 1) + assert.True(t, candidates[0] == winners3[0]) + assert.True(t, winners3[0].(*Element).reward == 1000) + resetReward(candidates) + + winners4 := RandomSamplingWithoutReplacement(0, candidates, 0, 0, 1000) + assert.True(t, len(winners4) == 0) + resetReward(candidates) +} + +// test samplingThreshold +func TestRandomSamplingWithoutReplacementSamplingThreshold(t *testing.T) { + candidates := newCandidates(100, func(i int) uint64 { return uint64(1000 * (i + 1)) }) + + for i := 1; i <= 100; i++ { + winners := RandomSamplingWithoutReplacement(0, candidates, i, 0.001, 1000) + assert.True(t, len(winners) == i) + resetReward(candidates) } +} - candidates2 := newCandidates(100, func(i int) uint64 { return uint64(i) }) - _ = RandomSamplingToMax(0, candidates2, 10, sumTotalPriority(candidates2)) +// test priorityRateThrshold +func TestRandomSamplingWithoutReplacementPriorityRateThrshold(t *testing.T) { + candidates := newCandidates(100, func(i int) uint64 { return uint64(1000) }) - if !sameCandidates(candidates1, candidates2) { - t.Error("The two voter sets elected by the same seed are different.") + for i := 1; i <= 100; i++ { + winners := RandomSamplingWithoutReplacement(0, candidates, 1, 0.01*float64(i), 1000) + assert.True(t, sumTotalPriority(winners) >= uint64(math.Ceil(0.01*float64(i)*float64(sumTotalPriority(candidates))))) + resetReward(candidates) } +} - candidates3 := newCandidates(0, func(i int) uint64 { return uint64(i) }) - voters3 := RandomSamplingToMax(0, candidates3, 0, sumTotalPriority(candidates3)) - if voters3 != 0 { - t.Errorf(fmt.Sprintf("unexpected totalWins: %d", voters3)) +// test random election should be deterministic +func TestRandomSamplingWithoutReplacementDeterministic(t *testing.T) { + candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) + candidates2 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) + for i := 1; i <= 100; i++ { + winners1 := RandomSamplingWithoutReplacement(uint64(i), candidates1, 50, 0.5, 1000) + winners2 := RandomSamplingWithoutReplacement(uint64(i), candidates2, 50, 0.5, 1000) + sameCandidates(winners1, winners2) + resetReward(candidates1) + resetReward(candidates2) } } -func TestRandomSamplingToMaxPanic(t *testing.T) { +func accumulateAndResetReward(candidate []Candidate, acc []uint64) { + for i, c := range candidate { + acc[i] += c.(*Element).reward + c.(*Element).reward = 0 + } +} + +// test reward fairness +func TestRandomSamplingWithoutReplacementReward(t *testing.T) { + candidates := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) + + accumulatedRewards := make([]uint64, 100) + for i := 0; i < 100000; i++ { + // 20 samplingThreshold is minimum to pass this test + // If samplingThreshold is less than 20, the result says the reward is not fair + RandomSamplingWithoutReplacement(uint64(i), candidates, 20, 0.2, 10) + accumulateAndResetReward(candidates, accumulatedRewards) + } + for i := 0; i < 99; i++ { + assert.True(t, accumulatedRewards[i] < accumulatedRewards[i+1]) + } + + accumulatedRewards = make([]uint64, 100) + for i := 0; i < 50000; i++ { + RandomSamplingWithoutReplacement(uint64(i), candidates, 50, 0.5, 10) + accumulateAndResetReward(candidates, accumulatedRewards) + } + for i := 0; i < 99; i++ { + assert.True(t, accumulatedRewards[i] < accumulatedRewards[i+1]) + } + + accumulatedRewards = make([]uint64, 100) + for i := 0; i < 10000; i++ { + RandomSamplingWithoutReplacement(uint64(i), candidates, 100, 1, 10) + accumulateAndResetReward(candidates, accumulatedRewards) + } + for i := 0; i < 99; i++ { + assert.True(t, accumulatedRewards[i] < accumulatedRewards[i+1]) + } +} + +func TestRandomSamplingWithoutReplacementPanic(t *testing.T) { type Case struct { - Candidates []Candidate - TotalPriority uint64 + Candidates []Candidate + SamplingThreshold int + PriorityRateThreshold float64 } cases := [...]*Case{ - // specified total priority is greater than actual one - {newCandidates(10, func(i int) uint64 { return 1 }), 50000}, - // limitCandidates is greater than the number of candidates - {newCandidates(5, func(i int) uint64 { return 10 }), 5}, + // samplingThreshold is greater than the number of candidates + {newCandidates(9, func(i int) uint64 { return 10 }), 10, 0.5}, + // priorityRateThreshold is greater than 1 + {newCandidates(10, func(i int) uint64 { return 10 }), 10, 1.01}, + // a candidate priority is 0 + {newCandidates(10, func(i int) uint64 { return uint64(i) }), 10, 0.99}, } for i, c := range cases { @@ -144,7 +230,7 @@ func TestRandomSamplingToMaxPanic(t *testing.T) { t.Errorf("expected panic didn't happen in case %d", i+1) } }() - RandomSamplingToMax(0, c.Candidates, 10, c.TotalPriority) + RandomSamplingWithoutReplacement(0, c.Candidates, c.SamplingThreshold, c.PriorityRateThreshold, 1000) }() } } @@ -164,10 +250,10 @@ func sameCandidates(c1 []Candidate, c2 []Candidate) bool { s.Slice(c1, func(i, j int) bool { return c1[i].LessThan(c1[j]) }) s.Slice(c2, func(i, j int) bool { return c2[i].LessThan(c2[j]) }) for i := 0; i < len(c1); i++ { - if c1[i].(*Element).ID != c2[i].(*Element).ID { + if c1[i].(*Element).id != c2[i].(*Element).id { return false } - if c1[i].(*Element).Win != c2[i].(*Element).Win { + if c1[i].(*Element).reward != c2[i].(*Element).reward { return false } } diff --git a/types/validator.go b/types/validator.go index 644df13f6..6c977c9b1 100644 --- a/types/validator.go +++ b/types/validator.go @@ -100,6 +100,26 @@ func (v *Validator) Bytes() []byte { }) } +// for implement Candidate of rand package +func (v *Validator) Priority() uint64 { + if v.StakingPower < 0 { + panic(fmt.Sprintf("staking power is negative: %d", v.StakingPower)) + } + return uint64(v.StakingPower) +} + +func (v *Validator) LessThan(other tmrand.Candidate) bool { + _, ok := other.(*Validator) + if !ok { + panic("incompatible type") + } + return bytes.Compare(v.Address, v.Address) < 0 +} + +func (v *Validator) Reward(rewards uint64) { + v.VotingPower = int64(rewards) +} + //---------------------------------------- // RandValidator diff --git a/types/validator_set.go b/types/validator_set.go index 4185379bc..95e761442 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -603,11 +603,10 @@ func (vals *ValidatorSet) SelectProposer(proofHash []byte, height int64, round i seed := hashToSeed(MakeRoundHash(proofHash, height, round)) candidates := make([]tmrand.Candidate, len(vals.Validators)) for i, val := range vals.Validators { - candidates[i] = &candidate{idx: i, val: val} + candidates[i] = val } samples := tmrand.RandomSamplingWithPriority(seed, candidates, 1, uint64(vals.TotalStakingPower())) - proposerIdx := samples[0].(*candidate).idx - return vals.Validators[proposerIdx] + return samples[0].(*Validator) } //---------------- diff --git a/types/voter_set.go b/types/voter_set.go index cb010fdc3..3aee8547b 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/binary" "fmt" - "math" "sort" "strings" @@ -15,7 +14,11 @@ import ( tmrand "github.com/tendermint/tendermint/libs/rand" ) -var MaxVoters = 20 +var ( + MinVoters = 20 + MinTotalVotingPowerRate = 0.6 + VotingPowerUnit = uint64(10000) +) // VoterSet represent a set of *Validator at a given height. type VoterSet struct { @@ -86,8 +89,9 @@ func copyValidatorListShallow(vals []*Validator) []*Validator { // VoterSet.Copy() copies validator list shallow func (voters *VoterSet) Copy() *VoterSet { return &VoterSet{ - Voters: copyValidatorListShallow(voters.Voters), - totalVotingPower: voters.totalVotingPower, + Voters: copyValidatorListShallow(voters.Voters), + totalStakingPower: voters.totalStakingPower, + totalVotingPower: voters.totalVotingPower, } } @@ -408,7 +412,7 @@ func (voters *VoterSet) StringIndented(indent string) string { func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { // TODO: decide MaxVoters; make it to config - if len(proofHash) == 0 || validators.Size() <= MaxVoters { + if len(proofHash) == 0 || validators.Size() <= MinVoters { // height 1 has voter set that is same to validator set return ToVoterAll(validators) } @@ -416,31 +420,16 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { seed := hashToSeed(proofHash) candidates := make([]tmrand.Candidate, len(validators.Validators)) for i, val := range validators.Validators { - candidates[i] = &candidate{idx: i, win: 0, val: val} - } - totalSampling := tmrand.RandomSamplingToMax(seed, candidates, MaxVoters, uint64(validators.TotalStakingPower())) - voters := 0 - for _, candi := range candidates { - if candi.(*candidate).win > 0 { - voters += 1 - } + candidates[i] = val.Copy() } - vals := make([]*Validator, voters) - index := 0 - for _, candi := range candidates { - if candi.(*candidate).win > 0 { - vals[index] = &Validator{Address: candi.(*candidate).val.Address, - PubKey: candi.(*candidate).val.PubKey, - StakingPower: candi.(*candidate).val.StakingPower, - // StakingPower = TotalStakingPower * win / totalSampling : can be overflow - VotingPower: validators.TotalStakingPower()/int64(totalSampling)*int64(candi.(*candidate).win) + - int64(math.Ceil(float64(validators.TotalStakingPower()%int64(totalSampling))/float64(int64(totalSampling))* - float64(candi.(*candidate).win)))} - index++ - } + winners := tmrand.RandomSamplingWithoutReplacement(seed, candidates, MinVoters, MinTotalVotingPowerRate, + VotingPowerUnit) + voters := make([]*Validator, len(winners)) + for i, winner := range winners { + voters[i] = winner.(*Validator) } - return NewVoterSet(vals) + return NewVoterSet(voters) } // This should be used in only test @@ -456,38 +445,11 @@ func ToVoterAll(validators *ValidatorSet) *VoterSet { } } voters := NewVoterSet(newVoters) - voters.updateTotalVotingPower() voters.updateTotalStakingPower() + voters.updateTotalVotingPower() return voters } -// candidate save simple validator data for selecting proposer -type candidate struct { - idx int - win uint64 - val *Validator -} - -func (c *candidate) Priority() uint64 { - // TODO Is it possible to have a negative StakingPower? - if c.val.StakingPower < 0 { - return 0 - } - return uint64(c.val.StakingPower) -} - -func (c *candidate) LessThan(other tmrand.Candidate) bool { - o, ok := other.(*candidate) - if !ok { - panic("incompatible type") - } - return bytes.Compare(c.val.Address, o.val.Address) < 0 -} - -func (c *candidate) IncreaseWin() { - c.win++ -} - func hashToSeed(hash []byte) uint64 { for len(hash) < 8 { hash = append(hash, byte(0)) diff --git a/types/voter_set_test.go b/types/voter_set_test.go index 0d769a07d..ebff36258 100644 --- a/types/voter_set_test.go +++ b/types/voter_set_test.go @@ -10,11 +10,19 @@ import ( ) func TestSelectVoter(t *testing.T) { - MaxVoters = 29 + MinVoters = 29 + MinTotalVotingPowerRate = 1 valSet := randValidatorSet(30) for i := 0; i < 10000; i++ { voterSet := SelectVoter(valSet, []byte{byte(i)}) - assert.True(t, math.Abs(float64(valSet.TotalStakingPower()-voterSet.TotalVotingPower())) <= 10) + assert.True(t, voterSet.Size() >= 29) + if voterSet.totalVotingPower <= 0 { + for j := 0; j < voterSet.Size(); j++ { + // TODO solve this problem!!! + t.Logf("voter voting power = %d", voterSet.Voters[j].VotingPower) + } + } + assert.True(t, voterSet.TotalVotingPower() > 0) } } @@ -59,7 +67,7 @@ func findLargestStakingPowerGap(t *testing.T, loopCount int, minMaxRate int, max Validators: toGenesisValidators(valSet.Validators), } hash := genDoc.Hash() - MaxVoters = maxVoters + MinVoters = maxVoters accumulation := make(map[string]int64) totalVoters := 0 for i := 0; i < loopCount; i++ { @@ -96,11 +104,10 @@ func TestSelectVoterMaxVarious(t *testing.T) { t.Logf("<<< min: 100, max: %d >>>", 100*minMaxRate) for validators := 16; validators <= 256; validators *= 4 { for voters := 1; voters <= validators; voters += 10 { - MaxVoters = voters + MinVoters = voters valSet, _ := randValidatorSetWithMinMax(validators, 100, 100*int64(minMaxRate)) voterSet := SelectVoter(valSet, []byte{byte(hash)}) - assert.True(t, int(math.Abs(float64(valSet.TotalStakingPower()-voterSet.TotalVotingPower()))) <= voters) - if voterSet.Size() < MaxVoters { + if voterSet.Size() < MinVoters { t.Logf("Cannot elect voters up to MaxVoters: validators=%d, MaxVoters=%d, actual voters=%d", validators, voters, voterSet.Size()) break From 3749b65911ef2a5221a746edd77840df2e9224e8 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Fri, 29 May 2020 16:52:26 +0900 Subject: [PATCH 04/38] fix: lint error --- libs/rand/sampling.go | 13 ++++++++----- libs/rand/sampling_test.go | 8 -------- types/validator.go | 4 ++-- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 4d94bcb77..247fbf0ff 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -73,8 +73,9 @@ func moveWinnerToLast(candidates []Candidate, winner int) { candidates[len(candidates)-1] = winnerCandidate } -// `RandomSamplingWithoutReplacement` elects winners among candidates without replacement so it updates rewards of winners. -// This function continues to elect winners until the both of two conditions(samplingThreshold, priorityRateThreshold) are met. +// `RandomSamplingWithoutReplacement` elects winners among candidates without replacement +// so it updates rewards of winners. This function continues to elect winners until the both of two +// conditions(samplingThreshold, priorityRateThreshold) are met. func RandomSamplingWithoutReplacement( seed uint64, candidates []Candidate, samplingThreshold int, priorityRateThreshold float64, rewardUnit uint64) ( winners []Candidate) { @@ -91,7 +92,8 @@ func RandomSamplingWithoutReplacement( totalPriority := sumTotalPriority(candidates) priorityThreshold := uint64(math.Ceil(float64(totalPriority) * priorityRateThreshold)) if priorityThreshold > totalPriority { - // This can be possible because of float64's precision when priorityRateThreshold is 1 and totalPriority is very big + // This can be possible because of float64's precision + // when priorityRateThreshold is 1 and totalPriority is very big priorityThreshold = totalPriority } candidates = sort(candidates) @@ -99,7 +101,8 @@ func RandomSamplingWithoutReplacement( losersPriorities := make([]uint64, len(candidates)) winnerNum := 0 for winnerNum < samplingThreshold || winnersPriority < priorityThreshold { - threshold := uint64(float64(nextRandom(&seed)&uint64Mask) / float64(uint64Mask+1) * float64(totalPriority-winnersPriority)) + threshold := uint64(float64(nextRandom(&seed)&uint64Mask) / + float64(uint64Mask+1) * float64(totalPriority-winnersPriority)) cumulativePriority := uint64(0) found := false for i, candidate := range candidates[:len(candidates)-winnerNum] { @@ -131,7 +134,7 @@ func RandomSamplingWithoutReplacement( // voter.Priority()*rewardUnit can be overflow, so we multiply voter.Priority() and rewardProportion at first winner.Reward(rewardUnit + uint64((float64(winner.Priority())*compensationRewardProportions[i])*float64(rewardUnit))) } - return + return winners } func sumTotalPriority(candidates []Candidate) (sum uint64) { diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 30929e942..d8ec299b4 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -97,14 +97,6 @@ func resetReward(candidate []Candidate) { } } -func totalReward(candidate []Candidate) uint64 { - total := uint64(0) - for _, candi := range candidate { - total += candi.(*Element).reward - } - return total -} - func TestRandomSamplingWithoutReplacement1Candidate(t *testing.T) { candidates := newCandidates(1, func(i int) uint64 { return uint64(1000 * (i + 1)) }) diff --git a/types/validator.go b/types/validator.go index 6c977c9b1..a041b6c29 100644 --- a/types/validator.go +++ b/types/validator.go @@ -109,11 +109,11 @@ func (v *Validator) Priority() uint64 { } func (v *Validator) LessThan(other tmrand.Candidate) bool { - _, ok := other.(*Validator) + o, ok := other.(*Validator) if !ok { panic("incompatible type") } - return bytes.Compare(v.Address, v.Address) < 0 + return bytes.Compare(v.Address, o.Address) < 0 } func (v *Validator) Reward(rewards uint64) { From 521cdc1e1ad0099aee188fd5f0913b3879cba2d0 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 2 Jun 2020 16:42:31 +0900 Subject: [PATCH 05/38] feat: implement assigning voting power --- consensus/common_test.go | 2 +- consensus/state_test.go | 2 +- libs/rand/sampling.go | 60 +++++++++++++++------------- libs/rand/sampling_test.go | 78 ++++++++++++++++++------------------- types/validator.go | 20 ---------- types/validator_set.go | 7 +++- types/validator_set_test.go | 36 ++++++++--------- types/voter_set.go | 47 ++++++++++++++++++---- types/voter_set_test.go | 2 +- 9 files changed, 137 insertions(+), 117 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index fb3a890ea..54eb94ea7 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -387,7 +387,7 @@ func loadPrivValidator(config *cfg.Config) *privval.FilePV { func randState(nValidators int) (*State, []*validatorStub) { // Get State state, privVals := randGenesisState(nValidators, false, 10) - state.LastProofHash = []byte{2} + state.LastProofHash = []byte{3} vss := make([]*validatorStub, nValidators) diff --git a/consensus/state_test.go b/consensus/state_test.go index a639e5ce8..12866869b 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -182,7 +182,7 @@ func TestStateBadProposal(t *testing.T) { proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) voteCh := subscribe(cs1.eventBus, types.EventQueryVote) - propBlock, _ := cs1.createProposalBlock(round) //changeProposer(t, cs1, vs2) + propBlock, _ := cs1.createProposalBlock(round) // changeProposer(t, cs1, vs2) // make the second validator the proposer by incrementing round round++ diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 247fbf0ff..d10c7be6d 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -2,7 +2,6 @@ package rand import ( "fmt" - "math" s "sort" ) @@ -10,10 +9,13 @@ import ( type Candidate interface { Priority() uint64 LessThan(other Candidate) bool - Reward(rewards uint64) + SetWinPoint(winPoint uint64) } -const uint64Mask = uint64(0x7FFFFFFFFFFFFFFF) +const ( + // Casting a larger number than this as float64 can result in that lower bytes can be truncated + MaxFloat64Significant = uint64(0x1FFFFFFFFFFFFF) +) // Select a specified number of candidates randomly from the candidate set based on each priority. This function is // deterministic and will produce the same result for the same input. @@ -34,7 +36,7 @@ func RandomSamplingWithPriority( thresholds := make([]uint64, sampleSize) for i := 0; i < sampleSize; i++ { // calculating [gross weights] × [(0,1] random number] - thresholds[i] = uint64(float64(nextRandom(&seed)&uint64Mask) / float64(uint64Mask+1) * float64(totalPriority)) + thresholds[i] = randomThreshold(&seed, totalPriority) } s.Slice(thresholds, func(i, j int) bool { return thresholds[i] < thresholds[j] }) @@ -73,36 +75,40 @@ func moveWinnerToLast(candidates []Candidate, winner int) { candidates[len(candidates)-1] = winnerCandidate } +func randomThreshold(seed *uint64, total uint64) uint64 { + return uint64(float64(nextRandom(seed)&MaxFloat64Significant) / + float64(MaxFloat64Significant+1) * float64(total)) +} + // `RandomSamplingWithoutReplacement` elects winners among candidates without replacement // so it updates rewards of winners. This function continues to elect winners until the both of two -// conditions(samplingThreshold, priorityRateThreshold) are met. +// conditions(minSamplingCount, minPriorityPercent) are met. func RandomSamplingWithoutReplacement( - seed uint64, candidates []Candidate, samplingThreshold int, priorityRateThreshold float64, rewardUnit uint64) ( + seed uint64, candidates []Candidate, minSamplingCount int, minPriorityPercent uint, winPointUnit uint64) ( winners []Candidate) { - if len(candidates) < samplingThreshold { - panic(fmt.Sprintf("The number of candidates(%d) cannot be less samplingThreshold %d", - len(candidates), samplingThreshold)) + if len(candidates) < minSamplingCount { + panic(fmt.Sprintf("The number of candidates(%d) cannot be less minSamplingCount %d", + len(candidates), minSamplingCount)) } - if priorityRateThreshold > 1 { - panic(fmt.Sprintf("priorityRateThreshold cannot be greater than 1.0: %f", priorityRateThreshold)) + if minPriorityPercent > 100 { + panic(fmt.Sprintf("minPriorityPercent must be equal or less than 100: %d", minPriorityPercent)) } totalPriority := sumTotalPriority(candidates) - priorityThreshold := uint64(math.Ceil(float64(totalPriority) * priorityRateThreshold)) - if priorityThreshold > totalPriority { - // This can be possible because of float64's precision - // when priorityRateThreshold is 1 and totalPriority is very big - priorityThreshold = totalPriority + if totalPriority > MaxFloat64Significant { + // totalPriority will be casting to float64, so it must be less than 0x1FFFFFFFFFFFFF(53bits) + panic(fmt.Sprintf("total priority cannot exceed %d: %d", MaxFloat64Significant, totalPriority)) } + + priorityThreshold := totalPriority * uint64(minPriorityPercent) / 100 candidates = sort(candidates) winnersPriority := uint64(0) losersPriorities := make([]uint64, len(candidates)) winnerNum := 0 - for winnerNum < samplingThreshold || winnersPriority < priorityThreshold { - threshold := uint64(float64(nextRandom(&seed)&uint64Mask) / - float64(uint64Mask+1) * float64(totalPriority-winnersPriority)) + for winnerNum < minSamplingCount || winnersPriority < priorityThreshold { + threshold := randomThreshold(&seed, totalPriority-winnersPriority) cumulativePriority := uint64(0) found := false for i, candidate := range candidates[:len(candidates)-winnerNum] { @@ -118,21 +124,19 @@ func RandomSamplingWithoutReplacement( } if !found { - panic(fmt.Sprintf("Cannot find random sample. winnerNum=%d, samplingThreshold=%d, "+ + panic(fmt.Sprintf("Cannot find random sample. winnerNum=%d, minSamplingCount=%d, "+ "winnersPriority=%d, priorityThreshold=%d, totalPriority=%d, threshold=%d", - winnerNum, samplingThreshold, winnersPriority, priorityThreshold, totalPriority, threshold)) + winnerNum, minSamplingCount, winnersPriority, priorityThreshold, totalPriority, threshold)) } } - - compensationRewardProportions := make([]float64, winnerNum) + compensationProportions := make([]float64, winnerNum) for i := winnerNum - 2; i >= 0; i-- { // last winner doesn't get compensation reward - compensationRewardProportions[i] = compensationRewardProportions[i+1] + 1/float64(losersPriorities[i]) + compensationProportions[i] = compensationProportions[i+1] + 1/float64(losersPriorities[i]) } winners = candidates[len(candidates)-winnerNum:] for i, winner := range winners { - // TODO protect overflow and verify the accuracy of the calculations. - // voter.Priority()*rewardUnit can be overflow, so we multiply voter.Priority() and rewardProportion at first - winner.Reward(rewardUnit + uint64((float64(winner.Priority())*compensationRewardProportions[i])*float64(rewardUnit))) + winner.SetWinPoint(winPointUnit + + uint64(float64(winner.Priority())*compensationProportions[i]*float64(winPointUnit))) } return winners } @@ -144,7 +148,7 @@ func sumTotalPriority(candidates []Candidate) (sum uint64) { } sum += candi.Priority() } - return + return sum } // SplitMix64 diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index d8ec299b4..83aa3bf8e 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -10,9 +10,9 @@ import ( ) type Element struct { - id uint32 - reward uint64 - weight uint64 + id uint32 + winPoint uint64 + weight uint64 } func (e *Element) Priority() uint64 { @@ -27,8 +27,8 @@ func (e *Element) LessThan(other Candidate) bool { return e.id < o.id } -func (e *Element) Reward(reward uint64) { - e.reward += reward +func (e *Element) SetWinPoint(winPoint uint64) { + e.winPoint += winPoint } func TestRandomSamplingWithPriority(t *testing.T) { @@ -91,36 +91,36 @@ func TestRandomSamplingPanicCase(t *testing.T) { } } -func resetReward(candidate []Candidate) { +func resetWinPoint(candidate []Candidate) { for _, c := range candidate { - c.(*Element).reward = 0 + c.(*Element).winPoint = 0 } } func TestRandomSamplingWithoutReplacement1Candidate(t *testing.T) { candidates := newCandidates(1, func(i int) uint64 { return uint64(1000 * (i + 1)) }) - winners := RandomSamplingWithoutReplacement(0, candidates, 1, 1, 1000) + winners := RandomSamplingWithoutReplacement(0, candidates, 1, 100, 1000) assert.True(t, len(winners) == 1) assert.True(t, candidates[0] == winners[0]) - assert.True(t, winners[0].(*Element).reward == 1000) - resetReward(candidates) + assert.True(t, winners[0].(*Element).winPoint == 1000) + resetWinPoint(candidates) - winners2 := RandomSamplingWithoutReplacement(0, candidates, 1, 0.5, 1000) + winners2 := RandomSamplingWithoutReplacement(0, candidates, 1, 50, 1000) assert.True(t, len(winners2) == 1) assert.True(t, candidates[0] == winners2[0]) - assert.True(t, winners2[0].(*Element).reward == 1000) - resetReward(candidates) + assert.True(t, winners2[0].(*Element).winPoint == 1000) + resetWinPoint(candidates) - winners3 := RandomSamplingWithoutReplacement(0, candidates, 0, 0.5, 1000) + winners3 := RandomSamplingWithoutReplacement(0, candidates, 0, 50, 1000) assert.True(t, len(winners3) == 1) assert.True(t, candidates[0] == winners3[0]) - assert.True(t, winners3[0].(*Element).reward == 1000) - resetReward(candidates) + assert.True(t, winners3[0].(*Element).winPoint == 1000) + resetWinPoint(candidates) winners4 := RandomSamplingWithoutReplacement(0, candidates, 0, 0, 1000) assert.True(t, len(winners4) == 0) - resetReward(candidates) + resetWinPoint(candidates) } // test samplingThreshold @@ -128,9 +128,9 @@ func TestRandomSamplingWithoutReplacementSamplingThreshold(t *testing.T) { candidates := newCandidates(100, func(i int) uint64 { return uint64(1000 * (i + 1)) }) for i := 1; i <= 100; i++ { - winners := RandomSamplingWithoutReplacement(0, candidates, i, 0.001, 1000) + winners := RandomSamplingWithoutReplacement(0, candidates, i, 1, 1000) assert.True(t, len(winners) == i) - resetReward(candidates) + resetWinPoint(candidates) } } @@ -138,10 +138,10 @@ func TestRandomSamplingWithoutReplacementSamplingThreshold(t *testing.T) { func TestRandomSamplingWithoutReplacementPriorityRateThrshold(t *testing.T) { candidates := newCandidates(100, func(i int) uint64 { return uint64(1000) }) - for i := 1; i <= 100; i++ { - winners := RandomSamplingWithoutReplacement(0, candidates, 1, 0.01*float64(i), 1000) - assert.True(t, sumTotalPriority(winners) >= uint64(math.Ceil(0.01*float64(i)*float64(sumTotalPriority(candidates))))) - resetReward(candidates) + for i := 7; i <= 100; i++ { + winners := RandomSamplingWithoutReplacement(0, candidates, 1, uint(i), 1000) + assert.True(t, sumTotalPriority(winners) >= uint64(0.01*float64(i)*float64(sumTotalPriority(candidates)))) + resetWinPoint(candidates) } } @@ -150,18 +150,18 @@ func TestRandomSamplingWithoutReplacementDeterministic(t *testing.T) { candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) candidates2 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) for i := 1; i <= 100; i++ { - winners1 := RandomSamplingWithoutReplacement(uint64(i), candidates1, 50, 0.5, 1000) - winners2 := RandomSamplingWithoutReplacement(uint64(i), candidates2, 50, 0.5, 1000) + winners1 := RandomSamplingWithoutReplacement(uint64(i), candidates1, 50, 50, 1000) + winners2 := RandomSamplingWithoutReplacement(uint64(i), candidates2, 50, 50, 1000) sameCandidates(winners1, winners2) - resetReward(candidates1) - resetReward(candidates2) + resetWinPoint(candidates1) + resetWinPoint(candidates2) } } func accumulateAndResetReward(candidate []Candidate, acc []uint64) { for i, c := range candidate { - acc[i] += c.(*Element).reward - c.(*Element).reward = 0 + acc[i] += c.(*Element).winPoint + c.(*Element).winPoint = 0 } } @@ -171,9 +171,9 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards := make([]uint64, 100) for i := 0; i < 100000; i++ { - // 20 samplingThreshold is minimum to pass this test - // If samplingThreshold is less than 20, the result says the reward is not fair - RandomSamplingWithoutReplacement(uint64(i), candidates, 20, 0.2, 10) + // 23 samplingThreshold is minimum to pass this test + // If samplingThreshold is less than 23, the result says the reward is not fair + RandomSamplingWithoutReplacement(uint64(i), candidates, 23, 23, 10) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -182,7 +182,7 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards = make([]uint64, 100) for i := 0; i < 50000; i++ { - RandomSamplingWithoutReplacement(uint64(i), candidates, 50, 0.5, 10) + RandomSamplingWithoutReplacement(uint64(i), candidates, 50, 50, 10) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -191,7 +191,7 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards = make([]uint64, 100) for i := 0; i < 10000; i++ { - RandomSamplingWithoutReplacement(uint64(i), candidates, 100, 1, 10) + RandomSamplingWithoutReplacement(uint64(i), candidates, 100, 100, 10) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -203,16 +203,16 @@ func TestRandomSamplingWithoutReplacementPanic(t *testing.T) { type Case struct { Candidates []Candidate SamplingThreshold int - PriorityRateThreshold float64 + PriorityRateThreshold uint } cases := [...]*Case{ // samplingThreshold is greater than the number of candidates - {newCandidates(9, func(i int) uint64 { return 10 }), 10, 0.5}, + {newCandidates(9, func(i int) uint64 { return 10 }), 10, 50}, // priorityRateThreshold is greater than 1 - {newCandidates(10, func(i int) uint64 { return 10 }), 10, 1.01}, + {newCandidates(10, func(i int) uint64 { return 10 }), 10, 101}, // a candidate priority is 0 - {newCandidates(10, func(i int) uint64 { return uint64(i) }), 10, 0.99}, + {newCandidates(10, func(i int) uint64 { return uint64(i) }), 10, 99}, } for i, c := range cases { @@ -245,7 +245,7 @@ func sameCandidates(c1 []Candidate, c2 []Candidate) bool { if c1[i].(*Element).id != c2[i].(*Element).id { return false } - if c1[i].(*Element).reward != c2[i].(*Element).reward { + if c1[i].(*Element).winPoint != c2[i].(*Element).winPoint { return false } } diff --git a/types/validator.go b/types/validator.go index a041b6c29..644df13f6 100644 --- a/types/validator.go +++ b/types/validator.go @@ -100,26 +100,6 @@ func (v *Validator) Bytes() []byte { }) } -// for implement Candidate of rand package -func (v *Validator) Priority() uint64 { - if v.StakingPower < 0 { - panic(fmt.Sprintf("staking power is negative: %d", v.StakingPower)) - } - return uint64(v.StakingPower) -} - -func (v *Validator) LessThan(other tmrand.Candidate) bool { - o, ok := other.(*Validator) - if !ok { - panic("incompatible type") - } - return bytes.Compare(v.Address, o.Address) < 0 -} - -func (v *Validator) Reward(rewards uint64) { - v.VotingPower = int64(rewards) -} - //---------------------------------------- // RandValidator diff --git a/types/validator_set.go b/types/validator_set.go index 95e761442..f7e52cb71 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -603,10 +603,13 @@ func (vals *ValidatorSet) SelectProposer(proofHash []byte, height int64, round i seed := hashToSeed(MakeRoundHash(proofHash, height, round)) candidates := make([]tmrand.Candidate, len(vals.Validators)) for i, val := range vals.Validators { - candidates[i] = val + candidates[i] = &candidate{ + priority: uint64(val.StakingPower), + val: val, // don't need to assign the copy + } } samples := tmrand.RandomSamplingWithPriority(seed, candidates, 1, uint64(vals.TotalStakingPower())) - return samples[0].(*Validator) + return samples[0].(*candidate).val } //---------------- diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 724e1b7fe..d78972bd7 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -176,10 +176,10 @@ func TestProposerSelection1(t *testing.T) { val := vset.SelectProposer([]byte{}, int64(i), 0) proposers = append(proposers, string(val.Address)) } - expected := `foo foo foo foo bar bar foo bar foo baz bar foo baz baz baz foo foo bar foo bar baz bar foo baz foo ` + - `foo baz foo foo baz foo foo baz bar foo foo foo baz foo baz baz bar foo foo foo foo baz bar bar bar bar foo ` + - `foo foo baz foo foo foo foo foo foo baz foo foo baz bar bar foo bar foo foo baz bar foo foo baz foo foo baz ` + - `foo foo bar foo foo baz foo foo foo bar foo foo baz baz foo foo bar baz foo baz` + expected := `foo foo foo baz bar foo baz baz foo foo baz bar foo foo foo foo bar foo foo foo bar foo foo foo bar ` + + `baz foo foo bar baz foo bar baz bar foo foo foo bar foo foo bar bar foo foo bar foo baz bar foo foo baz foo ` + + `baz foo baz foo foo bar foo bar baz foo foo baz foo foo foo foo bar baz foo baz foo baz foo foo bar baz foo ` + + `baz foo foo baz foo baz foo baz foo baz foo foo foo foo foo bar bar foo bar foo` if expected != strings.Join(proposers, " ") { t.Errorf("expected sequence of proposers was\n%v\nbut got \n%v", expected, strings.Join(proposers, " ")) } @@ -194,24 +194,24 @@ func TestProposerSelection2(t *testing.T) { val0, val1, val2 := newValidator(addr0, 100), newValidator(addr1, 100), newValidator(addr2, 100) valList := []*Validator{val0, val1, val2} vals := NewValidatorSet(valList) - expected := []int{0, 1, 0, 0, 2, 2, 0, 2, 1, 2, 2, 1, 2, 2, 2} + expected := []int{1, 1, 0, 1, 2, 0, 2, 2, 0, 0, 1, 2, 0, 1, 1} for i := 0; i < len(valList)*5; i++ { prop := vals.SelectProposer([]byte{}, int64(i), 0) if bytesToInt(prop.Address) != expected[i] { t.Fatalf("(%d): Expected %d. Got %d", i, expected[i], bytesToInt(prop.Address)) } } - verifyWinningRate(t, vals, 10000, 0.01) + verifyWinningRate(t, vals, 10000, 0.02) // One validator has more than the others *val2 = *newValidator(addr2, 400) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 10000, 0.01) + verifyWinningRate(t, vals, 10000, 0.02) // One validator has more than the others *val2 = *newValidator(addr2, 401) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 100000, 0.01) + verifyWinningRate(t, vals, 100000, 0.02) // each validator should be the proposer a proportional number of times val0, val1, val2 = newValidator(addr0, 4), newValidator(addr1, 5), newValidator(addr2, 3) @@ -223,30 +223,30 @@ func TestProposerSelection2(t *testing.T) { prop := vals.SelectProposer([]byte{}, int64(i), 0) propCount[bytesToInt(prop.Address)]++ } - fmt.Printf("%v", propCount) + fmt.Printf("%v\n", propCount) - if propCount[0] != 40257 { + if propCount[0] != 40038 { t.Fatalf( "Expected prop count for validator with 4/12 of voting power to be %d/%d. Got %d/%d", - 40185, + 40038, 10000*N, propCount[0], 10000*N, ) } - if propCount[1] != 50017 { + if propCount[1] != 50077 { t.Fatalf( "Expected prop count for validator with 5/12 of voting power to be %d/%d. Got %d/%d", - 40185, + 50077, 10000*N, propCount[1], 10000*N, ) } - if propCount[2] != 29726 { + if propCount[2] != 29885 { t.Fatalf( "Expected prop count for validator with 3/12 of voting power to be %d/%d. Got %d/%d", - 40185, + 29885, 10000*N, propCount[2], 10000*N, @@ -484,7 +484,7 @@ func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { 0 + 6*vp1 - total, // mostest once up to here 0 + 6*vp2}, 6, - vals.Validators[2]}, + vals.Validators[0]}, 6: { vals.Copy(), []int64{ @@ -500,7 +500,7 @@ func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { 0 + 8*vp1 - total, 0 + 8*vp2}, 8, - vals.Validators[2]}, + vals.Validators[0]}, 8: { vals.Copy(), []int64{ @@ -524,7 +524,7 @@ func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { 0 + 11*vp1 - total, // after 6 iters this val is "mostest" once and not in between 0 + 11*vp2 - total}, // after 10 iters this val is "mostest" once 11, - vals.Validators[1]}, + vals.Validators[0]}, } for i, tc := range tcs { tc.vals.IncrementProposerPriority(tc.times) diff --git a/types/voter_set.go b/types/voter_set.go index 3aee8547b..cb1b53837 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -15,9 +15,9 @@ import ( ) var ( - MinVoters = 20 - MinTotalVotingPowerRate = 0.6 - VotingPowerUnit = uint64(10000) + MinVoters = 20 + MinTotalVotingPowerPercent = uint(60) + VotingPowerUnit = uint64(10000) ) // VoterSet represent a set of *Validator at a given height. @@ -410,8 +410,30 @@ func (voters *VoterSet) StringIndented(indent string) string { } +type candidate struct { + priority uint64 + val *Validator +} + +// for implement Candidate of rand package +func (c *candidate) Priority() uint64 { + return c.priority +} + +func (c *candidate) LessThan(other tmrand.Candidate) bool { + o, ok := other.(*candidate) + if !ok { + panic("incompatible type") + } + return bytes.Compare(c.val.Address, o.val.Address) < 0 +} + +func (c *candidate) SetWinPoint(winPoint uint64) { + c.val.VotingPower = int64(winPoint) +} + func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { - // TODO: decide MaxVoters; make it to config + // TODO: decide MinVoters, MinTotalVotingPowerPercent; make it to config if len(proofHash) == 0 || validators.Size() <= MinVoters { // height 1 has voter set that is same to validator set return ToVoterAll(validators) @@ -419,15 +441,26 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { seed := hashToSeed(proofHash) candidates := make([]tmrand.Candidate, len(validators.Validators)) + totalPriority := uint64(0) for i, val := range validators.Validators { - candidates[i] = val.Copy() + candidates[i] = &candidate{ + priority: uint64(val.StakingPower), + val: val.Copy(), + } + totalPriority += candidates[i].Priority() } - winners := tmrand.RandomSamplingWithoutReplacement(seed, candidates, MinVoters, MinTotalVotingPowerRate, + if totalPriority > tmrand.MaxFloat64Significant { + scale := float64(totalPriority) / float64(tmrand.MaxFloat64Significant) + for _, candi := range candidates { + candi.(*candidate).priority = uint64(float64(candi.(*candidate).priority) / scale) + } + } + winners := tmrand.RandomSamplingWithoutReplacement(seed, candidates, MinVoters, MinTotalVotingPowerPercent, VotingPowerUnit) voters := make([]*Validator, len(winners)) for i, winner := range winners { - voters[i] = winner.(*Validator) + voters[i] = winner.(*candidate).val } return NewVoterSet(voters) } diff --git a/types/voter_set_test.go b/types/voter_set_test.go index ebff36258..af46ef67f 100644 --- a/types/voter_set_test.go +++ b/types/voter_set_test.go @@ -11,7 +11,7 @@ import ( func TestSelectVoter(t *testing.T) { MinVoters = 29 - MinTotalVotingPowerRate = 1 + MinTotalVotingPowerPercent = 100 valSet := randValidatorSet(30) for i := 0; i < 10000; i++ { voterSet := SelectVoter(valSet, []byte{byte(i)}) From 85a08e1c6b5a846376bf634d6fdeba33ad347046 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 2 Jun 2020 17:11:17 +0900 Subject: [PATCH 06/38] fix: lint error --- types/validator_set_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/validator_set_test.go b/types/validator_set_test.go index bbf460c69..15a9e22b3 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -518,14 +518,14 @@ func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { []int64{ 0 + 10*vp0 - 8*total, // after 10 iters this is mostest again 0 + 10*vp1 - total, // after 6 iters this val is "mostest" once and not in between - 0 + 10*vp2 - total}, // in between 10 iters this val is "mostest" once + 0 + 10*vp2 - total}, // in between 10 iters this val is "mostest" once 10, vals.Validators[0]}, 10: { vals.Copy(), []int64{ 0 + 11*vp0 - 9*total, - 0 + 11*vp1 - total, // after 6 iters this val is "mostest" once and not in between + 0 + 11*vp1 - total, // after 6 iters this val is "mostest" once and not in between 0 + 11*vp2 - total}, // after 10 iters this val is "mostest" once 11, vals.Validators[0]}, From c99b7bd90280a8b9e65563122c2797bd06299b1e Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 2 Jun 2020 17:12:25 +0900 Subject: [PATCH 07/38] fix: lint error --- state/state_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/state/state_test.go b/state/state_test.go index eca00fe92..ef89b95f1 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -726,7 +726,9 @@ func TestLargeGenesisValidator(t *testing.T) { // see: https://github.com/tendermint/tendermint/issues/2960 firstAddedValPubKey := ed25519.GenPrivKey().PubKey() firstAddedValStakingPower := int64(10) - firstAddedVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(firstAddedValPubKey), Power: firstAddedValStakingPower} + firstAddedVal := abci.ValidatorUpdate{ + PubKey: types.TM2PB.PubKey(firstAddedValPubKey), + Power: firstAddedValStakingPower} validatorUpdates, err := types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{firstAddedVal}) assert.NoError(t, err) abciResponses := &sm.ABCIResponses{ From 41fe4c24caba99847530cf0de8b89f9e877c8019 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 2 Jun 2020 17:44:31 +0900 Subject: [PATCH 08/38] fix: lite2 test failure --- lite2/client.go | 10 +++++----- types/voter_set.go | 11 ++++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lite2/client.go b/lite2/client.go index dc58ac0e2..ee33d4ddb 100644 --- a/lite2/client.go +++ b/lite2/client.go @@ -376,26 +376,26 @@ func (c *Client) initializeWithTrustOptions(options TrustOptions) error { } // 2) Fetch and verify the vals. - vals, err := c.validatorSetFromPrimary(options.Height) + voters, err := c.validatorSetFromPrimary(options.Height) if err != nil { return err } - if !bytes.Equal(h.VotersHash, vals.Hash()) { + if !bytes.Equal(h.VotersHash, voters.Hash()) { return fmt.Errorf("expected header's validators (%X) to match those that were supplied (%X)", h.VotersHash, - vals.Hash(), + voters.Hash(), ) } // Ensure that +2/3 of validators signed correctly. - err = vals.VerifyCommit(c.chainID, h.Commit.BlockID, h.Height, h.Commit) + err = voters.VerifyCommit(c.chainID, h.Commit.BlockID, h.Height, h.Commit) if err != nil { return fmt.Errorf("invalid commit: %w", err) } // 3) Persist both of them and continue. - return c.updateTrustedHeaderAndVals(h, vals) + return c.updateTrustedHeaderAndVals(h, voters) } // TrustedHeader returns a trusted header at the given height (0 - the latest). diff --git a/types/voter_set.go b/types/voter_set.go index cb1b53837..fd4820d3e 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -32,7 +32,7 @@ type VoterSet struct { func NewVoterSet(valz []*Validator) *VoterSet { sort.Sort(ValidatorsByAddress(valz)) - vals := &VoterSet{Voters: copyValidatorListShallow(valz), totalVotingPower: 0} + vals := &VoterSet{Voters: copyValidatorListForVoter(valz), totalVotingPower: 0} vals.updateTotalStakingPower() vals.updateTotalVotingPower() return vals @@ -86,6 +86,15 @@ func copyValidatorListShallow(vals []*Validator) []*Validator { return result } +func copyValidatorListForVoter(vals []*Validator) []*Validator { + result := make([]*Validator, len(vals)) + for i, v := range vals { + result[i] = v.Copy() + result[i].VotingPower = v.StakingPower + } + return result +} + // VoterSet.Copy() copies validator list shallow func (voters *VoterSet) Copy() *VoterSet { return &VoterSet{ From 0f2d1984481c8ba72cf0aba2a4af4ea864e5dcf2 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 2 Jun 2020 17:48:58 +0900 Subject: [PATCH 09/38] fix: proto generated file --- abci/types/types.pb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go index 996677469..51ff4aedd 100644 --- a/abci/types/types.pb.go +++ b/abci/types/types.pb.go @@ -16205,7 +16205,7 @@ func (m *Evidence) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TotalStakingPower", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TotalVotingPower", wireType) } m.TotalVotingPower = 0 for shift := uint(0); ; shift += 7 { From 37a2431358f335d81f8cc5308830eb9cd096a656 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 11:36:14 +0900 Subject: [PATCH 10/38] fix: diable proto-checking of circle-ci --- .github/workflows/proto.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/proto.yml b/.github/workflows/proto.yml index ddc9ee4c4..ca30d0291 100644 --- a/.github/workflows/proto.yml +++ b/.github/workflows/proto.yml @@ -8,5 +8,5 @@ jobs: - uses: docker-practice/actions-setup-docker@master - name: lint run: make proto-lint - - name: check-breakage - run: make proto-check-breaking-ci +# - name: check-breakage +# run: make proto-check-breaking-ci From 52de8808d64591614e5d379f5a14e39f6d9ddb0f Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 17:57:46 +0900 Subject: [PATCH 11/38] fix: apply comment; use VotingPower on adding vote --- types/validator.go | 2 +- types/vote_set.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/types/validator.go b/types/validator.go index f58a79497..ab34c318c 100644 --- a/types/validator.go +++ b/types/validator.go @@ -10,7 +10,7 @@ import ( ) // Volatile state for each Validator -// NOTE: The ProposerPriority is not included in Validator.Hash(); +// NOTE: The ProposerPriority, VotingPower is not included in Validator.Hash(); // make sure to update that method if changes are made here // StakingPower is the potential voting power proportional to the amount of stake, // and VotingPower is the actual voting power granted by the election process. diff --git a/types/vote_set.go b/types/vote_set.go index 902e4efcd..1b343f3e5 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -175,8 +175,8 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { } // Ensure that signer is a validator. - lookupAddr, val := voteSet.voterSet.GetByIndex(valIndex) - if val == nil { + lookupAddr, voter := voteSet.voterSet.GetByIndex(valIndex) + if voter == nil { return false, errors.Wrapf(ErrVoteInvalidValidatorIndex, "Cannot find voter %d in voterSet of size %d", valIndex, voteSet.voterSet.Size()) } @@ -198,14 +198,14 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { } // Check signature. - if err := vote.Verify(voteSet.chainID, val.PubKey); err != nil { - return false, errors.Wrapf(err, "Failed to verify vote with ChainID %s and PubKey %s", voteSet.chainID, val.PubKey) + if err := vote.Verify(voteSet.chainID, voter.PubKey); err != nil { + return false, errors.Wrapf(err, "Failed to verify vote with ChainID %s and PubKey %s", voteSet.chainID, voter.PubKey) } // Add vote and get conflicting vote if any. - added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.StakingPower) + added, conflicting := voteSet.addVerifiedVote(vote, blockKey, voter.VotingPower) if conflicting != nil { - return added, NewConflictingVoteError(val, conflicting, vote) + return added, NewConflictingVoteError(voter, conflicting, vote) } if !added { panic("Expected to add non-conflicting vote") From b2247ba870de981182fd1129296b748c89eeaf56 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:03:13 +0900 Subject: [PATCH 12/38] fix: apply comment; remove totalStakingPower from VoterSet --- consensus/reactor_test.go | 32 ++++++++++++++++---------------- state/state_test.go | 2 +- types/voter_set.go | 27 --------------------------- 3 files changed, 17 insertions(+), 44 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 359d38d1a..5124238fb 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -368,48 +368,48 @@ func TestReactorStakingPowerChange(t *testing.T) { require.NoError(t, err) val1PubKeyABCI := types.TM2PB.PubKey(val1PubKey) updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25) - previousTotalStakingPower := css[0].GetRoundState().LastVoters.TotalStakingPower() + previousTotalVotingPower := css[0].GetRoundState().LastVoters.TotalVotingPower() waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) - if css[0].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { + if css[0].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { t.Fatalf( "expected staking power to change (before: %d, after: %d)", - previousTotalStakingPower, - css[0].GetRoundState().LastVoters.TotalStakingPower()) + previousTotalVotingPower, + css[0].GetRoundState().LastVoters.TotalVotingPower()) } updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2) - previousTotalStakingPower = css[0].GetRoundState().LastVoters.TotalStakingPower() + previousTotalVotingPower = css[0].GetRoundState().LastVoters.TotalVotingPower() waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) - if css[0].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { + if css[0].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { t.Fatalf( "expected voting power to change (before: %d, after: %d)", - previousTotalStakingPower, - css[0].GetRoundState().LastVoters.TotalStakingPower()) + previousTotalVotingPower, + css[0].GetRoundState().LastVoters.TotalVotingPower()) } updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26) - previousTotalStakingPower = css[0].GetRoundState().LastVoters.TotalStakingPower() + previousTotalVotingPower = css[0].GetRoundState().LastVoters.TotalVotingPower() waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) - if css[0].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { + if css[0].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { t.Fatalf( "expected voting power to change (before: %d, after: %d)", - previousTotalStakingPower, - css[0].GetRoundState().LastVoters.TotalStakingPower()) + previousTotalVotingPower, + css[0].GetRoundState().LastVoters.TotalVotingPower()) } } @@ -477,18 +477,18 @@ func TestReactorValidatorSetChanges(t *testing.T) { require.NoError(t, err) updatePubKey1ABCI := types.TM2PB.PubKey(updateValidatorPubKey1) updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) - previousTotalStakingPower := css[nVals].GetRoundState().LastVoters.TotalStakingPower() + previousTotalVotingPower := css[nVals].GetRoundState().LastVoters.TotalVotingPower() waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1) waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1) waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css) waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css) - if css[nVals].GetRoundState().LastVoters.TotalStakingPower() == previousTotalStakingPower { + if css[nVals].GetRoundState().LastVoters.TotalVotingPower() == previousTotalVotingPower { t.Errorf( "expected staking power to change (before: %d, after: %d)", - previousTotalStakingPower, - css[nVals].GetRoundState().LastVoters.TotalStakingPower()) + previousTotalVotingPower, + css[nVals].GetRoundState().LastVoters.TotalVotingPower()) } //--------------------------------------------------------------------------- diff --git a/state/state_test.go b/state/state_test.go index ef89b95f1..5f4163e92 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -363,7 +363,7 @@ func genValSetWithPowers(powers []int64) *types.ValidatorSet { func testProposerFreq(t *testing.T, caseNum int, valSet *types.ValidatorSet) { voterSet := types.ToVoterAll(valSet) N := voterSet.Size() - totalPower := voterSet.TotalStakingPower() + totalPower := voterSet.TotalVotingPower() // run the proposer selection and track frequencies runMult := 1 diff --git a/types/voter_set.go b/types/voter_set.go index fd4820d3e..dcbd9f2a8 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -27,13 +27,11 @@ type VoterSet struct { // cached (unexported) totalVotingPower int64 - totalStakingPower int64 } func NewVoterSet(valz []*Validator) *VoterSet { sort.Sort(ValidatorsByAddress(valz)) vals := &VoterSet{Voters: copyValidatorListForVoter(valz), totalVotingPower: 0} - vals.updateTotalStakingPower() vals.updateTotalVotingPower() return vals } @@ -99,7 +97,6 @@ func copyValidatorListForVoter(vals []*Validator) []*Validator { func (voters *VoterSet) Copy() *VoterSet { return &VoterSet{ Voters: copyValidatorListShallow(voters.Voters), - totalStakingPower: voters.totalStakingPower, totalVotingPower: voters.totalVotingPower, } } @@ -122,22 +119,6 @@ func (voters *VoterSet) updateTotalVotingPower() { voters.totalVotingPower = sum } -func (voters *VoterSet) updateTotalStakingPower() { - sum := int64(0) - for _, val := range voters.Voters { - // mind overflow - sum = safeAddClip(sum, val.StakingPower) - if sum > MaxTotalStakingPower { - panic(fmt.Sprintf( - "Total voting power should be guarded to not exceed %v; got: %v", - MaxTotalStakingPower, - sum)) - } - } - - voters.totalStakingPower = sum -} - func (voters *VoterSet) TotalVotingPower() int64 { if voters.totalVotingPower == 0 { voters.updateTotalVotingPower() @@ -145,13 +126,6 @@ func (voters *VoterSet) TotalVotingPower() int64 { return voters.totalVotingPower } -func (voters *VoterSet) TotalStakingPower() int64 { - if voters.totalStakingPower == 0 { - voters.updateTotalStakingPower() - } - return voters.totalStakingPower -} - // Hash returns the Merkle root hash build using validators (as leaves) in the // set. func (voters *VoterSet) Hash() []byte { @@ -487,7 +461,6 @@ func ToVoterAll(validators *ValidatorSet) *VoterSet { } } voters := NewVoterSet(newVoters) - voters.updateTotalStakingPower() voters.updateTotalVotingPower() return voters } From 8e74da4d3700c0e0c716567966c93b7aa154cc63 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:31:02 +0900 Subject: [PATCH 13/38] fix: apply comment; fix NewVoterSet --- consensus/replay.go | 2 +- lite/client/provider.go | 2 +- lite2/helpers_test.go | 2 +- lite2/provider/http/http.go | 2 +- lite2/rpc/client.go | 2 +- state/state.go | 2 +- state/state_test.go | 2 +- types/protobuf_test.go | 2 +- types/validator_set_test.go | 6 +++--- types/voter_set.go | 25 ++++++++++++------------- 10 files changed, 23 insertions(+), 24 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index bca685d8b..3bc20a7ec 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -329,7 +329,7 @@ func (h *Handshaker) ReplayBlocks( return nil, err } state.Validators = types.NewValidatorSet(vals) - state.Voters = types.ToVoterAll(state.Validators) + state.Voters = types.ToVoterAll(state.Validators.Validators) // Should sync it with MakeGenesisState() state.NextValidators = types.NewValidatorSet(vals) state.NextVoters = types.SelectVoter(state.NextValidators, h.genDoc.Hash()) diff --git a/lite/client/provider.go b/lite/client/provider.go index 2173af233..5122eafa6 100644 --- a/lite/client/provider.go +++ b/lite/client/provider.go @@ -119,7 +119,7 @@ func (p *provider) getVoterSet(chainID string, height int64) (valset *types.Vote // TODO pass through other types of errors. return nil, lerr.ErrUnknownValidators(chainID, height) } - valset = types.NewVoterSet(res.Voters) + valset = types.WrapValidatorsToVoterSet(res.Voters) return } diff --git a/lite2/helpers_test.go b/lite2/helpers_test.go index c04cd7936..01cb743f6 100644 --- a/lite2/helpers_test.go +++ b/lite2/helpers_test.go @@ -74,7 +74,7 @@ func (pkz privKeys) ToVoters(init, inc int64) *types.VoterSet { for i, k := range pkz { res[i] = types.NewValidator(k.PubKey(), init+int64(i)*inc) } - return types.NewVoterSet(res) + return types.ToVoterAll(res) } // signHeader properly signs the header with all keys from first to last exclusive. diff --git a/lite2/provider/http/http.go b/lite2/provider/http/http.go index 91b19a761..20b26eedb 100644 --- a/lite2/provider/http/http.go +++ b/lite2/provider/http/http.go @@ -123,7 +123,7 @@ func (p *http) VoterSet(height int64) (*types.VoterSet, error) { page++ } - return types.NewVoterSet(vals), nil + return types.WrapValidatorsToVoterSet(vals), nil } func validateHeight(height int64) (*int64, error) { diff --git a/lite2/rpc/client.go b/lite2/rpc/client.go index bc95bf780..53c41a7a8 100644 --- a/lite2/rpc/client.go +++ b/lite2/rpc/client.go @@ -375,7 +375,7 @@ func (c *Client) Validators(height *int64, page, perPage int) (*ctypes.ResultVot // Verify validators. if res.Count <= res.Total { - if rH, tH := types.NewVoterSet(res.Voters).Hash(), h.VotersHash; !bytes.Equal(rH, tH) { + if rH, tH := types.WrapValidatorsToVoterSet(res.Voters).Hash(), h.VotersHash; !bytes.Equal(rH, tH) { return nil, fmt.Errorf("validators %X does not match with trusted validators %X", rH, tH) } diff --git a/state/state.go b/state/state.go index 45c66f19d..b721ba092 100644 --- a/state/state.go +++ b/state/state.go @@ -262,7 +262,7 @@ func MakeGenesisState(genDoc *types.GenesisDoc) (State, error) { NextValidators: nextValidatorSet, NextVoters: types.SelectVoter(nextValidatorSet, genDoc.Hash()), Validators: validatorSet, - Voters: types.ToVoterAll(validatorSet), + Voters: types.ToVoterAll(validatorSet.Validators), LastVoters: &types.VoterSet{}, LastHeightValidatorsChanged: 1, diff --git a/state/state_test.go b/state/state_test.go index 5f4163e92..489ebc932 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -361,7 +361,7 @@ func genValSetWithPowers(powers []int64) *types.ValidatorSet { // test a proposer appears as frequently as expected func testProposerFreq(t *testing.T, caseNum int, valSet *types.ValidatorSet) { - voterSet := types.ToVoterAll(valSet) + voterSet := types.ToVoterAll(valSet.Validators) N := voterSet.Size() totalPower := voterSet.TotalVotingPower() diff --git a/types/protobuf_test.go b/types/protobuf_test.go index ad10f50cb..ae6734af3 100644 --- a/types/protobuf_test.go +++ b/types/protobuf_test.go @@ -142,7 +142,7 @@ func TestABCIEvidence(t *testing.T) { } abciEv := TM2PB.Evidence( ev, - ToVoterAll(NewValidatorSet([]*Validator{NewValidator(pubKey, 10)})), + ToVoterAll([]*Validator{NewValidator(pubKey, 10)}), time.Now(), ) diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 15a9e22b3..33827d9c3 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -578,7 +578,7 @@ func TestValidatorSetVerifyCommit(t *testing.T) { privKey := ed25519.GenPrivKey() pubKey := privKey.PubKey() v1 := NewValidator(pubKey, 1000) - vset := ToVoterAll(NewValidatorSet([]*Validator{v1})) + vset := ToVoterAll([]*Validator{v1}) // good var ( @@ -1344,8 +1344,8 @@ func TestVerifyCommitTrusting(t *testing.T) { }, // good - first two are different but the rest of the same -> >1/3 2: { - //voterSet: NewValidatorSet(append(newValSet.Validators, originalValset.Validators...)), - voterSet: NewVoterSet(append(newVoterSet.Voters, originalVoterSet.Voters...)), + //voterSet: WrapValidatorsToVoterSet(append(newValSet.Validators, originalValset.Validators...)), + voterSet: WrapValidatorsToVoterSet(append(newVoterSet.Voters, originalVoterSet.Voters...)), err: false, }, } diff --git a/types/voter_set.go b/types/voter_set.go index dcbd9f2a8..9e6b2ae54 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -29,11 +29,11 @@ type VoterSet struct { totalVotingPower int64 } -func NewVoterSet(valz []*Validator) *VoterSet { - sort.Sort(ValidatorsByAddress(valz)) - vals := &VoterSet{Voters: copyValidatorListForVoter(valz), totalVotingPower: 0} - vals.updateTotalVotingPower() - return vals +func WrapValidatorsToVoterSet(vals []*Validator) *VoterSet { + sort.Sort(ValidatorsByAddress(vals)) + voterSet := &VoterSet{Voters: vals, totalVotingPower: 0} + voterSet.updateTotalVotingPower() + return voterSet } // IsNilOrEmpty returns true if validator set is nil or empty. @@ -419,7 +419,7 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { // TODO: decide MinVoters, MinTotalVotingPowerPercent; make it to config if len(proofHash) == 0 || validators.Size() <= MinVoters { // height 1 has voter set that is same to validator set - return ToVoterAll(validators) + return ToVoterAll(validators.Validators) } seed := hashToSeed(proofHash) @@ -445,13 +445,13 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { for i, winner := range winners { voters[i] = winner.(*candidate).val } - return NewVoterSet(voters) + return WrapValidatorsToVoterSet(voters) } // This should be used in only test -func ToVoterAll(validators *ValidatorSet) *VoterSet { - newVoters := make([]*Validator, validators.Size()) - for i, val := range validators.Validators { +func ToVoterAll(validators []*Validator) *VoterSet { + newVoters := make([]*Validator, len(validators)) + for i, val := range validators { newVoters[i] = &Validator{ Address: val.Address, PubKey: val.PubKey, @@ -460,9 +460,8 @@ func ToVoterAll(validators *ValidatorSet) *VoterSet { ProposerPriority: val.ProposerPriority, } } - voters := NewVoterSet(newVoters) - voters.updateTotalVotingPower() - return voters + sort.Sort(ValidatorsByAddress(newVoters)) + return WrapValidatorsToVoterSet(newVoters) } func hashToSeed(hash []byte) uint64 { From 8eda4c12c11656a30bbb354febfa976bfbd07b49 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:40:23 +0900 Subject: [PATCH 14/38] fix: apply comment; rename validatorSet to voterSet and fix compile errors --- lite2/client.go | 12 ++++++------ lite2/client_test.go | 8 ++++---- lite2/verifier_test.go | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lite2/client.go b/lite2/client.go index ee33d4ddb..949e27663 100644 --- a/lite2/client.go +++ b/lite2/client.go @@ -376,7 +376,7 @@ func (c *Client) initializeWithTrustOptions(options TrustOptions) error { } // 2) Fetch and verify the vals. - voters, err := c.validatorSetFromPrimary(options.Height) + voters, err := c.voterSetFromPrimary(options.Height) if err != nil { return err } @@ -872,7 +872,7 @@ func (c *Client) fetchHeaderAndValsAtHeight(height int64) (*types.SignedHeader, if err != nil { return nil, nil, fmt.Errorf("failed to obtain the header #%d: %w", height, err) } - vals, err := c.validatorSetFromPrimary(height) + vals, err := c.voterSetFromPrimary(height) if err != nil { return nil, nil, fmt.Errorf("failed to obtain the vals #%d: %w", height, err) } @@ -1083,13 +1083,13 @@ func (c *Client) signedHeaderFromPrimary(height int64) (*types.SignedHeader, err // validatorSetFromPrimary retrieves the VoterSet from the primary provider // at the specified height. Handles dropout by the primary provider after 5 // attempts by replacing it with an alternative provider. -func (c *Client) validatorSetFromPrimary(height int64) (*types.VoterSet, error) { +func (c *Client) voterSetFromPrimary(height int64) (*types.VoterSet, error) { for attempt := uint16(1); attempt <= c.maxRetryAttempts; attempt++ { c.providerMutex.Lock() - vals, err := c.primary.VoterSet(height) + voters, err := c.primary.VoterSet(height) c.providerMutex.Unlock() if err == nil || err == provider.ErrValidatorSetNotFound { - return vals, err + return voters, err } c.logger.Error("Failed to get validator set from primary", "attempt", attempt, "err", err) time.Sleep(backoffTimeout(attempt)) @@ -1101,7 +1101,7 @@ func (c *Client) validatorSetFromPrimary(height int64) (*types.VoterSet, error) return nil, err } - return c.validatorSetFromPrimary(height) + return c.voterSetFromPrimary(height) } // exponential backoff (with jitter) diff --git a/lite2/client_test.go b/lite2/client_test.go index d260a9a28..9cb99eb74 100644 --- a/lite2/client_test.go +++ b/lite2/client_test.go @@ -24,7 +24,7 @@ const ( var ( keys = genPrivKeys(4) - vals = types.ToVoterAll(keys.ToValidators(20, 10)) + vals = types.ToVoterAll(keys.ToValidators(20, 10).Validators) bTime, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") h1 = keys.GenSignedHeader(chainID, 1, bTime, nil, vals, vals, []byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)) @@ -64,7 +64,7 @@ var ( func TestClient_SequentialVerification(t *testing.T) { newKeys := genPrivKeys(4) - newVals := types.ToVoterAll(newKeys.ToValidators(10, 1)) + newVals := types.ToVoterAll(newKeys.ToValidators(10, 1).Validators) testCases := []struct { name string @@ -178,11 +178,11 @@ func TestClient_SequentialVerification(t *testing.T) { func TestClient_SkippingVerification(t *testing.T) { // required for 2nd test case newKeys := genPrivKeys(4) - newVals := types.ToVoterAll(newKeys.ToValidators(10, 1)) + newVals := types.ToVoterAll(newKeys.ToValidators(10, 1).Validators) // 1/3+ of vals, 2/3- of newVals transitKeys := keys.Extend(3) - transitVals := types.ToVoterAll(transitKeys.ToValidators(10, 1)) + transitVals := types.ToVoterAll(transitKeys.ToValidators(10, 1).Validators) testCases := []struct { name string diff --git a/lite2/verifier_test.go b/lite2/verifier_test.go index 1bfac9a9f..398574f43 100644 --- a/lite2/verifier_test.go +++ b/lite2/verifier_test.go @@ -26,7 +26,7 @@ func TestVerifyAdjacentHeaders(t *testing.T) { var ( keys = genPrivKeys(4) // 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do! - vals = types.ToVoterAll(keys.ToValidators(20, 10)) + vals = types.ToVoterAll(keys.ToValidators(20, 10).Validators) bTime, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") header = keys.GenSignedHeader(chainID, lastHeight, bTime, nil, vals, vals, []byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)) @@ -124,9 +124,9 @@ func TestVerifyAdjacentHeaders(t *testing.T) { // vals does not match with what we have -> error 8: { keys.GenSignedHeader(chainID, nextHeight, bTime.Add(1*time.Hour), nil, - types.ToVoterAll(keys.ToValidators(10, 1)), vals, []byte("app_hash"), []byte("cons_hash"), + types.ToVoterAll(keys.ToValidators(10, 1).Validators), vals, []byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)), - types.ToVoterAll(keys.ToValidators(10, 1)), + types.ToVoterAll(keys.ToValidators(10, 1).Validators), 3 * time.Hour, bTime.Add(2 * time.Hour), nil, @@ -136,7 +136,7 @@ func TestVerifyAdjacentHeaders(t *testing.T) { 9: { keys.GenSignedHeader(chainID, nextHeight, bTime.Add(1*time.Hour), nil, vals, vals, []byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)), - types.ToVoterAll(keys.ToValidators(10, 1)), + types.ToVoterAll(keys.ToValidators(10, 1).Validators), 3 * time.Hour, bTime.Add(2 * time.Hour), nil, @@ -146,7 +146,7 @@ func TestVerifyAdjacentHeaders(t *testing.T) { 10: { keys.GenSignedHeader(chainID, nextHeight, bTime.Add(1*time.Hour), nil, vals, vals, []byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)), - types.ToVoterAll(keys.ToValidators(10, 1)), + types.ToVoterAll(keys.ToValidators(10, 1).Validators), 1 * time.Hour, bTime.Add(1 * time.Hour), nil, @@ -180,22 +180,22 @@ func TestVerifyNonAdjacentHeaders(t *testing.T) { var ( keys = genPrivKeys(4) // 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do! - vals = types.ToVoterAll(keys.ToValidators(20, 10)) + vals = types.ToVoterAll(keys.ToValidators(20, 10).Validators) bTime, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") header = keys.GenSignedHeader(chainID, lastHeight, bTime, nil, vals, vals, []byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)) // 30, 40, 50 twoThirds = keys[1:] - twoThirdsVals = types.ToVoterAll(twoThirds.ToValidators(30, 10)) + twoThirdsVals = types.ToVoterAll(twoThirds.ToValidators(30, 10).Validators) // 50 oneThird = keys[len(keys)-1:] - oneThirdVals = types.ToVoterAll(oneThird.ToValidators(50, 10)) + oneThirdVals = types.ToVoterAll(oneThird.ToValidators(50, 10).Validators) // 20 lessThanOneThird = keys[0:1] - lessThanOneThirdVals = types.ToVoterAll(lessThanOneThird.ToValidators(20, 10)) + lessThanOneThirdVals = types.ToVoterAll(lessThanOneThird.ToValidators(20, 10).Validators) ) testCases := []struct { @@ -296,7 +296,7 @@ func TestVerifyReturnsErrorIfTrustLevelIsInvalid(t *testing.T) { var ( keys = genPrivKeys(4) // 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do! - vals = types.ToVoterAll(keys.ToValidators(20, 10)) + vals = types.ToVoterAll(keys.ToValidators(20, 10).Validators) bTime, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") header = keys.GenSignedHeader(chainID, lastHeight, bTime, nil, vals, vals, []byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)) From b85eedb8d3c80aa5b98cb3f0f25e5bd31735fd47 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:43:04 +0900 Subject: [PATCH 15/38] fix: apply comment; use VotingPower on consensus --- consensus/state.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 3adf231be..11d9dd871 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1547,7 +1547,7 @@ func (cs *State) recordMetrics(height int64, block *types.Block) { commitSig := block.LastCommit.Signatures[i] if commitSig.Absent() { missingVoters++ - missingVotersPower += val.StakingPower + missingVotersPower += val.VotingPower } if cs.privValidator != nil { @@ -1561,7 +1561,7 @@ func (cs *State) recordMetrics(height int64, block *types.Block) { label := []string{ "validator_address", val.Address.String(), } - cs.metrics.VoterPower.With(label...).Set(float64(val.StakingPower)) + cs.metrics.VoterPower.With(label...).Set(float64(val.VotingPower)) if commitSig.ForBlock() { cs.metrics.VoterLastSignedHeight.With(label...).Set(float64(height)) } else { @@ -1578,7 +1578,7 @@ func (cs *State) recordMetrics(height int64, block *types.Block) { byzantineVotersPower := int64(0) for _, ev := range block.Evidence.Evidence { if _, val := cs.Voters.GetByAddress(ev.Address()); val != nil { - byzantineVotersPower += val.StakingPower + byzantineVotersPower += val.VotingPower } } cs.metrics.ByzantineVotersPower.Set(float64(byzantineVotersPower)) From fcd158b6f8765eff74d3073b33211d4193ddfee4 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:45:22 +0900 Subject: [PATCH 16/38] fix: lint error --- types/voter_set.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/voter_set.go b/types/voter_set.go index 9e6b2ae54..09f08c3a0 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -26,7 +26,7 @@ type VoterSet struct { Voters []*Validator `json:"voters"` // cached (unexported) - totalVotingPower int64 + totalVotingPower int64 } func WrapValidatorsToVoterSet(vals []*Validator) *VoterSet { @@ -96,8 +96,8 @@ func copyValidatorListForVoter(vals []*Validator) []*Validator { // VoterSet.Copy() copies validator list shallow func (voters *VoterSet) Copy() *VoterSet { return &VoterSet{ - Voters: copyValidatorListShallow(voters.Voters), - totalVotingPower: voters.totalVotingPower, + Voters: copyValidatorListShallow(voters.Voters), + totalVotingPower: voters.totalVotingPower, } } From af8ea828ed54fe4f2c552f880c9bc2212516257d Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:47:14 +0900 Subject: [PATCH 17/38] fix: lint error --- lite/dbprovider.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lite/dbprovider.go b/lite/dbprovider.go index eaf3ada4b..ca27f6016 100644 --- a/lite/dbprovider.go +++ b/lite/dbprovider.go @@ -171,7 +171,6 @@ func (dbp *DBProvider) getVoterSet(chainID string, height int64) (voterSet *type // equivalence using assert.Equal (tests for deep equality) in our tests, // which also tests for unexported/private field equivalence. voterSet.TotalVotingPower() - voterSet.TotalStakingPower() return } From aac3c8b76fc41cb5ae9c41b74cfa693df19282c1 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:52:04 +0900 Subject: [PATCH 18/38] fix: lite test compile error --- lite/base_verifier_test.go | 5 +++-- lite/dynamic_verifier_test.go | 16 ++++++++-------- lite/provider_test.go | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lite/base_verifier_test.go b/lite/base_verifier_test.go index 4a971dc2d..f52c3df7d 100644 --- a/lite/base_verifier_test.go +++ b/lite/base_verifier_test.go @@ -14,7 +14,7 @@ func TestBaseCert(t *testing.T) { keys := genPrivKeys(4) // 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do! - vals := types.ToVoterAll(keys.ToValidators(20, 10)) + vals := types.ToVoterAll(keys.ToValidators(20, 10).Validators) // and a Verifier based on our known set chainID := "test-static" cert := NewBaseVerifier(chainID, 2, vals) @@ -37,7 +37,8 @@ func TestBaseCert(t *testing.T) { {keys, vals, 4, 0, len(keys) - 1, false, false}, // Changing the power a little bit breaks the static validator. // The sigs are enough, but the validator hash is unknown. - {keys, types.ToVoterAll(keys.ToValidators(20, 11)), 5, 0, len(keys), false, true}, + {keys, types.ToVoterAll(keys.ToValidators(20, 11).Validators), + 5, 0, len(keys), false, true}, } for _, tc := range cases { diff --git a/lite/dynamic_verifier_test.go b/lite/dynamic_verifier_test.go index f85146437..abb7d3cc5 100644 --- a/lite/dynamic_verifier_test.go +++ b/lite/dynamic_verifier_test.go @@ -33,8 +33,8 @@ func TestInquirerValidPath(t *testing.T) { count := 50 fcz := make([]FullCommit, count) for i := 0; i < count; i++ { - vals := types.ToVoterAll(keys.ToValidators(vote, 0)) - nextVals := types.ToVoterAll(nkeys.ToValidators(vote, 0)) + vals := types.ToVoterAll(keys.ToValidators(vote, 0).Validators) + nextVals := types.ToVoterAll(nkeys.ToValidators(vote, 0).Validators) h := int64(1 + i) appHash := []byte(fmt.Sprintf("h=%d", h)) fcz[i] = keys.GenFullCommit( @@ -89,9 +89,9 @@ func TestDynamicVerify(t *testing.T) { chainID := "dynamic-verifier" power := int64(10) keys1 := genPrivKeys(5) - vals1 := types.ToVoterAll(keys1.ToValidators(power, 0)) + vals1 := types.ToVoterAll(keys1.ToValidators(power, 0).Validators) keys2 := genPrivKeys(5) - vals2 := types.ToVoterAll(keys2.ToValidators(power, 0)) + vals2 := types.ToVoterAll(keys2.ToValidators(power, 0).Validators) // make some commits with the first for i := 0; i < n1; i++ { @@ -154,8 +154,8 @@ func TestInquirerVerifyHistorical(t *testing.T) { consHash := []byte("special-params") fcz := make([]FullCommit, count) for i := 0; i < count; i++ { - vals := types.ToVoterAll(keys.ToValidators(vote, 0)) - nextVals := types.ToVoterAll(nkeys.ToValidators(vote, 0)) + vals := types.ToVoterAll(keys.ToValidators(vote, 0).Validators) + nextVals := types.ToVoterAll(nkeys.ToValidators(vote, 0).Validators) h := int64(1 + i) appHash := []byte(fmt.Sprintf("h=%d", h)) resHash := []byte(fmt.Sprintf("res=%d", h)) @@ -237,8 +237,8 @@ func TestConcurrencyInquirerVerify(t *testing.T) { consHash := []byte("special-params") fcz := make([]FullCommit, count) for i := 0; i < count; i++ { - vals := types.ToVoterAll(keys.ToValidators(vote, 0)) - nextVals := types.ToVoterAll(nkeys.ToValidators(vote, 0)) + vals := types.ToVoterAll(keys.ToValidators(vote, 0).Validators) + nextVals := types.ToVoterAll(nkeys.ToValidators(vote, 0).Validators) h := int64(1 + i) appHash := []byte(fmt.Sprintf("h=%d", h)) resHash := []byte(fmt.Sprintf("res=%d", h)) diff --git a/lite/provider_test.go b/lite/provider_test.go index c147e933a..31d5fe199 100644 --- a/lite/provider_test.go +++ b/lite/provider_test.go @@ -55,7 +55,7 @@ func checkProvider(t *testing.T, p PersistentProvider, chainID, app string) { // Make a bunch of full commits. fcz := make([]FullCommit, count) for i := 0; i < count; i++ { - vals := types.ToVoterAll(keys.ToValidators(10, int64(count/2))) + vals := types.ToVoterAll(keys.ToValidators(10, int64(count/2)).Validators) h := int64(20 + 10*i) fcz[i] = keys.GenFullCommit(chainID, h, nil, vals, vals, appHash, []byte("params"), []byte("results"), 0, 5) } @@ -119,7 +119,7 @@ func TestMultiLatestFullCommit(t *testing.T) { // Set a bunch of full commits. for i := 0; i < count; i++ { - vals := types.ToVoterAll(keys.ToValidators(10, int64(count/2))) + vals := types.ToVoterAll(keys.ToValidators(10, int64(count/2)).Validators) h := int64(10 * (i + 1)) fc := keys.GenFullCommit(chainID, h, nil, vals, vals, appHash, []byte("params"), []byte("results"), 0, 5) err := p2.SaveFullCommit(fc) From 9e8dc55d211d16c7483ddb8a7fc4e0c0e7eadc1a Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Wed, 3 Jun 2020 18:55:01 +0900 Subject: [PATCH 19/38] fix: remove unused function --- types/voter_set.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/types/voter_set.go b/types/voter_set.go index 09f08c3a0..bacdb703d 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -84,15 +84,6 @@ func copyValidatorListShallow(vals []*Validator) []*Validator { return result } -func copyValidatorListForVoter(vals []*Validator) []*Validator { - result := make([]*Validator, len(vals)) - for i, v := range vals { - result[i] = v.Copy() - result[i].VotingPower = v.StakingPower - } - return result -} - // VoterSet.Copy() copies validator list shallow func (voters *VoterSet) Copy() *VoterSet { return &VoterSet{ From d5cf9b815416bf8aab8b8b7fcb29c96bff011bed Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Thu, 4 Jun 2020 15:31:28 +0900 Subject: [PATCH 20/38] fix: modify validator to voter in comments --- lite2/client.go | 70 ++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/lite2/client.go b/lite2/client.go index 949e27663..29c380eb6 100644 --- a/lite2/client.go +++ b/lite2/client.go @@ -50,11 +50,11 @@ func SequentialVerification() Option { } // SkippingVerification option configures the light client to skip headers as -// long as {trustLevel} of the old validator set signed the new header. The +// long as {trustLevel} of the old voter set signed the new header. The // bisection algorithm from the specification is used for finding the minimal // "trust path". // -// trustLevel - fraction of the old validator set (in terms of voting power), +// trustLevel - fraction of the old voter set (in terms of voting power), // which must sign the new header in order for us to trust it. NOTE this only // applies to non-adjacent headers. For adjacent headers, sequential // verification is used. @@ -65,9 +65,9 @@ func SkippingVerification(trustLevel tmmath.Fraction) Option { } } -// PruningSize option sets the maximum amount of headers & validator set pairs +// PruningSize option sets the maximum amount of headers & voter set pairs // that the light client stores. When Prune() is run, all headers (along with -// the associated validator sets) that are earlier than the h amount of headers +// the associated voter sets) that are earlier than the h amount of headers // will be removed from the store. Default: 1000. A pruning size of 0 will not // prune the lite client at all. func PruningSize(h uint16) Option { @@ -132,8 +132,8 @@ type Client struct { trustedStore store.Store // Highest trusted header from the store (height=H). latestTrustedHeader *types.SignedHeader - // Highest validator set from the store (height=H). - latestTrustedVals *types.VoterSet + // Highest voter set from the store (height=H). + latestTrustedVoters *types.VoterSet // See RemoveNoLongerTrustedHeadersPeriod option pruningSize uint16 @@ -262,13 +262,13 @@ func (c *Client) restoreTrustedHeaderAndVals() error { trustedVals, err := c.trustedStore.VoterSet(lastHeight) if err != nil { - return fmt.Errorf("can't get last trusted validators: %w", err) + return fmt.Errorf("can't get last trusted voters: %w", err) } c.latestTrustedHeader = trustedHeader - c.latestTrustedVals = trustedVals + c.latestTrustedVoters = trustedVals - c.logger.Info("Restored trusted header and vals", "height", lastHeight) + c.logger.Info("Restored trusted header and voters", "height", lastHeight) } return nil @@ -375,20 +375,20 @@ func (c *Client) initializeWithTrustOptions(options TrustOptions) error { return err } - // 2) Fetch and verify the vals. + // 2) Fetch and verify the voters. voters, err := c.voterSetFromPrimary(options.Height) if err != nil { return err } if !bytes.Equal(h.VotersHash, voters.Hash()) { - return fmt.Errorf("expected header's validators (%X) to match those that were supplied (%X)", + return fmt.Errorf("expected header's voters (%X) to match those that were supplied (%X)", h.VotersHash, voters.Hash(), ) } - // Ensure that +2/3 of validators signed correctly. + // Ensure that +2/3 of voters signed correctly. err = voters.VerifyCommit(c.chainID, h.Commit.BlockID, h.Height, h.Commit) if err != nil { return fmt.Errorf("invalid commit: %w", err) @@ -400,7 +400,7 @@ func (c *Client) initializeWithTrustOptions(options TrustOptions) error { // TrustedHeader returns a trusted header at the given height (0 - the latest). // -// Headers along with validator sets, which can't be trusted anymore, are +// Headers along with voter sets, which can't be trusted anymore, are // removed once a day (can be changed with RemoveNoLongerTrustedHeadersPeriod // option). // . @@ -421,13 +421,13 @@ func (c *Client) TrustedHeader(height int64) (*types.SignedHeader, error) { return c.trustedStore.SignedHeader(height) } -// TrustedVoterSet returns a trusted validator set at the given height (0 - +// TrustedVoterSet returns a trusted voter set at the given height (0 - // latest). The second return parameter is the height used (useful if 0 was // passed; otherwise can be ignored). // // height must be >= 0. // -// Headers along with validator sets are +// Headers along with voter sets are // removed once a day (can be changed with RemoveNoLongerTrustedHeadersPeriod // option). // @@ -435,7 +435,7 @@ func (c *Client) TrustedHeader(height int64) (*types.SignedHeader, error) { // - there are some issues with the trusted store, although that should not // happen normally; // - negative height is passed; -// - header signed by that validator set has not been verified yet +// - header signed by that voter set has not been verified yet // // Safe for concurrent use by multiple goroutines. func (c *Client) TrustedVoterSet(height int64) (valSet *types.VoterSet, heightUsed int64, err error) { @@ -471,7 +471,7 @@ func (c *Client) compareWithLatestHeight(height int64) (int64, error) { return height, nil } -// VerifyHeaderAtHeight fetches header and validators at the given height +// VerifyHeaderAtHeight fetches header and voters at the given height // and calls VerifyHeader. It returns header immediately if such exists in // trustedStore (no verification is needed). // @@ -505,12 +505,12 @@ func (c *Client) VerifyHeaderAtHeight(height int64, now time.Time) (*types.Signe // immediately if newHeader exists in trustedStore (no verification is // needed). Else it performs one of the two types of verification: // -// SequentialVerification: verifies that 2/3 of the trusted validator set has +// SequentialVerification: verifies that 2/3 of the trusted voter set has // signed the new header. If the headers are not adjacent, **all** intermediate // headers will be requested. Intermediate headers are not saved to database. // // SkippingVerification(trustLevel): verifies that {trustLevel} of the trusted -// validator set has signed the new header. If it's not the case and the +// voter set has signed the new header. If it's not the case and the // headers are not adjacent, bisection is performed and necessary (not all) // intermediate headers will be requested. See the specification for details. // Intermediate headers are not saved to database. @@ -561,7 +561,7 @@ func (c *Client) verifyHeader(newHeader *types.SignedHeader, newVals *types.Vote case sequential: err = c.sequence(c.latestTrustedHeader, newHeader, newVals, now) case skipping: - err = c.bisection(c.latestTrustedHeader, c.latestTrustedVals, newHeader, newVals, now) + err = c.bisection(c.latestTrustedHeader, c.latestTrustedVoters, newHeader, newVals, now) default: panic(fmt.Sprintf("Unknown verification mode: %b", c.verificationMode)) } @@ -598,7 +598,7 @@ func (c *Client) verifyHeader(newHeader *types.SignedHeader, newVals *types.Vote } else { closestVotorSet, _, err = c.TrustedVoterSet(closestHeader.Height) if err != nil { - return fmt.Errorf("can't get validator set at height %d: %w", closestHeader.Height, err) + return fmt.Errorf("can't get voter set at height %d: %w", closestHeader.Height, err) } err = c.bisection(closestHeader, closestVotorSet, newHeader, newVals, now) } @@ -801,16 +801,16 @@ func (c *Client) Witnesses() []provider.Provider { return c.witnesses } -// Cleanup removes all the data (headers and validator sets) stored. Note: the +// Cleanup removes all the data (headers and voter sets) stored. Note: the // client must be stopped at this point. func (c *Client) Cleanup() error { c.logger.Info("Removing all the data") c.latestTrustedHeader = nil - c.latestTrustedVals = nil + c.latestTrustedVoters = nil return c.trustedStore.Prune(0) } -// cleanupAfter deletes all headers & validator sets after +height+. It also +// cleanupAfter deletes all headers & voter sets after +height+. It also // resets latestTrustedHeader to the latest header. func (c *Client) cleanupAfter(height int64) error { prevHeight := c.latestTrustedHeader.Height @@ -825,7 +825,7 @@ func (c *Client) cleanupAfter(height int64) error { err = c.trustedStore.DeleteSignedHeaderAndValidatorSet(h.Height) if err != nil { - c.logger.Error("can't remove a trusted header & validator set", "err", err, + c.logger.Error("can't remove a trusted header & voter set", "err", err, "height", h.Height) } @@ -833,7 +833,7 @@ func (c *Client) cleanupAfter(height int64) error { } c.latestTrustedHeader = nil - c.latestTrustedVals = nil + c.latestTrustedVoters = nil err := c.restoreTrustedHeaderAndVals() if err != nil { return err @@ -842,12 +842,12 @@ func (c *Client) cleanupAfter(height int64) error { return nil } -func (c *Client) updateTrustedHeaderAndVals(h *types.SignedHeader, vals *types.VoterSet) error { - if !bytes.Equal(h.VotersHash, vals.Hash()) { - return fmt.Errorf("expected validator's hash %X, but got %X", h.VotersHash, vals.Hash()) +func (c *Client) updateTrustedHeaderAndVals(h *types.SignedHeader, voters *types.VoterSet) error { + if !bytes.Equal(h.VotersHash, voters.Hash()) { + return fmt.Errorf("expected voter's hash %X, but got %X", h.VotersHash, voters.Hash()) } - if err := c.trustedStore.SaveSignedHeaderAndValidatorSet(h, vals); err != nil { + if err := c.trustedStore.SaveSignedHeaderAndValidatorSet(h, voters); err != nil { return fmt.Errorf("failed to save trusted header: %w", err) } @@ -859,13 +859,13 @@ func (c *Client) updateTrustedHeaderAndVals(h *types.SignedHeader, vals *types.V if c.latestTrustedHeader == nil || h.Height > c.latestTrustedHeader.Height { c.latestTrustedHeader = h - c.latestTrustedVals = vals + c.latestTrustedVoters = voters } return nil } -// fetch header and validators for the given height (0 - latest) from primary +// fetch header and voters for the given height (0 - latest) from primary // provider. func (c *Client) fetchHeaderAndValsAtHeight(height int64) (*types.SignedHeader, *types.VoterSet, error) { h, err := c.signedHeaderFromPrimary(height) @@ -956,7 +956,7 @@ func (c *Client) compareNewHeaderWithWitnesses(h *types.SignedHeader) error { } if !bytes.Equal(h.Hash(), altH.Hash()) { - if err = c.latestTrustedVals.VerifyCommitTrusting(c.chainID, altH.Commit.BlockID, + if err = c.latestTrustedVoters.VerifyCommitTrusting(c.chainID, altH.Commit.BlockID, altH.Height, altH.Commit, c.trustLevel); err != nil { c.logger.Error("Witness sent us incorrect header", "err", err, "witness", witness) witnessesToRemove = append(witnessesToRemove, i) @@ -1080,7 +1080,7 @@ func (c *Client) signedHeaderFromPrimary(height int64) (*types.SignedHeader, err return c.signedHeaderFromPrimary(height) } -// validatorSetFromPrimary retrieves the VoterSet from the primary provider +// voterSetFromPrimary retrieves the VoterSet from the primary provider // at the specified height. Handles dropout by the primary provider after 5 // attempts by replacing it with an alternative provider. func (c *Client) voterSetFromPrimary(height int64) (*types.VoterSet, error) { @@ -1091,7 +1091,7 @@ func (c *Client) voterSetFromPrimary(height int64) (*types.VoterSet, error) { if err == nil || err == provider.ErrValidatorSetNotFound { return voters, err } - c.logger.Error("Failed to get validator set from primary", "attempt", attempt, "err", err) + c.logger.Error("Failed to get voter set from primary", "attempt", attempt, "err", err) time.Sleep(backoffTimeout(attempt)) } From cbe2847bc71e15c657e253933f94216a1a37520d Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Fri, 5 Jun 2020 09:46:07 +0900 Subject: [PATCH 21/38] fix: total voting power overflow --- types/voter_set.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/types/voter_set.go b/types/voter_set.go index bacdb703d..2e7f56550 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -92,27 +92,34 @@ func (voters *VoterSet) Copy() *VoterSet { } } +func (voters *VoterSet) downscaleVotingPower() { + for _, v := range voters.Voters { + v.VotingPower = int64(float64(v.VotingPower) / 10) + } +} + // Forces recalculation of the set's total voting power. // Panics if total voting power is bigger than MaxTotalStakingPower. func (voters *VoterSet) updateTotalVotingPower() { sum := int64(0) - for _, val := range voters.Voters { - // mind overflow - sum = safeAddClip(sum, val.VotingPower) - if sum > MaxTotalStakingPower { - panic(fmt.Sprintf( - "Total voting power should be guarded to not exceed %v; got: %v", - MaxTotalStakingPower, - sum)) + for needSum := true; needSum; { + needSum = false + for _, val := range voters.Voters { + // mind overflow + sum = safeAddClip(sum, val.VotingPower) + if sum > MaxTotalStakingPower { + needSum = true + voters.downscaleVotingPower() + break + } } } - voters.totalVotingPower = sum } func (voters *VoterSet) TotalVotingPower() int64 { - if voters.totalVotingPower == 0 { - voters.updateTotalVotingPower() + if voters.totalVotingPower == 0 && voters.Size() > 0 { + panic("VoterSet total voting power cannot be 0") } return voters.totalVotingPower } From 45e573ebc965c83d4c3a4e09675c9d119f146376 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Fri, 5 Jun 2020 10:02:55 +0900 Subject: [PATCH 22/38] fix: update total voting power if 0 --- types/voter_set.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/voter_set.go b/types/voter_set.go index 2e7f56550..77ca63114 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -118,8 +118,8 @@ func (voters *VoterSet) updateTotalVotingPower() { } func (voters *VoterSet) TotalVotingPower() int64 { - if voters.totalVotingPower == 0 && voters.Size() > 0 { - panic("VoterSet total voting power cannot be 0") + if voters.totalVotingPower == 0 { + voters.updateTotalVotingPower() } return voters.totalVotingPower } From cb7f0456fde2497e89ea63e77d2bef5d5212323e Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 8 Jun 2020 10:18:28 +0900 Subject: [PATCH 23/38] docs: change log --- CHANGELOG_PENDING.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index b5247514f..5a8d4e220 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -12,7 +12,12 @@ - Go API + - [types] [\#83](https://github.com/line/tendermint/pull/83) Add `StakingPower` to `Validator` + - [consensus] [\#83](https://github.com/line/tendermint/pull/83) Change calculation of `VotingPower` + ### FEATURES: +- [rpc] [\#78](https://github.com/line/tendermint/pull/78) Add `Voters` rpc +- [consensus] [\#83](https://github.com/line/tendermint/pull/83) Selection voters using random sampling without replacement ### IMPROVEMENTS: From a3835cbd44551c57040e4c2b4a94c3717772627c Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Thu, 11 Jun 2020 09:00:02 +0900 Subject: [PATCH 24/38] fix: apply comments --- libs/rand/sampling.go | 13 +++++++++---- libs/rand/sampling_test.go | 19 +++++++++++++++++-- types/validator_set_test.go | 6 +++--- types/voter_set.go | 4 ++++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index d10c7be6d..1e8aa975a 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -108,6 +108,11 @@ func RandomSamplingWithoutReplacement( losersPriorities := make([]uint64, len(candidates)) winnerNum := 0 for winnerNum < minSamplingCount || winnersPriority < priorityThreshold { + if totalPriority-winnersPriority == 0 { + // it's possible if some candidates have zero priority + // if then, we can't elect voter any more; we should holt electing not to fall in infinity loop + break + } threshold := randomThreshold(&seed, totalPriority-winnersPriority) cumulativePriority := uint64(0) found := false @@ -143,12 +148,12 @@ func RandomSamplingWithoutReplacement( func sumTotalPriority(candidates []Candidate) (sum uint64) { for _, candi := range candidates { - if candi.Priority() == 0 { - panic("candidate(%d) priority must not be 0") - } sum += candi.Priority() } - return sum + if sum == 0 { + panic("all candidates have zero priority") + } + return } // SplitMix64 diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 83aa3bf8e..803b071dc 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -158,6 +158,23 @@ func TestRandomSamplingWithoutReplacementDeterministic(t *testing.T) { } } +func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) { + // first candidate's priority is 0 + candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i) }) + winners1 := RandomSamplingWithoutReplacement(0, candidates1, 100, 100, 1000) + assert.True(t, len(winners1) == 99) + + candidates2 := newCandidates(100, func(i int) uint64 { + if i < 10 { + return 0 + } else { + return uint64(i) + } + }) + winners2 := RandomSamplingWithoutReplacement(0, candidates2, 95, 95, 1000) + assert.True(t, len(winners2) == 90) +} + func accumulateAndResetReward(candidate []Candidate, acc []uint64) { for i, c := range candidate { acc[i] += c.(*Element).winPoint @@ -211,8 +228,6 @@ func TestRandomSamplingWithoutReplacementPanic(t *testing.T) { {newCandidates(9, func(i int) uint64 { return 10 }), 10, 50}, // priorityRateThreshold is greater than 1 {newCandidates(10, func(i int) uint64 { return 10 }), 10, 101}, - // a candidate priority is 0 - {newCandidates(10, func(i int) uint64 { return uint64(i) }), 10, 99}, } for i, c := range cases { diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 33827d9c3..24343f62a 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -204,17 +204,17 @@ func TestProposerSelection2(t *testing.T) { t.Fatalf("(%d): Expected %d. Got %d", i, expected[i], bytesToInt(prop.Address)) } } - verifyWinningRate(t, vals, 10000, 0.02) + verifyWinningRate(t, vals, 50000, 0.01) // One validator has more than the others *val2 = *newValidator(addr2, 400) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 10000, 0.02) + verifyWinningRate(t, vals, 50000, 0.01) // One validator has more than the others *val2 = *newValidator(addr2, 401) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 100000, 0.02) + verifyWinningRate(t, vals, 100000, 0.01) // each validator should be the proposer a proportional number of times val0, val1, val2 = newValidator(addr0, 4), newValidator(addr1, 5), newValidator(addr2, 3) diff --git a/types/voter_set.go b/types/voter_set.go index 77ca63114..a81ec1a0a 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -450,6 +450,10 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { func ToVoterAll(validators []*Validator) *VoterSet { newVoters := make([]*Validator, len(validators)) for i, val := range validators { + if val.StakingPower == 0 { + // remove the validator with the staking power of 0 from the voter set + continue + } newVoters[i] = &Validator{ Address: val.Address, PubKey: val.PubKey, From b1f68f5b710624675028a30964101ff36031d2ae Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Thu, 11 Jun 2020 09:16:10 +0900 Subject: [PATCH 25/38] fix: lint error --- libs/rand/sampling_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 803b071dc..7cc5b5e50 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -167,9 +167,8 @@ func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) candidates2 := newCandidates(100, func(i int) uint64 { if i < 10 { return 0 - } else { - return uint64(i) } + return uint64(i) }) winners2 := RandomSamplingWithoutReplacement(0, candidates2, 95, 95, 1000) assert.True(t, len(winners2) == 90) From 083f6379a8910126ee13c4572f421efd02b7fc64 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Thu, 11 Jun 2020 18:31:35 +0900 Subject: [PATCH 26/38] fix: rewrite randomThreshold; remove priorityRateThreshold; some test cases --- consensus/common_test.go | 2 +- libs/rand/sampling.go | 70 ++++++++++++++++++++++++------------- libs/rand/sampling_test.go | 64 +++++++++++---------------------- types/validator_set_test.go | 45 +++++++++++++----------- types/voter_set.go | 36 +++++++++---------- types/voter_set_test.go | 32 +++++++++++++++-- 6 files changed, 141 insertions(+), 108 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index 950237164..f32ba1032 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -412,7 +412,7 @@ func loadPrivValidator(config *cfg.Config) *privval.FilePV { func randState(nValidators int) (*State, []*validatorStub) { // Get State state, privVals := randGenesisState(nValidators, false, 10) - state.LastProofHash = []byte{3} + state.LastProofHash = []byte{2} vss := make([]*validatorStub, nValidators) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 1e8aa975a..73c46bbc9 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -2,6 +2,8 @@ package rand import ( "fmt" + "math" + "math/big" s "sort" ) @@ -9,14 +11,9 @@ import ( type Candidate interface { Priority() uint64 LessThan(other Candidate) bool - SetWinPoint(winPoint uint64) + SetWinPoint(winPoint int64) } -const ( - // Casting a larger number than this as float64 can result in that lower bytes can be truncated - MaxFloat64Significant = uint64(0x1FFFFFFFFFFFFF) -) - // Select a specified number of candidates randomly from the candidate set based on each priority. This function is // deterministic and will produce the same result for the same input. // @@ -75,16 +72,31 @@ func moveWinnerToLast(candidates []Candidate, winner int) { candidates[len(candidates)-1] = winnerCandidate } +const uint64Mask = uint64(0x7FFFFFFFFFFFFFFF) + +var divider *big.Int + +func init() { + divider = big.NewInt(int64(uint64Mask)) + divider.Add(divider, big.NewInt(1)) +} + func randomThreshold(seed *uint64, total uint64) uint64 { - return uint64(float64(nextRandom(seed)&MaxFloat64Significant) / - float64(MaxFloat64Significant+1) * float64(total)) + if int64(total) < 0 { + panic(fmt.Sprintf("total priority is overflow: %d", total)) + } + totalBig := big.NewInt(int64(total)) + a := big.NewInt(int64(nextRandom(seed) & uint64Mask)) + a.Mul(a, totalBig) + a.Div(a, divider) + return a.Uint64() } // `RandomSamplingWithoutReplacement` elects winners among candidates without replacement // so it updates rewards of winners. This function continues to elect winners until the both of two // conditions(minSamplingCount, minPriorityPercent) are met. func RandomSamplingWithoutReplacement( - seed uint64, candidates []Candidate, minSamplingCount int, minPriorityPercent uint, winPointUnit uint64) ( + seed uint64, candidates []Candidate, minSamplingCount int, winPointUnit uint64) ( winners []Candidate) { if len(candidates) < minSamplingCount { @@ -92,22 +104,12 @@ func RandomSamplingWithoutReplacement( len(candidates), minSamplingCount)) } - if minPriorityPercent > 100 { - panic(fmt.Sprintf("minPriorityPercent must be equal or less than 100: %d", minPriorityPercent)) - } - totalPriority := sumTotalPriority(candidates) - if totalPriority > MaxFloat64Significant { - // totalPriority will be casting to float64, so it must be less than 0x1FFFFFFFFFFFFF(53bits) - panic(fmt.Sprintf("total priority cannot exceed %d: %d", MaxFloat64Significant, totalPriority)) - } - - priorityThreshold := totalPriority * uint64(minPriorityPercent) / 100 candidates = sort(candidates) winnersPriority := uint64(0) losersPriorities := make([]uint64, len(candidates)) winnerNum := 0 - for winnerNum < minSamplingCount || winnersPriority < priorityThreshold { + for winnerNum < minSamplingCount { if totalPriority-winnersPriority == 0 { // it's possible if some candidates have zero priority // if then, we can't elect voter any more; we should holt electing not to fall in infinity loop @@ -130,8 +132,8 @@ func RandomSamplingWithoutReplacement( if !found { panic(fmt.Sprintf("Cannot find random sample. winnerNum=%d, minSamplingCount=%d, "+ - "winnersPriority=%d, priorityThreshold=%d, totalPriority=%d, threshold=%d", - winnerNum, minSamplingCount, winnersPriority, priorityThreshold, totalPriority, threshold)) + "winnersPriority=%d, totalPriority=%d, threshold=%d", + winnerNum, minSamplingCount, winnersPriority, totalPriority, threshold)) } } compensationProportions := make([]float64, winnerNum) @@ -139,9 +141,29 @@ func RandomSamplingWithoutReplacement( compensationProportions[i] = compensationProportions[i+1] + 1/float64(losersPriorities[i]) } winners = candidates[len(candidates)-winnerNum:] + winPoints := make([]float64, len(winners)) + downscaleNeeded := false for i, winner := range winners { - winner.SetWinPoint(winPointUnit + - uint64(float64(winner.Priority())*compensationProportions[i]*float64(winPointUnit))) + winPoints[i] = float64(winPointUnit) + + float64(winner.Priority())*compensationProportions[i]*float64(winPointUnit) + if int64(winPoints[i]) < 0 { + downscaleNeeded = true + } + } + for downscaleNeeded { + downscaleNeeded = false + for i := range winPoints { + winPoints[i] = winPoints[i] / 10 + if int64(winPoints[i]) < 0 { + downscaleNeeded = true + } + } + } + for i, winner := range winners { + if winPoints[i] > math.MaxInt64 || winPoints[i] < 0 { + panic(fmt.Sprintf("winPoint is invalid: %f", winPoints[i])) + } + winner.SetWinPoint(int64(winPoints[i])) } return winners } diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 7cc5b5e50..750857026 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -11,7 +11,7 @@ import ( type Element struct { id uint32 - winPoint uint64 + winPoint int64 weight uint64 } @@ -27,7 +27,7 @@ func (e *Element) LessThan(other Candidate) bool { return e.id < o.id } -func (e *Element) SetWinPoint(winPoint uint64) { +func (e *Element) SetWinPoint(winPoint int64) { e.winPoint += winPoint } @@ -100,25 +100,17 @@ func resetWinPoint(candidate []Candidate) { func TestRandomSamplingWithoutReplacement1Candidate(t *testing.T) { candidates := newCandidates(1, func(i int) uint64 { return uint64(1000 * (i + 1)) }) - winners := RandomSamplingWithoutReplacement(0, candidates, 1, 100, 1000) + winners := RandomSamplingWithoutReplacement(0, candidates, 1, 1000) assert.True(t, len(winners) == 1) assert.True(t, candidates[0] == winners[0]) assert.True(t, winners[0].(*Element).winPoint == 1000) resetWinPoint(candidates) - winners2 := RandomSamplingWithoutReplacement(0, candidates, 1, 50, 1000) - assert.True(t, len(winners2) == 1) - assert.True(t, candidates[0] == winners2[0]) - assert.True(t, winners2[0].(*Element).winPoint == 1000) + winners2 := RandomSamplingWithoutReplacement(0, candidates, 0, 1000) + assert.True(t, len(winners2) == 0) resetWinPoint(candidates) - winners3 := RandomSamplingWithoutReplacement(0, candidates, 0, 50, 1000) - assert.True(t, len(winners3) == 1) - assert.True(t, candidates[0] == winners3[0]) - assert.True(t, winners3[0].(*Element).winPoint == 1000) - resetWinPoint(candidates) - - winners4 := RandomSamplingWithoutReplacement(0, candidates, 0, 0, 1000) + winners4 := RandomSamplingWithoutReplacement(0, candidates, 0, 1000) assert.True(t, len(winners4) == 0) resetWinPoint(candidates) } @@ -128,30 +120,19 @@ func TestRandomSamplingWithoutReplacementSamplingThreshold(t *testing.T) { candidates := newCandidates(100, func(i int) uint64 { return uint64(1000 * (i + 1)) }) for i := 1; i <= 100; i++ { - winners := RandomSamplingWithoutReplacement(0, candidates, i, 1, 1000) + winners := RandomSamplingWithoutReplacement(0, candidates, i, 1000) assert.True(t, len(winners) == i) resetWinPoint(candidates) } } -// test priorityRateThrshold -func TestRandomSamplingWithoutReplacementPriorityRateThrshold(t *testing.T) { - candidates := newCandidates(100, func(i int) uint64 { return uint64(1000) }) - - for i := 7; i <= 100; i++ { - winners := RandomSamplingWithoutReplacement(0, candidates, 1, uint(i), 1000) - assert.True(t, sumTotalPriority(winners) >= uint64(0.01*float64(i)*float64(sumTotalPriority(candidates)))) - resetWinPoint(candidates) - } -} - // test random election should be deterministic func TestRandomSamplingWithoutReplacementDeterministic(t *testing.T) { candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) candidates2 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) for i := 1; i <= 100; i++ { - winners1 := RandomSamplingWithoutReplacement(uint64(i), candidates1, 50, 50, 1000) - winners2 := RandomSamplingWithoutReplacement(uint64(i), candidates2, 50, 50, 1000) + winners1 := RandomSamplingWithoutReplacement(uint64(i), candidates1, 50, 1000) + winners2 := RandomSamplingWithoutReplacement(uint64(i), candidates2, 50, 1000) sameCandidates(winners1, winners2) resetWinPoint(candidates1) resetWinPoint(candidates2) @@ -161,7 +142,7 @@ func TestRandomSamplingWithoutReplacementDeterministic(t *testing.T) { func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) { // first candidate's priority is 0 candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i) }) - winners1 := RandomSamplingWithoutReplacement(0, candidates1, 100, 100, 1000) + winners1 := RandomSamplingWithoutReplacement(0, candidates1, 100, 1000) assert.True(t, len(winners1) == 99) candidates2 := newCandidates(100, func(i int) uint64 { @@ -170,13 +151,13 @@ func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) } return uint64(i) }) - winners2 := RandomSamplingWithoutReplacement(0, candidates2, 95, 95, 1000) + winners2 := RandomSamplingWithoutReplacement(0, candidates2, 95, 1000) assert.True(t, len(winners2) == 90) } func accumulateAndResetReward(candidate []Candidate, acc []uint64) { for i, c := range candidate { - acc[i] += c.(*Element).winPoint + acc[i] += uint64(c.(*Element).winPoint) c.(*Element).winPoint = 0 } } @@ -187,9 +168,9 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards := make([]uint64, 100) for i := 0; i < 100000; i++ { - // 23 samplingThreshold is minimum to pass this test - // If samplingThreshold is less than 23, the result says the reward is not fair - RandomSamplingWithoutReplacement(uint64(i), candidates, 23, 23, 10) + // 24 samplingThreshold is minimum to pass this test + // If samplingThreshold is less than 24, the result says the reward is not fair + RandomSamplingWithoutReplacement(uint64(i), candidates, 24, 10) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -198,7 +179,7 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards = make([]uint64, 100) for i := 0; i < 50000; i++ { - RandomSamplingWithoutReplacement(uint64(i), candidates, 50, 50, 10) + RandomSamplingWithoutReplacement(uint64(i), candidates, 50, 10) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -207,7 +188,7 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards = make([]uint64, 100) for i := 0; i < 10000; i++ { - RandomSamplingWithoutReplacement(uint64(i), candidates, 100, 100, 10) + RandomSamplingWithoutReplacement(uint64(i), candidates, 100, 10) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -217,16 +198,13 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { func TestRandomSamplingWithoutReplacementPanic(t *testing.T) { type Case struct { - Candidates []Candidate - SamplingThreshold int - PriorityRateThreshold uint + Candidates []Candidate + SamplingThreshold int } cases := [...]*Case{ // samplingThreshold is greater than the number of candidates - {newCandidates(9, func(i int) uint64 { return 10 }), 10, 50}, - // priorityRateThreshold is greater than 1 - {newCandidates(10, func(i int) uint64 { return 10 }), 10, 101}, + {newCandidates(9, func(i int) uint64 { return 10 }), 10}, } for i, c := range cases { @@ -236,7 +214,7 @@ func TestRandomSamplingWithoutReplacementPanic(t *testing.T) { t.Errorf("expected panic didn't happen in case %d", i+1) } }() - RandomSamplingWithoutReplacement(0, c.Candidates, c.SamplingThreshold, c.PriorityRateThreshold, 1000) + RandomSamplingWithoutReplacement(0, c.Candidates, c.SamplingThreshold, 1000) }() } } diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 24343f62a..3318f3388 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -179,10 +179,11 @@ func TestProposerSelection1(t *testing.T) { val := vset.SelectProposer([]byte{}, int64(i), 0) proposers = append(proposers, string(val.Address)) } - expected := `foo foo foo baz bar foo baz baz foo foo baz bar foo foo foo foo bar foo foo foo bar foo foo foo bar ` + - `baz foo foo bar baz foo bar baz bar foo foo foo bar foo foo bar bar foo foo bar foo baz bar foo foo baz foo ` + - `baz foo baz foo foo bar foo bar baz foo foo baz foo foo foo foo bar baz foo baz foo baz foo foo bar baz foo ` + - `baz foo foo baz foo baz foo baz foo baz foo foo foo foo foo bar bar foo bar foo` + expected := `foo foo foo foo bar bar foo bar foo baz bar foo baz baz baz foo foo bar foo bar baz bar foo baz foo ` + + `foo baz foo foo baz foo foo baz bar foo foo foo baz foo baz baz bar foo foo foo foo baz bar bar bar bar foo ` + + `foo foo baz foo foo foo foo foo foo baz foo foo baz bar bar foo bar foo foo baz bar foo foo baz foo foo baz ` + + `foo foo bar foo foo baz foo foo foo bar foo foo baz baz foo foo bar baz foo baz` + if expected != strings.Join(proposers, " ") { t.Errorf("expected sequence of proposers was\n%v\nbut got \n%v", expected, strings.Join(proposers, " ")) } @@ -197,24 +198,24 @@ func TestProposerSelection2(t *testing.T) { val0, val1, val2 := newValidator(addr0, 100), newValidator(addr1, 100), newValidator(addr2, 100) valList := []*Validator{val0, val1, val2} vals := NewValidatorSet(valList) - expected := []int{1, 1, 0, 1, 2, 0, 2, 2, 0, 0, 1, 2, 0, 1, 1} + expected := []int{0, 1, 0, 0, 2, 2, 0, 2, 1, 2, 2, 1, 2, 2, 2} for i := 0; i < len(valList)*5; i++ { prop := vals.SelectProposer([]byte{}, int64(i), 0) if bytesToInt(prop.Address) != expected[i] { t.Fatalf("(%d): Expected %d. Got %d", i, expected[i], bytesToInt(prop.Address)) } } - verifyWinningRate(t, vals, 50000, 0.01) + verifyWinningRate(t, vals, 10000, 0.01) // One validator has more than the others *val2 = *newValidator(addr2, 400) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 50000, 0.01) + verifyWinningRate(t, vals, 10000, 0.01) // One validator has more than the others *val2 = *newValidator(addr2, 401) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 100000, 0.01) + verifyWinningRate(t, vals, 10000, 0.01) // each validator should be the proposer a proportional number of times val0, val1, val2 = newValidator(addr0, 4), newValidator(addr1, 5), newValidator(addr2, 3) @@ -228,7 +229,7 @@ func TestProposerSelection2(t *testing.T) { } fmt.Printf("%v\n", propCount) - if propCount[0] != 40038 { + if propCount[0] != 40257 { t.Fatalf( "Expected prop count for validator with 4/12 of voting power to be %d/%d. Got %d/%d", 40038, @@ -237,7 +238,7 @@ func TestProposerSelection2(t *testing.T) { 10000*N, ) } - if propCount[1] != 50077 { + if propCount[1] != 50017 { t.Fatalf( "Expected prop count for validator with 5/12 of voting power to be %d/%d. Got %d/%d", 50077, @@ -246,7 +247,7 @@ func TestProposerSelection2(t *testing.T) { 10000*N, ) } - if propCount[2] != 29885 { + if propCount[2] != 29726 { t.Fatalf( "Expected prop count for validator with 3/12 of voting power to be %d/%d. Got %d/%d", 29885, @@ -267,18 +268,22 @@ func randPubKey() crypto.PubKey { return ed25519.PubKeyEd25519(pubKey) } -func max(a, b int64) int64 { - if a >= b { - return a +func defendLimit(a int64) int64 { + if a <= 0 { + return 1 + } + if a > MaxTotalStakingPower/8 { + a = MaxTotalStakingPower / 8 } - return b + return a } func randValidator(totalStakingPower int64) *Validator { // this modulo limits the ProposerPriority/StakingPower to stay in the // bounds of MaxTotalStakingPower minus the already existing voting power: - val := NewValidator(randPubKey(), max(int64(tmrand.Uint64()%uint64((MaxTotalStakingPower-totalStakingPower))), 1)) - val.ProposerPriority = tmrand.Int64() % (MaxTotalStakingPower - totalStakingPower) + stakingPower := defendLimit(int64(tmrand.Uint64() % uint64(MaxTotalStakingPower-totalStakingPower))) + val := NewValidator(randPubKey(), stakingPower) + val.ProposerPriority = stakingPower return val } @@ -488,7 +493,7 @@ func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { 0 + 6*vp1 - total, // mostest once up to here 0 + 6*vp2}, 6, - vals.Validators[0]}, + vals.Validators[2]}, 6: { vals.Copy(), []int64{ @@ -504,7 +509,7 @@ func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { 0 + 8*vp1 - total, 0 + 8*vp2}, 8, - vals.Validators[0]}, + vals.Validators[2]}, 8: { vals.Copy(), []int64{ @@ -528,7 +533,7 @@ func TestAveragingInIncrementProposerPriorityWithStakingPower(t *testing.T) { 0 + 11*vp1 - total, // after 6 iters this val is "mostest" once and not in between 0 + 11*vp2 - total}, // after 10 iters this val is "mostest" once 11, - vals.Validators[0]}, + vals.Validators[1]}, } for i, tc := range tcs { tc.vals.IncrementProposerPriority(tc.times) diff --git a/types/voter_set.go b/types/voter_set.go index a81ec1a0a..16fb6ffa7 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -15,9 +15,8 @@ import ( ) var ( - MinVoters = 20 - MinTotalVotingPowerPercent = uint(60) - VotingPowerUnit = uint64(10000) + MinVoters = 20 + VotingPowerUnit = uint64(10000) ) // VoterSet represent a set of *Validator at a given height. @@ -108,6 +107,7 @@ func (voters *VoterSet) updateTotalVotingPower() { // mind overflow sum = safeAddClip(sum, val.VotingPower) if sum > MaxTotalStakingPower { + sum = 0 needSum = true voters.downscaleVotingPower() break @@ -409,8 +409,11 @@ func (c *candidate) LessThan(other tmrand.Candidate) bool { return bytes.Compare(c.val.Address, o.val.Address) < 0 } -func (c *candidate) SetWinPoint(winPoint uint64) { - c.val.VotingPower = int64(winPoint) +func (c *candidate) SetWinPoint(winPoint int64) { + if winPoint < 0 { + panic(fmt.Sprintf("VotingPower must not be negative: %d", winPoint)) + } + c.val.VotingPower = winPoint } func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { @@ -422,23 +425,14 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { seed := hashToSeed(proofHash) candidates := make([]tmrand.Candidate, len(validators.Validators)) - totalPriority := uint64(0) for i, val := range validators.Validators { candidates[i] = &candidate{ priority: uint64(val.StakingPower), val: val.Copy(), } - totalPriority += candidates[i].Priority() } - if totalPriority > tmrand.MaxFloat64Significant { - scale := float64(totalPriority) / float64(tmrand.MaxFloat64Significant) - for _, candi := range candidates { - candi.(*candidate).priority = uint64(float64(candi.(*candidate).priority) / scale) - } - } - winners := tmrand.RandomSamplingWithoutReplacement(seed, candidates, MinVoters, MinTotalVotingPowerPercent, - VotingPowerUnit) + winners := tmrand.RandomSamplingWithoutReplacement(seed, candidates, MinVoters, VotingPowerUnit) voters := make([]*Validator, len(winners)) for i, winner := range winners { voters[i] = winner.(*candidate).val @@ -446,21 +440,27 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { return WrapValidatorsToVoterSet(voters) } -// This should be used in only test func ToVoterAll(validators []*Validator) *VoterSet { newVoters := make([]*Validator, len(validators)) - for i, val := range validators { + voterCount := 0 + for _, val := range validators { if val.StakingPower == 0 { // remove the validator with the staking power of 0 from the voter set continue } - newVoters[i] = &Validator{ + newVoters[voterCount] = &Validator{ Address: val.Address, PubKey: val.PubKey, StakingPower: val.StakingPower, VotingPower: val.StakingPower, ProposerPriority: val.ProposerPriority, } + voterCount++ + } + if voterCount < len(newVoters) { + zeroRemoved := make([]*Validator, voterCount) + copy(zeroRemoved, newVoters[:voterCount]) + newVoters = zeroRemoved } sort.Sort(ValidatorsByAddress(newVoters)) return WrapValidatorsToVoterSet(newVoters) diff --git a/types/voter_set_test.go b/types/voter_set_test.go index af46ef67f..db635b44d 100644 --- a/types/voter_set_test.go +++ b/types/voter_set_test.go @@ -9,13 +9,23 @@ import ( tmtime "github.com/tendermint/tendermint/types/time" ) +func countZeroStakingPower(vals []*Validator) int { + count := 0 + for _, v := range vals { + if v.StakingPower == 0 { + count++ + } + } + return count +} + func TestSelectVoter(t *testing.T) { MinVoters = 29 - MinTotalVotingPowerPercent = 100 valSet := randValidatorSet(30) + zeroVals := countZeroStakingPower(valSet.Validators) for i := 0; i < 10000; i++ { voterSet := SelectVoter(valSet, []byte{byte(i)}) - assert.True(t, voterSet.Size() >= 29) + assert.True(t, voterSet.Size() >= 29-zeroVals) if voterSet.totalVotingPower <= 0 { for j := 0; j < voterSet.Size(); j++ { // TODO solve this problem!!! @@ -26,6 +36,24 @@ func TestSelectVoter(t *testing.T) { } } +func TestToVoterAll(t *testing.T) { + valSet := randValidatorSet(30) + vals := valSet.Validators + vals[0].StakingPower = 0 + vals[5].StakingPower = 0 + vals[28].StakingPower = 0 + zeroRemovedVoters := ToVoterAll(vals) + assert.True(t, zeroRemovedVoters.Size() == 27) + + valSet = randValidatorSet(3) + vals = valSet.Validators + vals[0].StakingPower = 0 + vals[1].StakingPower = 0 + vals[2].StakingPower = 0 + zeroRemovedVoters = ToVoterAll(vals) + assert.True(t, zeroRemovedVoters.Size() == 0) +} + func toGenesisValidators(vals []*Validator) []GenesisValidator { genVals := make([]GenesisValidator, len(vals)) for i, val := range vals { From fdb7a022b139ae40a6da6bcc24497ff53a7e6c3f Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Thu, 11 Jun 2020 18:38:15 +0900 Subject: [PATCH 27/38] fix: lint error --- libs/rand/sampling.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 73c46bbc9..3381858c3 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -153,7 +153,7 @@ func RandomSamplingWithoutReplacement( for downscaleNeeded { downscaleNeeded = false for i := range winPoints { - winPoints[i] = winPoints[i] / 10 + winPoints[i] /= 10 if int64(winPoints[i]) < 0 { downscaleNeeded = true } From e33896f5749abfd86b991a905937c603bc9ee3fd Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Fri, 12 Jun 2020 09:12:05 +0900 Subject: [PATCH 28/38] test: add test for randomThreshold --- libs/rand/sampling_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 750857026..f75faeaaf 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -3,6 +3,7 @@ package rand import ( "fmt" "math" + "math/rand" s "sort" "testing" @@ -162,6 +163,39 @@ func accumulateAndResetReward(candidate []Candidate, acc []uint64) { } } +func TestDivider(t *testing.T) { + assert.True(t, divider.Uint64() == uint64Mask+1) +} + +func TestRandomThreshold(t *testing.T) { + loopCount := 100000 + + // randomThreshold() should not return a value greater than total. + for i := 0; i < loopCount; i++ { + seed := rand.Uint64() + total := rand.Int63() + random := randomThreshold(&seed, uint64(total)) + assert.True(t, random < uint64(total)) + } + + // test randomness + total := math.MaxInt64 + bitHit := make([]int, 63) + for i := 0; i < loopCount; i++ { + seed := rand.Uint64() + random := randomThreshold(&seed, uint64(total)) + for j := 0; j < 63; j++ { + if random&(1< 0 { + bitHit[j]++ + } + } + } + // all bit hit count should be near at loopCount/2 + for i := 0; i < len(bitHit); i++ { + assert.True(t, math.Abs(float64(bitHit[i])-float64(loopCount/2))/float64(loopCount/2) < 0.01) + } +} + // test reward fairness func TestRandomSamplingWithoutReplacementReward(t *testing.T) { candidates := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) From 0a189694314eac3fa7a5939ae2950aa3bcea6f48 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Fri, 12 Jun 2020 12:08:00 +0900 Subject: [PATCH 29/38] test: add testing for verifying idempotence of randomThreshold --- libs/rand/sampling_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index f75faeaaf..1b69412a7 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -194,6 +194,21 @@ func TestRandomThreshold(t *testing.T) { for i := 0; i < len(bitHit); i++ { assert.True(t, math.Abs(float64(bitHit[i])-float64(loopCount/2))/float64(loopCount/2) < 0.01) } + + // verify idempotence + expect := [][]uint64{ + {7070836379803831726, 3176749709313725329, 6607573645926202312, 3491641484182981082, 3795411888399561855,}, + {1227844342346046656, 2900311180284727168, 8193302169476290588, 2343329048962716018, 6435608444680946564,}, + {1682153688901572301, 5713119979229610871, 1690050691353843586, 6615539178087966730, 965357176598405746}, + {2092789425003139052, 7803713333738082738, 391680292209432075, 3242280302033391430, 2071067388247806529,}, + {7958955049054603977, 5770386275058218277, 6648532499409218539, 5505026356475271777, 3466385424369377032,}} + for i := 0; i < len(expect); i++ { + seed := uint64(i) + for j := 0; j < len(expect[i]); j++ { + seed = randomThreshold(&seed, uint64(total)) + assert.True(t, seed == expect[i][j]) + } + } } // test reward fairness From f08edeb598ddfa1453c4a6c6e70b91ebc399fb4e Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Fri, 12 Jun 2020 12:11:20 +0900 Subject: [PATCH 30/38] fix: lint error --- libs/rand/sampling_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index 1b69412a7..ae3095d60 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -197,11 +197,11 @@ func TestRandomThreshold(t *testing.T) { // verify idempotence expect := [][]uint64{ - {7070836379803831726, 3176749709313725329, 6607573645926202312, 3491641484182981082, 3795411888399561855,}, - {1227844342346046656, 2900311180284727168, 8193302169476290588, 2343329048962716018, 6435608444680946564,}, + {7070836379803831726, 3176749709313725329, 6607573645926202312, 3491641484182981082, 3795411888399561855}, + {1227844342346046656, 2900311180284727168, 8193302169476290588, 2343329048962716018, 6435608444680946564}, {1682153688901572301, 5713119979229610871, 1690050691353843586, 6615539178087966730, 965357176598405746}, - {2092789425003139052, 7803713333738082738, 391680292209432075, 3242280302033391430, 2071067388247806529,}, - {7958955049054603977, 5770386275058218277, 6648532499409218539, 5505026356475271777, 3466385424369377032,}} + {2092789425003139052, 7803713333738082738, 391680292209432075, 3242280302033391430, 2071067388247806529}, + {7958955049054603977, 5770386275058218277, 6648532499409218539, 5505026356475271777, 3466385424369377032}} for i := 0; i < len(expect); i++ { seed := uint64(i) for j := 0; j < len(expect[i]); j++ { From 08b78350b106990442289db310b677fc8a85f770 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 15 Jun 2020 11:03:15 +0900 Subject: [PATCH 31/38] fix: improve voting power polacy --- libs/rand/sampling.go | 23 ++---- libs/rand/sampling_test.go | 141 +++++++++++++++++++++++++++++++++---- 2 files changed, 132 insertions(+), 32 deletions(-) diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go index 3381858c3..2f4000259 100644 --- a/libs/rand/sampling.go +++ b/libs/rand/sampling.go @@ -96,8 +96,7 @@ func randomThreshold(seed *uint64, total uint64) uint64 { // so it updates rewards of winners. This function continues to elect winners until the both of two // conditions(minSamplingCount, minPriorityPercent) are met. func RandomSamplingWithoutReplacement( - seed uint64, candidates []Candidate, minSamplingCount int, winPointUnit uint64) ( - winners []Candidate) { + seed uint64, candidates []Candidate, minSamplingCount int) (winners []Candidate) { if len(candidates) < minSamplingCount { panic(fmt.Sprintf("The number of candidates(%d) cannot be less minSamplingCount %d", @@ -142,28 +141,16 @@ func RandomSamplingWithoutReplacement( } winners = candidates[len(candidates)-winnerNum:] winPoints := make([]float64, len(winners)) - downscaleNeeded := false + totalWinPoint := float64(0) for i, winner := range winners { - winPoints[i] = float64(winPointUnit) + - float64(winner.Priority())*compensationProportions[i]*float64(winPointUnit) - if int64(winPoints[i]) < 0 { - downscaleNeeded = true - } - } - for downscaleNeeded { - downscaleNeeded = false - for i := range winPoints { - winPoints[i] /= 10 - if int64(winPoints[i]) < 0 { - downscaleNeeded = true - } - } + winPoints[i] = 1 + float64(winner.Priority())*compensationProportions[i] + totalWinPoint += winPoints[i] } for i, winner := range winners { if winPoints[i] > math.MaxInt64 || winPoints[i] < 0 { panic(fmt.Sprintf("winPoint is invalid: %f", winPoints[i])) } - winner.SetWinPoint(int64(winPoints[i])) + winner.SetWinPoint(int64(float64(totalPriority) * winPoints[i] / totalWinPoint)) } return winners } diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index ae3095d60..a3da73b2d 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -101,17 +101,17 @@ func resetWinPoint(candidate []Candidate) { func TestRandomSamplingWithoutReplacement1Candidate(t *testing.T) { candidates := newCandidates(1, func(i int) uint64 { return uint64(1000 * (i + 1)) }) - winners := RandomSamplingWithoutReplacement(0, candidates, 1, 1000) + winners := RandomSamplingWithoutReplacement(0, candidates, 1) assert.True(t, len(winners) == 1) assert.True(t, candidates[0] == winners[0]) assert.True(t, winners[0].(*Element).winPoint == 1000) resetWinPoint(candidates) - winners2 := RandomSamplingWithoutReplacement(0, candidates, 0, 1000) + winners2 := RandomSamplingWithoutReplacement(0, candidates, 0) assert.True(t, len(winners2) == 0) resetWinPoint(candidates) - winners4 := RandomSamplingWithoutReplacement(0, candidates, 0, 1000) + winners4 := RandomSamplingWithoutReplacement(0, candidates, 0) assert.True(t, len(winners4) == 0) resetWinPoint(candidates) } @@ -121,19 +121,39 @@ func TestRandomSamplingWithoutReplacementSamplingThreshold(t *testing.T) { candidates := newCandidates(100, func(i int) uint64 { return uint64(1000 * (i + 1)) }) for i := 1; i <= 100; i++ { - winners := RandomSamplingWithoutReplacement(0, candidates, i, 1000) + winners := RandomSamplingWithoutReplacement(0, candidates, i) assert.True(t, len(winners) == i) resetWinPoint(candidates) } } +// test downscale of win point cases +func TestRandomSamplingWithoutReplacementDownscale(t *testing.T) { + candidates := newCandidates(10, func(i int) uint64 { + if i == 0 { + return math.MaxInt64 >> 1 + } + if i == 1 { + return 1 << 55 + } + if i == 3 { + return 1 << 54 + } + if i == 4 { + return 1 << 53 + } + return uint64(i) + }) + RandomSamplingWithoutReplacement(0, candidates, 5) +} + // test random election should be deterministic func TestRandomSamplingWithoutReplacementDeterministic(t *testing.T) { candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) candidates2 := newCandidates(100, func(i int) uint64 { return uint64(i + 1) }) for i := 1; i <= 100; i++ { - winners1 := RandomSamplingWithoutReplacement(uint64(i), candidates1, 50, 1000) - winners2 := RandomSamplingWithoutReplacement(uint64(i), candidates2, 50, 1000) + winners1 := RandomSamplingWithoutReplacement(uint64(i), candidates1, 50) + winners2 := RandomSamplingWithoutReplacement(uint64(i), candidates2, 50) sameCandidates(winners1, winners2) resetWinPoint(candidates1) resetWinPoint(candidates2) @@ -143,7 +163,7 @@ func TestRandomSamplingWithoutReplacementDeterministic(t *testing.T) { func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) { // first candidate's priority is 0 candidates1 := newCandidates(100, func(i int) uint64 { return uint64(i) }) - winners1 := RandomSamplingWithoutReplacement(0, candidates1, 100, 1000) + winners1 := RandomSamplingWithoutReplacement(0, candidates1, 100) assert.True(t, len(winners1) == 99) candidates2 := newCandidates(100, func(i int) uint64 { @@ -152,7 +172,7 @@ func TestRandomSamplingWithoutReplacementIncludingZeroStakingPower(t *testing.T) } return uint64(i) }) - winners2 := RandomSamplingWithoutReplacement(0, candidates2, 95, 1000) + winners2 := RandomSamplingWithoutReplacement(0, candidates2, 95) assert.True(t, len(winners2) == 90) } @@ -217,9 +237,9 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards := make([]uint64, 100) for i := 0; i < 100000; i++ { - // 24 samplingThreshold is minimum to pass this test - // If samplingThreshold is less than 24, the result says the reward is not fair - RandomSamplingWithoutReplacement(uint64(i), candidates, 24, 10) + // 25 samplingThreshold is minimum to pass this test + // If samplingThreshold is less than 25, the result says the reward is not fair + RandomSamplingWithoutReplacement(uint64(i), candidates, 25) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -228,7 +248,7 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards = make([]uint64, 100) for i := 0; i < 50000; i++ { - RandomSamplingWithoutReplacement(uint64(i), candidates, 50, 10) + RandomSamplingWithoutReplacement(uint64(i), candidates, 50) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -237,7 +257,7 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { accumulatedRewards = make([]uint64, 100) for i := 0; i < 10000; i++ { - RandomSamplingWithoutReplacement(uint64(i), candidates, 100, 10) + RandomSamplingWithoutReplacement(uint64(i), candidates, 100) accumulateAndResetReward(candidates, accumulatedRewards) } for i := 0; i < 99; i++ { @@ -245,6 +265,99 @@ func TestRandomSamplingWithoutReplacementReward(t *testing.T) { } } +/** +conditions for fair reward +1. even staking power(less difference between min staking and max staking) +2. large total staking(a small total staking power makes a large error when converting float into int) +3. many sampling count +4. loop count +*/ +func TestRandomSamplingWithoutReplacementEquity(t *testing.T) { + loopCount := 10000 + + // good condition + candidates := newCandidates(100, func(i int) uint64 { return 1000000 + rand.Uint64()&0xFFFFF }) + accumulatedRewards := make([]uint64, 100) + for i := 0; i < loopCount; i++ { + RandomSamplingWithoutReplacement(uint64(i), candidates, 99) + accumulateAndResetReward(candidates, accumulatedRewards) + } + for i := 0; i < 99; i++ { + rewardPerStakingDiff := + math.Abs(float64(accumulatedRewards[i])/float64(candidates[i].Priority())/float64(loopCount) - 1) + assert.True(t, rewardPerStakingDiff < 0.01) + } + + // violation of condition 1 + candidates = newCandidates(100, func(i int) uint64 { return rand.Uint64() & 0xFFFFFFFFF }) + accumulatedRewards = make([]uint64, 100) + for i := 0; i < loopCount; i++ { + RandomSamplingWithoutReplacement(uint64(i), candidates, 99) + accumulateAndResetReward(candidates, accumulatedRewards) + } + maxRewardPerStakingDiff := float64(0) + for i := 0; i < 99; i++ { + rewardPerStakingDiff := + math.Abs(float64(accumulatedRewards[i])/float64(candidates[i].Priority())/float64(loopCount) - 1) + if maxRewardPerStakingDiff < rewardPerStakingDiff { + maxRewardPerStakingDiff = rewardPerStakingDiff + } + } + t.Logf("[! condition 1] max reward per staking difference: %f", maxRewardPerStakingDiff) + + // violation of condition 2 + candidates = newCandidates(100, func(i int) uint64 { return uint64(1 + i) }) + accumulatedRewards = make([]uint64, 100) + for i := 0; i < loopCount; i++ { + RandomSamplingWithoutReplacement(uint64(i), candidates, 99) + accumulateAndResetReward(candidates, accumulatedRewards) + } + maxRewardPerStakingDiff = float64(0) + for i := 0; i < 99; i++ { + rewardPerStakingDiff := + math.Abs(float64(accumulatedRewards[i])/float64(candidates[i].Priority())/float64(loopCount) - 1) + if maxRewardPerStakingDiff < rewardPerStakingDiff { + maxRewardPerStakingDiff = rewardPerStakingDiff + } + } + t.Logf("[! condition 2] max reward per staking difference: %f", maxRewardPerStakingDiff) + + // violation of condition 3 + candidates = newCandidates(100, func(i int) uint64 { return 1000000 + rand.Uint64()&0xFFFFF }) + accumulatedRewards = make([]uint64, 100) + for i := 0; i < loopCount; i++ { + RandomSamplingWithoutReplacement(uint64(i), candidates, 10) + accumulateAndResetReward(candidates, accumulatedRewards) + } + maxRewardPerStakingDiff = float64(0) + for i := 0; i < 99; i++ { + rewardPerStakingDiff := + math.Abs(float64(accumulatedRewards[i])/float64(candidates[i].Priority())/float64(loopCount) - 1) + if maxRewardPerStakingDiff < rewardPerStakingDiff { + maxRewardPerStakingDiff = rewardPerStakingDiff + } + } + t.Logf("[! condition 3] max reward per staking difference: %f", maxRewardPerStakingDiff) + + // violation of condition 4 + loopCount = 100 + candidates = newCandidates(100, func(i int) uint64 { return 1000000 + rand.Uint64()&0xFFFFF }) + accumulatedRewards = make([]uint64, 100) + for i := 0; i < loopCount; i++ { + RandomSamplingWithoutReplacement(uint64(i), candidates, 99) + accumulateAndResetReward(candidates, accumulatedRewards) + } + maxRewardPerStakingDiff = float64(0) + for i := 0; i < 99; i++ { + rewardPerStakingDiff := + math.Abs(float64(accumulatedRewards[i])/float64(candidates[i].Priority())/float64(loopCount) - 1) + if maxRewardPerStakingDiff < rewardPerStakingDiff { + maxRewardPerStakingDiff = rewardPerStakingDiff + } + } + t.Logf("[! condition 4] max reward per staking difference: %f", maxRewardPerStakingDiff) +} + func TestRandomSamplingWithoutReplacementPanic(t *testing.T) { type Case struct { Candidates []Candidate @@ -263,7 +376,7 @@ func TestRandomSamplingWithoutReplacementPanic(t *testing.T) { t.Errorf("expected panic didn't happen in case %d", i+1) } }() - RandomSamplingWithoutReplacement(0, c.Candidates, c.SamplingThreshold, 1000) + RandomSamplingWithoutReplacement(0, c.Candidates, c.SamplingThreshold) }() } } From deb89eb1ac3e2b35267c520fa0906f735580bd89 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 15 Jun 2020 11:06:12 +0900 Subject: [PATCH 32/38] fix: compile error --- types/voter_set.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/types/voter_set.go b/types/voter_set.go index 16fb6ffa7..c2c6e27a8 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -16,7 +16,6 @@ import ( var ( MinVoters = 20 - VotingPowerUnit = uint64(10000) ) // VoterSet represent a set of *Validator at a given height. @@ -432,7 +431,7 @@ func SelectVoter(validators *ValidatorSet, proofHash []byte) *VoterSet { } } - winners := tmrand.RandomSamplingWithoutReplacement(seed, candidates, MinVoters, VotingPowerUnit) + winners := tmrand.RandomSamplingWithoutReplacement(seed, candidates, MinVoters) voters := make([]*Validator, len(winners)) for i, winner := range winners { voters[i] = winner.(*candidate).val From d30feec45b4b1b18a82c5e51dc3b9eac4a9b9cad Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 15 Jun 2020 11:13:04 +0900 Subject: [PATCH 33/38] fix: lint error --- types/voter_set.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/voter_set.go b/types/voter_set.go index c2c6e27a8..2ada45ce0 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -15,7 +15,7 @@ import ( ) var ( - MinVoters = 20 + MinVoters = 20 ) // VoterSet represent a set of *Validator at a given height. From 06e84e8088e129c542cae05372b11883c179886b Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 15 Jun 2020 11:26:23 +0900 Subject: [PATCH 34/38] fix: test case --- libs/rand/sampling_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index a3da73b2d..aff0d0f84 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -306,7 +306,7 @@ func TestRandomSamplingWithoutReplacementEquity(t *testing.T) { t.Logf("[! condition 1] max reward per staking difference: %f", maxRewardPerStakingDiff) // violation of condition 2 - candidates = newCandidates(100, func(i int) uint64 { return uint64(1 + i) }) + candidates = newCandidates(100, func(i int) uint64 { return rand.Uint64() & 0xFFFFF }) accumulatedRewards = make([]uint64, 100) for i := 0; i < loopCount; i++ { RandomSamplingWithoutReplacement(uint64(i), candidates, 99) From 4387cb6e98586b4c5959bedf720a05a1395ae5af Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Mon, 15 Jun 2020 21:09:27 +0900 Subject: [PATCH 35/38] test: add comment --- libs/rand/sampling_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go index aff0d0f84..1783b5801 100644 --- a/libs/rand/sampling_test.go +++ b/libs/rand/sampling_test.go @@ -288,6 +288,10 @@ func TestRandomSamplingWithoutReplacementEquity(t *testing.T) { assert.True(t, rewardPerStakingDiff < 0.01) } + // ======================================================================================================= + // The codes below are not test codes to verify logic, + // but codes to find out what parameters are that weaken the equity of rewards. + // violation of condition 1 candidates = newCandidates(100, func(i int) uint64 { return rand.Uint64() & 0xFFFFFFFFF }) accumulatedRewards = make([]uint64, 100) From 94b29753dde24a644a8f960e9ee38fff1f58b944 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 16 Jun 2020 08:36:35 +0900 Subject: [PATCH 36/38] fix: remove unused function --- types/validator_set.go | 2 +- types/voter_set.go | 26 +++++++++----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/types/validator_set.go b/types/validator_set.go index 11fc7aaee..e37a1462b 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -264,7 +264,7 @@ func (vals *ValidatorSet) updateTotalStakingPower() { sum = safeAddClip(sum, val.StakingPower) if sum > MaxTotalStakingPower { panic(fmt.Sprintf( - "Total voting power should be guarded to not exceed %v; got: %v", + "Total staking power should be guarded to not exceed %v; got: %v", MaxTotalStakingPower, sum)) } diff --git a/types/voter_set.go b/types/voter_set.go index 2ada45ce0..c265bcf99 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -90,27 +90,19 @@ func (voters *VoterSet) Copy() *VoterSet { } } -func (voters *VoterSet) downscaleVotingPower() { - for _, v := range voters.Voters { - v.VotingPower = int64(float64(v.VotingPower) / 10) - } -} - // Forces recalculation of the set's total voting power. // Panics if total voting power is bigger than MaxTotalStakingPower. func (voters *VoterSet) updateTotalVotingPower() { sum := int64(0) - for needSum := true; needSum; { - needSum = false - for _, val := range voters.Voters { - // mind overflow - sum = safeAddClip(sum, val.VotingPower) - if sum > MaxTotalStakingPower { - sum = 0 - needSum = true - voters.downscaleVotingPower() - break - } + for _, val := range voters.Voters { + // mind overflow + sum = safeAddClip(sum, val.VotingPower) + // total voting power cannot be exceed total staking power that cannot be exceed MaxTotalStakingPower + if sum > MaxTotalStakingPower { + panic(fmt.Sprintf( + "Total voting power should be guarded to not exceed %v; got: %v", + MaxTotalStakingPower, + sum)) } } voters.totalVotingPower = sum From 3efc1d7a97d8bba4f7a4d99887d0f8bfb3cff1d5 Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 16 Jun 2020 09:38:22 +0900 Subject: [PATCH 37/38] fix: define MaxTotalVotingPower --- types/validator_set.go | 11 +++++++++++ types/validator_set_test.go | 17 +++++++++++++++++ types/voter_set.go | 5 ++--- types/voter_set_test.go | 19 +++++++++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/types/validator_set.go b/types/validator_set.go index e37a1462b..8ff1afe25 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -24,6 +24,17 @@ const ( // and leaves room for defensive purposes. MaxTotalStakingPower = int64(math.MaxInt64) / 8 + // MaxTotalVotingPower should be same to MaxTotalStakingPower theoretically, + // but the value can be higher when it is type-casted as float64 + // because of the number of valid digits of float64. + // This phenomenon occurs in the following computations. + // + // `winner.SetWinPoint(int64(float64(totalPriority) * winPoints[i] / totalWinPoint))` lib/rand/sampling.go + // + // MaxTotalVotingPower can be as large as MaxTotalStakingPower+8. + // Please refer TestMaxVotingPowerTest for this. + MaxTotalVotingPower = MaxTotalStakingPower + 8 + // PriorityWindowSizeFactor - is a constant that when multiplied with the total voting power gives // the maximum allowed distance between validator priorities. PriorityWindowSizeFactor = 2 diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 3318f3388..a173425e4 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -22,6 +22,23 @@ import ( tmtime "github.com/tendermint/tendermint/types/time" ) +func TestMaxVotingPowerTest(t *testing.T) { + large := MaxTotalStakingPower + maxDiff := int64(0) + for i := 0; i < 8; i++ { + for j := 0; j < 8; j++ { + testNum := (large - int64(i)) >> j + casted := int64(float64(testNum)) + t.Logf("org=%d, casting=%d", testNum, casted) + if maxDiff < casted-testNum { + maxDiff = casted - testNum + } + } + } + t.Logf("max difference=%d", maxDiff) + assert.True(t, MaxTotalStakingPower+maxDiff <= MaxTotalVotingPower) +} + func TestValidatorSetBasic(t *testing.T) { // empty or nil validator lists are allowed, // but attempting to IncrementProposerPriority on them will panic. diff --git a/types/voter_set.go b/types/voter_set.go index c265bcf99..3ae4daf20 100644 --- a/types/voter_set.go +++ b/types/voter_set.go @@ -97,11 +97,10 @@ func (voters *VoterSet) updateTotalVotingPower() { for _, val := range voters.Voters { // mind overflow sum = safeAddClip(sum, val.VotingPower) - // total voting power cannot be exceed total staking power that cannot be exceed MaxTotalStakingPower - if sum > MaxTotalStakingPower { + if sum > MaxTotalVotingPower { panic(fmt.Sprintf( "Total voting power should be guarded to not exceed %v; got: %v", - MaxTotalStakingPower, + MaxTotalVotingPower, sum)) } } diff --git a/types/voter_set_test.go b/types/voter_set_test.go index db635b44d..2b353130c 100644 --- a/types/voter_set_test.go +++ b/types/voter_set_test.go @@ -19,6 +19,25 @@ func countZeroStakingPower(vals []*Validator) int { return count } +func TestMaxStakingPowerTest(t *testing.T) { + large := MaxTotalStakingPower + maxDiff := int64(0) + for i := 0; i < 8; i++ { + for j := 0; j < 3; j++ { + testNum := (large - int64(i)) >> j + casted := int64(float64(testNum)) + t.Logf("org=%d, casting=%d", testNum, casted) + if maxDiff < casted-testNum { + maxDiff = casted - testNum + } + } + } + t.Logf("max difference=%d", maxDiff) + stakingPower := MaxTotalStakingPower - 1 + votingPower := int64(float64(stakingPower) * 0.1 * 10) + assert.True(t, votingPower <= MaxTotalStakingPower+maxDiff) +} + func TestSelectVoter(t *testing.T) { MinVoters = 29 valSet := randValidatorSet(30) From 1c759e3411d2d4a732493f55371b23db8e8350ac Mon Sep 17 00:00:00 2001 From: Woosang Son Date: Tue, 16 Jun 2020 14:44:35 +0900 Subject: [PATCH 38/38] fix: remove useless test case, and leave todo --- types/validator_set.go | 6 ++++-- types/voter_set_test.go | 19 ------------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/types/validator_set.go b/types/validator_set.go index 8ff1afe25..4eb0093ea 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -31,9 +31,11 @@ const ( // // `winner.SetWinPoint(int64(float64(totalPriority) * winPoints[i] / totalWinPoint))` lib/rand/sampling.go // - // MaxTotalVotingPower can be as large as MaxTotalStakingPower+8. + // MaxTotalVotingPower can be as large as MaxTotalStakingPower+alpha + // but I don't know the exact alpha. 1000 seems to be enough by some examination. // Please refer TestMaxVotingPowerTest for this. - MaxTotalVotingPower = MaxTotalStakingPower + 8 + // TODO: 1000 is temporary limit, we should remove float calculation and then we can fix this limit + MaxTotalVotingPower = MaxTotalStakingPower + 1000 // PriorityWindowSizeFactor - is a constant that when multiplied with the total voting power gives // the maximum allowed distance between validator priorities. diff --git a/types/voter_set_test.go b/types/voter_set_test.go index 2b353130c..db635b44d 100644 --- a/types/voter_set_test.go +++ b/types/voter_set_test.go @@ -19,25 +19,6 @@ func countZeroStakingPower(vals []*Validator) int { return count } -func TestMaxStakingPowerTest(t *testing.T) { - large := MaxTotalStakingPower - maxDiff := int64(0) - for i := 0; i < 8; i++ { - for j := 0; j < 3; j++ { - testNum := (large - int64(i)) >> j - casted := int64(float64(testNum)) - t.Logf("org=%d, casting=%d", testNum, casted) - if maxDiff < casted-testNum { - maxDiff = casted - testNum - } - } - } - t.Logf("max difference=%d", maxDiff) - stakingPower := MaxTotalStakingPower - 1 - votingPower := int64(float64(stakingPower) * 0.1 * 10) - assert.True(t, votingPower <= MaxTotalStakingPower+maxDiff) -} - func TestSelectVoter(t *testing.T) { MinVoters = 29 valSet := randValidatorSet(30)