From c47f395f871608e0dde8117f78b2aaface68ffac Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Tue, 20 Aug 2019 19:56:18 -0700 Subject: [PATCH 1/4] Implement kickout for chunk producers --- chain/chain/src/chain.rs | 4 +- chain/epoch_manager/src/lib.rs | 267 +++++++++++++++---- chain/epoch_manager/src/reward_calculator.rs | 3 + chain/epoch_manager/src/types.rs | 4 +- core/primitives/src/block.rs | 19 +- core/protos/protos/chain.proto | 1 + near/src/runtime.rs | 267 ++++++++++++------- near/tests/sync_nodes.rs | 1 + runtime/runtime/src/lib.rs | 1 + 9 files changed, 416 insertions(+), 151 deletions(-) diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index ce2261a857c..794d42d431c 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -362,7 +362,7 @@ impl Chain { header.height, header.validator_proposal.clone(), vec![], - vec![], + header.chunk_mask.clone(), header.gas_used, header.gas_price, header.total_supply, @@ -1449,7 +1449,7 @@ impl<'a> ChainUpdate<'a> { block.header.height, block.header.validator_proposal.clone(), vec![], - vec![], + block.header.chunk_mask.clone(), block.header.gas_used, block.header.gas_price, block.header.total_supply, diff --git a/chain/epoch_manager/src/lib.rs b/chain/epoch_manager/src/lib.rs index 984d38d9884..bb9334896f1 100644 --- a/chain/epoch_manager/src/lib.rs +++ b/chain/epoch_manager/src/lib.rs @@ -69,6 +69,78 @@ impl EpochManager { Ok(epoch_manager) } + fn compute_kickout_info( + &self, + epoch_info: &EpochInfo, + block_validator_tracker: &HashMap, + num_expected_blocks: &HashMap, + chunk_validator_tracker: &HashMap>, + num_expected_chunks: &HashMap>, + slashed: &HashSet, + ) -> (HashSet, HashMap) { + let mut all_kicked_out = true; + let mut maximum_block_prod = 0; + let mut max_validator_id = None; + let validator_kickout_threshold = self.config.validator_kickout_threshold; + let mut validator_online_ratio = HashMap::new(); + let mut validator_kickout = HashSet::new(); + + for (i, _) in epoch_info.validators.iter().enumerate() { + let account_id = epoch_info.validators[i].account_id.clone(); + if slashed.contains(&account_id) { + continue; + } + let num_blocks = block_validator_tracker.get(&i).unwrap_or(&0).clone(); + // Note, validator_kickout_threshold is 0..100, so we use * 100 to keep this in integer space. + let expected_blocks = *num_expected_blocks.get(&i).unwrap_or(&0); + if num_blocks * 100 < (validator_kickout_threshold as u64) * expected_blocks { + validator_kickout.insert(account_id); + continue; + } + let mut total_chunks_expected = 0; + let mut total_chunks_produced = 0; + for (shard_id, tracker) in num_expected_chunks.iter() { + if tracker.contains_key(&i) { + let num_expected = *tracker.get(&i).unwrap(); + let num_produced = *chunk_validator_tracker + .get(shard_id) + .and_then(|t| t.get(&i)) + .unwrap_or(&0); + total_chunks_expected += num_expected; + total_chunks_produced += num_produced; + } + } + if total_chunks_produced * 100 + < validator_kickout_threshold as u64 * total_chunks_expected + { + validator_kickout.insert(account_id.clone()); + continue; + } + + // Given the number of blocks we plan to have in one epoch, the following code should not overflow + validator_online_ratio.insert( + account_id.clone(), + ( + num_blocks * total_chunks_expected + expected_blocks * total_chunks_produced, + total_chunks_expected * expected_blocks * 2, + ), + ); + if !validator_kickout.contains(&account_id) { + all_kicked_out = false; + } + if num_blocks > maximum_block_prod { + maximum_block_prod = num_blocks; + max_validator_id = Some(i); + } + } + if all_kicked_out { + if let Some(validator_id) = max_validator_id { + validator_kickout.remove(&epoch_info.validators[validator_id].account_id); + } + } + (validator_kickout, validator_online_ratio) + } + fn collect_blocks_info( &mut self, epoch_id: &EpochId, @@ -76,7 +148,8 @@ impl EpochManager { ) -> Result { let mut proposals = BTreeMap::new(); let mut validator_kickout = HashSet::new(); - let mut validator_tracker = HashMap::new(); + let mut block_validator_tracker = HashMap::new(); + let mut chunk_validator_tracker = HashMap::new(); let mut total_gas_used = 0; let epoch_info = self.get_epoch_info(epoch_id)?.clone(); @@ -107,9 +180,18 @@ impl EpochManager { proposals.entry(proposal.account_id.clone()).or_insert(proposal); } } - let validator_id = self.block_producer_from_info(&epoch_info, info.index); - // println!(" validator: {:?}", validator_id); - validator_tracker.entry(validator_id).and_modify(|e| *e += 1).or_insert(1); + let block_validator_id = self.block_producer_from_info(&epoch_info, info.index); + block_validator_tracker.entry(block_validator_id).and_modify(|e| *e += 1).or_insert(1); + for (i, mask) in info.chunk_mask.iter().enumerate() { + let chunk_validator_id = + self.chunk_producer_from_info(&epoch_info, info.index, i as ShardId); + let tracker = + chunk_validator_tracker.entry(i as ShardId).or_insert_with(|| HashMap::new()); + if *mask { + tracker.entry(chunk_validator_id).and_modify(|e| *e += 1).or_insert(1); + } + } + total_gas_used += info.gas_used; hash = info.prev_hash; @@ -119,46 +201,22 @@ impl EpochManager { let last_block_info = self.get_block_info(&last_block_hash)?.clone(); let first_block_info = self.get_block_info(&last_block_info.epoch_first_block)?.clone(); - let num_expected_blocks = self.get_num_expected_blocks(&epoch_info, &first_block_info)?; - // println!("Num expected: {:?}, validators: {:?}", num_expected_blocks, epoch_info); + let (num_expected_blocks, num_expected_chunks) = + self.get_num_expected_blocks_and_chunks(&epoch_info, &first_block_info)?; // Compute kick outs for validators who are offline. - let mut all_kicked_out = true; - let mut maximum_block_prod = 0; - let mut max_validator_id = None; - let validator_kickout_threshold = self.config.validator_kickout_threshold; - let mut validator_online_ratio = HashMap::new(); - // println!("{}: {:?} {:?}", first_block_info.index, num_expected_blocks, validator_tracker); - - for (i, _) in epoch_info.validators.iter().enumerate() { - let mut num_blocks = validator_tracker.get(&i).unwrap_or(&0).clone(); - let account_id = epoch_info.validators[i].account_id.clone(); - // Note, validator_kickout_threshold is 0..100, so we use * 100 to keep this in integer space. - let expected_blocks = *num_expected_blocks.get(&i).unwrap_or(&0); - validator_online_ratio.insert(account_id.clone(), (num_blocks, expected_blocks)); - if num_blocks * 100 < (validator_kickout_threshold as u64) * expected_blocks { - validator_kickout.insert(account_id); - } else { - if !validator_kickout.contains(&account_id) { - all_kicked_out = false; - } else { - num_blocks = 0; - } - } - if num_blocks > maximum_block_prod { - maximum_block_prod = num_blocks; - max_validator_id = Some(i); - } - } - // If all validators kicked out, keep the one that produce most of the blocks. - if all_kicked_out { - if let Some(validator_id) = max_validator_id { - validator_kickout.remove(&epoch_info.validators[validator_id].account_id); - } - } + let (kickout, validator_online_ratio) = self.compute_kickout_info( + &epoch_info, + &block_validator_tracker, + &num_expected_blocks, + &chunk_validator_tracker, + &num_expected_chunks, + &slashed_validators, + ); + validator_kickout = validator_kickout.union(&kickout).cloned().collect(); println!( - "All proposals: {:?}, Kickouts: {:?}, Tracker: {:?}, Num expected: {:?}", - all_proposals, validator_kickout, validator_tracker, num_expected_blocks + "All proposals: {:?}, Kickouts: {:?}, Block Tracker: {:?}, Shard Tracker: {:?}, Num expected: {:?}", + all_proposals, validator_kickout, block_validator_tracker, chunk_validator_tracker, num_expected_blocks ); Ok(EpochSummary { @@ -484,23 +542,34 @@ impl EpochManager { Ok(false) } - fn get_num_expected_blocks( + fn get_num_expected_blocks_and_chunks( &mut self, epoch_info: &EpochInfo, epoch_first_block_info: &BlockInfo, - ) -> Result, EpochError> { - let mut num_expected = HashMap::default(); + ) -> Result<(HashMap, HashMap>), EpochError> + { + let mut num_expected_blocks = HashMap::default(); + let mut num_expected_chunks = HashMap::default(); let prev_epoch_last_block = self.get_block_info(&epoch_first_block_info.prev_hash)?; + let num_shards = epoch_first_block_info.chunk_mask.len() as ShardId; // We iterate from next index after previous epoch's last block, for epoch_length blocks. for index in (prev_epoch_last_block.index + 1) ..=(prev_epoch_last_block.index + self.config.epoch_length) { - num_expected + num_expected_blocks .entry(self.block_producer_from_info(epoch_info, index)) .and_modify(|e| *e += 1) .or_insert(1); + for i in 0..num_shards { + num_expected_chunks + .entry(i) + .or_insert_with(|| HashMap::new()) + .entry(self.chunk_producer_from_info(epoch_info, index, i as ShardId)) + .and_modify(|e| *e += 1) + .or_insert(1); + } } - Ok(num_expected) + Ok((num_expected_blocks, num_expected_chunks)) } fn block_producer_from_info(&self, epoch_info: &EpochInfo, index: BlockIndex) -> ValidatorId { @@ -892,7 +961,7 @@ mod tests { 1, h[0], vec![], - vec![], + vec![true], slashed, 0, DEFAULT_GAS_PRICE, @@ -926,7 +995,7 @@ mod tests { vec![], change_stake(vec![("test1", 0), ("test2", amount_staked)]), 0, - reward(vec![("test1", 0), ("test2", 0), ("near", 0)]) + reward(vec![("test2", 0), ("near", 0)]) ) ); @@ -994,7 +1063,7 @@ mod tests { epoch_first_block: h[0], epoch_id: Default::default(), proposals: vec![], - validator_mask: vec![], + chunk_mask: vec![true], slashed: Default::default(), gas_used: 0, gas_price: DEFAULT_GAS_PRICE, @@ -1012,7 +1081,7 @@ mod tests { epoch_first_block: h[1], epoch_id: Default::default(), proposals: vec![], - validator_mask: vec![], + chunk_mask: vec![true], slashed: Default::default(), gas_used: 10, gas_price: DEFAULT_GAS_PRICE, @@ -1030,7 +1099,7 @@ mod tests { epoch_first_block: h[1], epoch_id: Default::default(), proposals: vec![], - validator_mask: vec![], + chunk_mask: vec![true], slashed: Default::default(), gas_used: 10, gas_price: DEFAULT_GAS_PRICE, @@ -1064,4 +1133,100 @@ mod tests { ) ); } + + #[test] + fn test_reward_multiple_shards() { + let stake_amount = 1_000_000; + let validators = vec![("test1", stake_amount), ("test2", stake_amount)]; + let epoch_length = 2; + let total_supply = stake_amount * validators.len() as u128; + let reward_calculator = RewardCalculator { + max_inflation_rate: 5, + num_blocks_per_year: 1_000_000, + epoch_length, + validator_reward_percentage: 60, + protocol_reward_percentage: 10, + protocol_treasury_account: "near".to_string(), + }; + let mut epoch_manager = + setup_epoch_manager(validators, epoch_length, 2, 2, 0, 90, reward_calculator.clone()); + let rng_seed = [0; 32]; + let h = hash_range(5); + epoch_manager + .record_block_info( + &h[0], + BlockInfo { + index: 0, + prev_hash: Default::default(), + epoch_first_block: h[0], + epoch_id: Default::default(), + proposals: vec![], + chunk_mask: vec![], + slashed: Default::default(), + gas_used: 0, + gas_price: DEFAULT_GAS_PRICE, + total_supply, + }, + rng_seed, + ) + .unwrap(); + epoch_manager + .record_block_info( + &h[1], + BlockInfo { + index: 1, + prev_hash: h[0], + epoch_first_block: h[1], + epoch_id: Default::default(), + proposals: vec![], + chunk_mask: vec![true, false], + slashed: Default::default(), + gas_used: 10, + gas_price: DEFAULT_GAS_PRICE, + total_supply, + }, + rng_seed, + ) + .unwrap(); + epoch_manager + .record_block_info( + &h[2], + BlockInfo { + index: 2, + prev_hash: h[1], + epoch_first_block: h[1], + epoch_id: Default::default(), + proposals: vec![], + chunk_mask: vec![true, true], + slashed: Default::default(), + gas_used: 10, + gas_price: DEFAULT_GAS_PRICE, + total_supply, + }, + rng_seed, + ) + .unwrap(); + let mut validator_online_ratio = HashMap::new(); + validator_online_ratio.insert("test2".to_string(), (1, 1)); + let validator_reward = reward_calculator.calculate_reward( + validator_online_ratio, + 20, + DEFAULT_GAS_PRICE, + total_supply, + ); + let test2_reward = *validator_reward.get("test2").unwrap(); + let protocol_reward = *validator_reward.get("near").unwrap(); + assert_eq!( + epoch_manager.get_epoch_info(&EpochId(h[2])).unwrap(), + &epoch_info( + vec![("test2", stake_amount + test2_reward)], + vec![0, 0], + vec![vec![0, 0], vec![0, 0]], + vec![], + change_stake(vec![("test1", 0), ("test2", stake_amount + test2_reward)]), + 20, + reward(vec![("test2", test2_reward), ("near", protocol_reward)]) + ) + ); + } } diff --git a/chain/epoch_manager/src/reward_calculator.rs b/chain/epoch_manager/src/reward_calculator.rs index 6cbe13405eb..e0c44f17a04 100644 --- a/chain/epoch_manager/src/reward_calculator.rs +++ b/chain/epoch_manager/src/reward_calculator.rs @@ -32,6 +32,9 @@ impl RewardCalculator { let epoch_protocol_treasury = epoch_total_reward * self.protocol_reward_percentage as u128 / 100; res.insert(self.protocol_treasury_account.clone(), epoch_protocol_treasury); + if num_validators == 0 { + return res; + } let epoch_per_validator_reward = (epoch_total_reward - epoch_protocol_treasury) / num_validators as u128; for (account_id, (num_blocks, expected_num_blocks)) in validator_online_ratio { diff --git a/chain/epoch_manager/src/types.rs b/chain/epoch_manager/src/types.rs index 768405061fe..7902030075b 100644 --- a/chain/epoch_manager/src/types.rs +++ b/chain/epoch_manager/src/types.rs @@ -57,7 +57,7 @@ pub struct BlockInfo { pub epoch_id: EpochId, pub proposals: Vec, - pub validator_mask: Vec, + pub chunk_mask: Vec, pub slashed: HashSet, pub gas_used: GasUsage, pub gas_price: Balance, @@ -79,7 +79,7 @@ impl BlockInfo { index, prev_hash, proposals, - validator_mask, + chunk_mask: validator_mask, slashed, gas_used, gas_price, diff --git a/core/primitives/src/block.rs b/core/primitives/src/block.rs index 00ab1f84b7d..77744cc470f 100644 --- a/core/primitives/src/block.rs +++ b/core/primitives/src/block.rs @@ -47,6 +47,8 @@ pub struct BlockHeader { pub total_weight: Weight, /// Validator proposals. pub validator_proposal: Vec, + /// Mask for new chunks included in the block + pub chunk_mask: Vec, /// Epoch start hash of the previous epoch. /// Used for retrieving validator information pub epoch_id: EpochId, @@ -81,6 +83,7 @@ impl BlockHeader { approval_sigs: Vec, total_weight: Weight, mut validator_proposal: Vec, + chunk_mask: Vec, epoch_id: EpochId, gas_used: GasUsage, gas_limit: GasUsage, @@ -101,6 +104,7 @@ impl BlockHeader { validator_proposal: RepeatedField::from_iter( validator_proposal.drain(..).map(std::convert::Into::into), ), + chunk_mask, epoch_id: epoch_id.0.into(), gas_used, gas_limit, @@ -121,6 +125,7 @@ impl BlockHeader { approval_sigs: Vec, total_weight: Weight, validator_proposal: Vec, + chunk_mask: Vec, epoch_id: EpochId, gas_used: GasUsage, gas_limit: GasUsage, @@ -138,6 +143,7 @@ impl BlockHeader { approval_sigs, total_weight, validator_proposal, + chunk_mask, epoch_id, gas_used, gas_limit, @@ -172,6 +178,7 @@ impl BlockHeader { vec![], 0.into(), vec![], + vec![], EpochId::default(), 0, initial_gas_limit, @@ -226,6 +233,7 @@ impl TryFrom for BlockHeader { .into_iter() .map(TryInto::try_into) .collect::, _>>()?; + let chunk_mask = body.chunk_mask; let epoch_id = EpochId(body.epoch_id.try_into()?); Ok(BlockHeader { height, @@ -237,6 +245,7 @@ impl TryFrom for BlockHeader { approval_sigs, total_weight, validator_proposal, + chunk_mask, epoch_id, gas_used: body.gas_used, gas_limit: body.gas_limit, @@ -265,6 +274,7 @@ impl From for chain_proto::BlockHeader { validator_proposal: RepeatedField::from_iter( header.validator_proposal.drain(..).map(std::convert::Into::into), ), + chunk_mask: header.chunk_mask, epoch_id: header.epoch_id.0.into(), gas_used: header.gas_used, gas_limit: header.gas_limit, @@ -280,9 +290,6 @@ impl From for chain_proto::BlockHeader { } } -#[derive(Serialize, Deserialize, Debug)] -pub struct Bytes(Vec); - #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] pub struct Block { pub header: BlockHeader, @@ -356,11 +363,16 @@ impl Block { let mut validator_proposals = vec![]; let mut gas_used = 0; let mut gas_limit = 0; + // This computation of chunk_mask relies on the fact that chunks are ordered by shard_id. + let mut chunk_mask = vec![]; for chunk in chunks.iter() { if chunk.height_included == height { validator_proposals.extend_from_slice(&chunk.validator_proposal); gas_used += chunk.gas_used; gas_limit += chunk.gas_limit; + chunk_mask.push(true); + } else { + chunk_mask.push(false); } } @@ -390,6 +402,7 @@ impl Block { approval_sigs, total_weight, validator_proposals, + chunk_mask, epoch_id, gas_used, gas_limit, diff --git a/core/protos/protos/chain.proto b/core/protos/protos/chain.proto index 279f3a8bba9..3e88e21282f 100644 --- a/core/protos/protos/chain.proto +++ b/core/protos/protos/chain.proto @@ -20,6 +20,7 @@ message BlockHeaderBody { uint64 gas_limit = 12; Uint128 gas_price = 13; Uint128 total_supply = 14; + repeated bool chunk_mask = 15; } message BlockHeader { diff --git a/near/src/runtime.rs b/near/src/runtime.rs index 8ad3f940334..f348a578743 100644 --- a/near/src/runtime.rs +++ b/near/src/runtime.rs @@ -96,7 +96,7 @@ impl NightshadeRuntime { /// and allocate rewards. fn update_validator_accounts( &self, - _shard_id: ShardId, + shard_id: ShardId, _state_root: &MerkleHash, block_hash: &CryptoHash, state_update: &mut TrieUpdate, @@ -105,31 +105,36 @@ impl NightshadeRuntime { let (stake_info, validator_reward) = epoch_manager.compute_stake_return_info(block_hash)?; for (account_id, max_of_stakes) in stake_info { - let account: Option = get_account(state_update, &account_id); - if let Some(mut account) = account { - if let Some(reward) = validator_reward.get(&account_id) { + if self.account_id_to_shard_id(&account_id) == shard_id { + let account: Option = get_account(state_update, &account_id); + if let Some(mut account) = account { + if let Some(reward) = validator_reward.get(&account_id) { + println!( + "account {} adding reward {} to stake {}", + account_id, reward, account.stake + ); + account.stake += *reward; + } + println!( - "account {} adding reward {} to stake {}", - account_id, reward, account.stake + "account {} stake {} max_of_stakes: {}", + account_id, account.stake, max_of_stakes ); - account.stake += *reward; - } - - println!( - "account {} stake {} max_of_stakes: {}", - account_id, account.stake, max_of_stakes - ); - assert!(account.stake >= max_of_stakes, "FATAL: staking invariance does not hold"); - let return_stake = account.stake - max_of_stakes; - account.stake -= return_stake; - account.amount += return_stake; + assert!( + account.stake >= max_of_stakes, + "FATAL: staking invariant does not hold" + ); + let return_stake = account.stake - max_of_stakes; + account.stake -= return_stake; + account.amount += return_stake; - set_account(state_update, &account_id, &account); + set_account(state_update, &account_id, &account); + } } } - if let Some(mut protocol_treasury_account) = - get_account(state_update, &self.genesis_config.protocol_treasury_account) - { + if self.account_id_to_shard_id(&self.genesis_config.protocol_treasury_account) == shard_id { + let mut protocol_treasury_account = + get_account(state_update, &self.genesis_config.protocol_treasury_account).unwrap(); protocol_treasury_account.amount += *validator_reward.get(&self.genesis_config.protocol_treasury_account).unwrap(); set_account( @@ -138,6 +143,7 @@ impl NightshadeRuntime { &protocol_treasury_account, ); } + Ok(()) } } @@ -366,7 +372,7 @@ impl RuntimeAdapter for NightshadeRuntime { block_index: BlockIndex, proposals: Vec, slashed_validators: Vec, - validator_mask: Vec, + chunk_mask: Vec, gas_used: GasUsage, gas_price: Balance, total_supply: Balance, @@ -384,7 +390,7 @@ impl RuntimeAdapter for NightshadeRuntime { block_index, parent_hash, proposals, - validator_mask, + chunk_mask, slashed, gas_used, gas_price, @@ -430,7 +436,9 @@ impl RuntimeAdapter for NightshadeRuntime { &mut state_update, ) .map_err(|e| Error::from(ErrorKind::ValidatorError(e.to_string())))?; + state_update.commit(); } + let apply_state = ApplyState { root: *state_root, shard_id, @@ -584,7 +592,7 @@ impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime { mod test { use tempdir::TempDir; - use near_chain::{RuntimeAdapter, Tip}; + use near_chain::{ReceiptResult, RuntimeAdapter, Tip}; use near_client::BlockProducer; use near_primitives::block::Weight; use near_primitives::crypto::signer::{EDSigner, InMemorySigner}; @@ -596,7 +604,7 @@ mod test { TransactionBody, }; use near_primitives::types::{ - AccountId, Balance, BlockIndex, EpochId, MerkleHash, Nonce, ValidatorStake, + AccountId, Balance, BlockIndex, EpochId, MerkleHash, Nonce, ShardId, ValidatorStake, }; use near_store::create_store; use node_runtime::adapter::ViewRuntimeAdapter; @@ -604,6 +612,7 @@ mod test { use crate::config::{TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; use crate::{get_store_path, GenesisConfig, NightshadeRuntime}; use near_chain::types::ValidatorSignatureVerificationResult; + use std::collections::{BTreeSet, HashMap}; fn stake(nonce: Nonce, sender: &BlockProducer, amount: Balance) -> SignedTransaction { TransactionBody::Stake(StakeTransaction { @@ -619,17 +628,17 @@ mod test { fn update( &self, state_root: &CryptoHash, + shard_id: ShardId, block_index: BlockIndex, prev_block_hash: &CryptoHash, block_hash: &CryptoHash, receipts: &Vec, transactions: &Vec, - ) -> (CryptoHash, Vec, Vec>) { - let mut root = *state_root; + ) -> (CryptoHash, Vec, ReceiptResult) { let result = self .apply_transactions( - 0, - &root, + shard_id, + &state_root, block_index, prev_block_hash, block_hash, @@ -640,9 +649,7 @@ mod test { let mut store_update = self.store.store_update(); result.trie_changes.insertions_into(&mut store_update).unwrap(); store_update.commit().unwrap(); - root = result.new_root; - let new_receipts = result.receipt_result.into_iter().map(|(_, v)| v).collect(); - (root, result.validator_proposals, new_receipts) + (result.new_root, result.validator_proposals, result.receipt_result) } } @@ -650,15 +657,24 @@ mod test { pub runtime: NightshadeRuntime, pub head: Tip, state_roots: Vec, - pub last_receipts: Vec>, + pub last_receipts: HashMap>, } impl TestEnv { - pub fn new(prefix: &str, validators: Vec, epoch_length: BlockIndex) -> Self { + pub fn new( + prefix: &str, + validators: Vec>, + epoch_length: BlockIndex, + ) -> Self { let dir = TempDir::new(prefix).unwrap(); let store = create_store(&get_store_path(dir.path())); - let mut genesis_config = - GenesisConfig::test(validators.iter().map(|v| v.as_str()).collect()); + let all_validators = validators.iter().fold(BTreeSet::new(), |acc, x| { + acc.union(&x.iter().map(|x| x.as_str()).collect()).cloned().collect() + }); + let mut genesis_config = GenesisConfig::test_sharded( + all_validators.into_iter().collect(), + validators.iter().map(|x| x.len()).collect(), + ); genesis_config.epoch_length = epoch_length; let runtime = NightshadeRuntime::new(dir.path(), store, genesis_config.clone()); let (store_update, state_roots) = runtime.genesis_state(); @@ -687,36 +703,49 @@ mod test { total_weight: Weight::default(), }, state_roots, - last_receipts: vec![], + last_receipts: HashMap::new(), } } - pub fn step(&mut self, transactions: Vec) { - // TODO: add support for shards. + pub fn step(&mut self, transactions: Vec>, chunk_mask: Vec) { let new_hash = hash(&vec![(self.head.height + 1) as u8]); - let (state_root, proposals, receipts) = self.runtime.update( - &self.state_roots[0], - self.head.height + 1, - &self.head.last_block_hash, - &new_hash, - &self.last_receipts.iter().flatten().cloned().collect::>(), - &transactions, - ); - self.state_roots[0] = state_root; - self.last_receipts = receipts; + let num_shards = self.runtime.num_shards(); + assert_eq!(transactions.len() as ShardId, num_shards); + let mut all_proposals = vec![]; + let mut new_receipts = HashMap::new(); + for i in 0..num_shards { + let (state_root, mut proposals, receipts) = self.runtime.update( + &self.state_roots[i as usize], + i, + self.head.height + 1, + &self.head.last_block_hash, + &new_hash, + self.last_receipts.get(&i).unwrap_or(&vec![]), + &transactions[i as usize], + ); + self.state_roots[i as usize] = state_root; + for (shard_id, mut shard_receipts) in receipts { + new_receipts + .entry(shard_id) + .or_insert_with(|| vec![]) + .append(&mut shard_receipts); + } + all_proposals.append(&mut proposals); + } self.runtime .add_validator_proposals( self.head.last_block_hash, new_hash, self.head.height + 1, - proposals, - vec![], + all_proposals, vec![], + chunk_mask, 0, self.runtime.genesis_config.gas_price, self.runtime.genesis_config.total_supply, ) .unwrap(); + self.last_receipts = new_receipts; self.head = Tip { last_block_hash: new_hash, prev_block_hash: self.head.last_block_hash, @@ -726,9 +755,16 @@ mod test { }; } + /// Step when there is only one shard + pub fn step_default(&mut self, transactions: Vec) { + self.step(vec![transactions], vec![true]); + } + pub fn view_account(&self, account_id: &str) -> AccountViewCallResult { - // TODO: add support for shards. - self.runtime.view_account(self.state_roots[0], &account_id.to_string()).unwrap() + let shard_id = self.runtime.account_id_to_shard_id(&account_id.to_string()); + self.runtime + .view_account(self.state_roots[shard_id as usize], &account_id.to_string()) + .unwrap() } /// Compute per epoch per validator reward and per epoch protocol treasury reward @@ -756,45 +792,44 @@ mod test { fn test_validator_rotation() { let num_nodes = 2; let validators = (0..num_nodes).map(|i| format!("test{}", i + 1)).collect::>(); - let mut env = TestEnv::new("test_validator_rotation", validators.clone(), 2); + let mut env = TestEnv::new("test_validator_rotation", vec![validators.clone()], 2); let block_producers: Vec<_> = validators.iter().map(|id| InMemorySigner::from_seed(id, id).into()).collect(); + // test1 doubles stake and the new account stakes the same, so test2 will be kicked out. let staking_transaction = stake(1, &block_producers[0], TESTING_INIT_STAKE * 2); + let new_account = format!("test{}", num_nodes + 1); + let new_validator: BlockProducer = + InMemorySigner::from_seed(&new_account, &new_account).into(); + let create_account_transaction = TransactionBody::CreateAccount(CreateAccountTransaction { + nonce: 2, + originator: block_producers[0].account_id.clone(), + new_account_id: new_account, + amount: TESTING_INIT_STAKE * 3, + public_key: new_validator.signer.public_key().0[..].to_vec(), + }) + .sign(&*block_producers[0].signer.clone()); - // test1 stakes twice the current stake, because test1 and test2 have the same amount of stake before, test2 will be - // kicked out. - env.step(vec![staking_transaction]); + env.step_default(vec![staking_transaction, create_account_transaction]); + env.step_default(vec![]); let account = env.view_account(&block_producers[0].account_id); assert_eq!( account, AccountViewCallResult { account_id: block_producers[0].account_id.clone(), - nonce: 1, - amount: TESTING_INIT_BALANCE - TESTING_INIT_STAKE * 2, + nonce: 2, + amount: TESTING_INIT_BALANCE - TESTING_INIT_STAKE * 5, stake: TESTING_INIT_STAKE * 2, public_keys: vec![block_producers[0].signer.public_key()], code_hash: account.code_hash, } ); - let new_account = format!("test{}", num_nodes + 1); - let new_validator: BlockProducer = - InMemorySigner::from_seed(&new_account, &new_account).into(); - let create_account_transaction = TransactionBody::CreateAccount(CreateAccountTransaction { - nonce: 2, - originator: block_producers[0].account_id.clone(), - new_account_id: new_account, - amount: TESTING_INIT_STAKE * 3, - public_key: new_validator.signer.public_key().0[..].to_vec(), - }) - .sign(&*block_producers[0].signer.clone()); let staking_transaction = stake(1, &new_validator, TESTING_INIT_STAKE * 2); - env.step(vec![create_account_transaction]); - env.step(vec![staking_transaction]); + env.step_default(vec![staking_transaction]); // Roll steps for 3 epochs to pass. for _ in 4..=9 { - env.step(vec![]); + env.step_default(vec![]); } let epoch_id = env.runtime.get_epoch_id_from_prev_block(&env.head.last_block_hash).unwrap(); @@ -840,13 +875,13 @@ mod test { fn test_validator_stake_change() { let num_nodes = 2; let validators = (0..num_nodes).map(|i| format!("test{}", i + 1)).collect::>(); - let mut env = TestEnv::new("test_validator_stake_change", validators.clone(), 2); + let mut env = TestEnv::new("test_validator_stake_change", vec![validators.clone()], 2); let block_producers: Vec<_> = validators.iter().map(|id| InMemorySigner::from_seed(id, id).into()).collect(); let (per_epoch_per_validator_reward, _) = env.compute_reward(num_nodes); let staking_transaction = stake(1, &block_producers[0], TESTING_INIT_STAKE - 1); - env.step(vec![staking_transaction]); + env.step_default(vec![staking_transaction]); let account = env.view_account(&block_producers[0].account_id); assert_eq!( account, @@ -860,7 +895,7 @@ mod test { } ); for _ in 2..=4 { - env.step(vec![]); + env.step_default(vec![]); } let account = env.view_account(&block_producers[0].account_id); @@ -877,7 +912,7 @@ mod test { ); for _ in 5..=7 { - env.step(vec![]); + env.step_default(vec![]); } let account = env.view_account(&block_producers[0].account_id); @@ -901,7 +936,7 @@ mod test { let num_nodes = 4; let validators = (0..num_nodes).map(|i| format!("test{}", i + 1)).collect::>(); let mut env = - TestEnv::new("test_validator_stake_change_multiple_times", validators.clone(), 4); + TestEnv::new("test_validator_stake_change_multiple_times", vec![validators.clone()], 4); let block_producers: Vec<_> = validators.iter().map(|id| InMemorySigner::from_seed(id, id).into()).collect(); let (per_epoch_per_validator_reward, _) = env.compute_reward(num_nodes); @@ -909,7 +944,7 @@ mod test { let staking_transaction = stake(1, &block_producers[0], TESTING_INIT_STAKE - 1); let staking_transaction1 = stake(2, &block_producers[0], TESTING_INIT_STAKE - 2); let staking_transaction2 = stake(1, &block_producers[1], TESTING_INIT_STAKE + 1); - env.step(vec![staking_transaction, staking_transaction1, staking_transaction2]); + env.step_default(vec![staking_transaction, staking_transaction1, staking_transaction2]); let account = env.view_account(&block_producers[0].account_id); assert_eq!( account, @@ -928,7 +963,7 @@ mod test { let staking_transaction2 = stake(3, &block_producers[1], TESTING_INIT_STAKE - 1); let staking_transaction3 = stake(1, &block_producers[3], TESTING_INIT_STAKE - per_epoch_per_validator_reward - 1); - env.step(vec![ + env.step_default(vec![ staking_transaction, staking_transaction1, staking_transaction2, @@ -936,7 +971,7 @@ mod test { ]); for _ in 3..=8 { - env.step(vec![]); + env.step_default(vec![]); } let account = env.view_account(&block_producers[0].account_id); @@ -993,7 +1028,7 @@ mod test { ); for _ in 9..=12 { - env.step(vec![]); + env.step_default(vec![]); } let account = env.view_account(&block_producers[0].account_id); @@ -1057,7 +1092,7 @@ mod test { ); for _ in 13..=16 { - env.step(vec![]); + env.step_default(vec![]); } let account = env.view_account(&block_producers[0].account_id); @@ -1121,7 +1156,7 @@ mod test { #[test] fn test_verify_validator_signature() { let validators = (0..2).map(|i| format!("test{}", i + 1)).collect::>(); - let env = TestEnv::new("verify_validator_signature_failure", validators.clone(), 2); + let env = TestEnv::new("verify_validator_signature_failure", vec![validators.clone()], 2); let data = [0; 32]; let signer = InMemorySigner::from_seed(&validators[0], &validators[0]); let signature = signer.sign(&data); @@ -1139,7 +1174,7 @@ mod test { #[test] fn test_verify_validator_signature_failure() { let validators = (0..2).map(|i| format!("test{}", i + 1)).collect::>(); - let env = TestEnv::new("verify_validator_signature_failure", validators.clone(), 2); + let env = TestEnv::new("verify_validator_signature_failure", vec![validators.clone()], 2); let data = [0; 32]; let signer = InMemorySigner::from_seed(&validators[0], &validators[0]); let signature = signer.sign(&data); @@ -1158,17 +1193,15 @@ mod test { fn test_state_sync() { let num_nodes = 2; let validators = (0..num_nodes).map(|i| format!("test{}", i + 1)).collect::>(); - let mut env = - TestEnv::new("test_validator_stake_change_multiple_times", validators.clone(), 2); + let mut env = TestEnv::new("test_state_sync", vec![validators.clone()], 2); let block_producers: Vec<_> = validators.iter().map(|id| InMemorySigner::from_seed(id, id).into()).collect(); let (per_epoch_per_validator_reward, _) = env.compute_reward(num_nodes); let staking_transaction = stake(1, &block_producers[0], TESTING_INIT_STAKE + 1); - env.step(vec![staking_transaction]); - env.step(vec![]); + env.step_default(vec![staking_transaction]); + env.step_default(vec![]); let state_dump = env.runtime.dump_state(0, env.state_roots[0]).unwrap(); - let mut new_env = - TestEnv::new("test_validator_stake_change_multiple_times1", validators.clone(), 2); + let mut new_env = TestEnv::new("test_state_sync", vec![validators.clone()], 2); for i in 1..=2 { let prev_hash = hash(&[new_env.head.height as u8]); let cur_hash = hash(&[(new_env.head.height + 1) as u8]); @@ -1189,7 +1222,7 @@ mod test { i, proposals, vec![], - vec![], + vec![true], 0, new_env.runtime.genesis_config.gas_price, new_env.runtime.genesis_config.total_supply, @@ -1202,7 +1235,7 @@ mod test { new_env.runtime.set_state(0, env.state_roots[0], state_dump).unwrap(); new_env.state_roots[0] = env.state_roots[0]; for _ in 3..=5 { - new_env.step(vec![]); + new_env.step_default(vec![]); } let account = new_env.view_account(&block_producers[0].account_id); @@ -1232,4 +1265,52 @@ mod test { } ); } + + /// Test two shards: the first shard has 2 validators (test1, test4) and the second shard + /// has 4 validators (test1, test2, test3, test4). Test that kickout and stake change + /// work properly. + #[test] + fn test_multiple_shards() { + let num_nodes = 4; + let first_shard_validators = (0..2).map(|i| format!("test{}", i + 1)).collect::>(); + let second_shard_validators = + (0..num_nodes).map(|i| format!("test{}", i + 1)).collect::>(); + let validators = second_shard_validators.clone(); + let mut env = TestEnv::new( + "test_multiple_shards", + vec![first_shard_validators, second_shard_validators], + 4, + ); + let block_producers: Vec<_> = + validators.iter().map(|id| InMemorySigner::from_seed(id, id).into()).collect(); + let (per_epoch_per_validator_reward, _) = env.compute_reward(num_nodes); + let staking_transaction = stake(1, &block_producers[0], TESTING_INIT_STAKE - 1); + let first_account_shard_id = env.runtime.account_id_to_shard_id(&"test1".to_string()); + let transactions = if first_account_shard_id == 0 { + vec![vec![staking_transaction], vec![]] + } else { + vec![vec![], vec![staking_transaction]] + }; + env.step(transactions, vec![false, true]); + for _ in 2..10 { + env.step(vec![vec![], vec![]], vec![true, true]); + } + let account = env.view_account(&block_producers[3].account_id); + assert_eq!(account.stake, TESTING_INIT_STAKE); + assert_eq!( + account.amount, + TESTING_INIT_BALANCE - TESTING_INIT_STAKE + per_epoch_per_validator_reward + ); + + let account = env.view_account(&block_producers[0].account_id); + assert_eq!(account.stake, TESTING_INIT_STAKE + per_epoch_per_validator_reward - 1); + + for _ in 10..13 { + env.step(vec![vec![], vec![]], vec![true, true]); + } + let account = env.view_account(&block_producers[3].account_id); + assert_eq!(account.stake, 0); + let account = env.view_account(&block_producers[0].account_id); + assert_eq!(account.stake, TESTING_INIT_STAKE - 1); + } } diff --git a/near/tests/sync_nodes.rs b/near/tests/sync_nodes.rs index b0e709acc37..01e76b8afb8 100644 --- a/near/tests/sync_nodes.rs +++ b/near/tests/sync_nodes.rs @@ -207,6 +207,7 @@ fn sync_state_stake_change() { let mut genesis_config = GenesisConfig::test(vec!["test1"]); genesis_config.epoch_length = 5; + genesis_config.validator_kickout_threshold = 80; let (port1, port2) = (open_port(), open_port()); let mut near1 = load_test_config("test1", port1, &genesis_config); diff --git a/runtime/runtime/src/lib.rs b/runtime/runtime/src/lib.rs index baaac3e95de..fda8b0737ed 100644 --- a/runtime/runtime/src/lib.rs +++ b/runtime/runtime/src/lib.rs @@ -764,6 +764,7 @@ impl Runtime { } Err(s) => { state_update.rollback(); + println!("runtime error: {}", s); result.logs.push(format!("Runtime error: {}", s)); result.status = TransactionStatus::Failed; } From ba17dd455a80df2498bf4213efb4131140b0bbcf Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Wed, 21 Aug 2019 11:07:00 -0700 Subject: [PATCH 2/4] fix tests --- chain/epoch_manager/src/lib.rs | 3 ++- near/src/runtime.rs | 14 +++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/chain/epoch_manager/src/lib.rs b/chain/epoch_manager/src/lib.rs index bb9334896f1..6a18bb9b4d1 100644 --- a/chain/epoch_manager/src/lib.rs +++ b/chain/epoch_manager/src/lib.rs @@ -961,7 +961,7 @@ mod tests { 1, h[0], vec![], - vec![true], + vec![], slashed, 0, DEFAULT_GAS_PRICE, @@ -986,6 +986,7 @@ mod tests { record_block(&mut epoch_manager, h[4], h[5], 5, vec![]); let epoch_id = epoch_manager.get_epoch_id(&h[5]).unwrap(); + assert_eq!(epoch_id.0, h[2]); assert_eq!( epoch_manager.get_epoch_info(&epoch_id).unwrap(), &epoch_info( diff --git a/near/src/runtime.rs b/near/src/runtime.rs index f348a578743..b937a93ce84 100644 --- a/near/src/runtime.rs +++ b/near/src/runtime.rs @@ -97,7 +97,6 @@ impl NightshadeRuntime { fn update_validator_accounts( &self, shard_id: ShardId, - _state_root: &MerkleHash, block_hash: &CryptoHash, state_update: &mut TrieUpdate, ) -> Result<(), Box> { @@ -429,13 +428,8 @@ impl RuntimeAdapter for NightshadeRuntime { // If we are starting to apply 1st block in the new epoch. if should_update_account { println!("block index: {}", block_index); - self.update_validator_accounts( - shard_id, - state_root, - prev_block_hash, - &mut state_update, - ) - .map_err(|e| Error::from(ErrorKind::ValidatorError(e.to_string())))?; + self.update_validator_accounts(shard_id, prev_block_hash, &mut state_update) + .map_err(|e| Error::from(ErrorKind::ValidatorError(e.to_string())))?; state_update.commit(); } @@ -1305,12 +1299,10 @@ mod test { let account = env.view_account(&block_producers[0].account_id); assert_eq!(account.stake, TESTING_INIT_STAKE + per_epoch_per_validator_reward - 1); - for _ in 10..13 { + for _ in 10..14 { env.step(vec![vec![], vec![]], vec![true, true]); } let account = env.view_account(&block_producers[3].account_id); assert_eq!(account.stake, 0); - let account = env.view_account(&block_producers[0].account_id); - assert_eq!(account.stake, TESTING_INIT_STAKE - 1); } } From 9bca65ad94567134306a81b979e559617f07d0b9 Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Thu, 22 Aug 2019 09:47:11 -0700 Subject: [PATCH 3/4] address comments --- chain/epoch_manager/src/lib.rs | 4 ++-- near/src/runtime.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/chain/epoch_manager/src/lib.rs b/chain/epoch_manager/src/lib.rs index 6a18bb9b4d1..d767c5dfb7e 100644 --- a/chain/epoch_manager/src/lib.rs +++ b/chain/epoch_manager/src/lib.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; -use log::warn; +use log::{info, warn}; use near_primitives::hash::CryptoHash; use near_primitives::types::{ @@ -214,7 +214,7 @@ impl EpochManager { &slashed_validators, ); validator_kickout = validator_kickout.union(&kickout).cloned().collect(); - println!( + info!( "All proposals: {:?}, Kickouts: {:?}, Block Tracker: {:?}, Shard Tracker: {:?}, Num expected: {:?}", all_proposals, validator_kickout, block_validator_tracker, chunk_validator_tracker, num_expected_blocks ); diff --git a/near/src/runtime.rs b/near/src/runtime.rs index b937a93ce84..30aa1ecc30a 100644 --- a/near/src/runtime.rs +++ b/near/src/runtime.rs @@ -430,7 +430,6 @@ impl RuntimeAdapter for NightshadeRuntime { println!("block index: {}", block_index); self.update_validator_accounts(shard_id, prev_block_hash, &mut state_update) .map_err(|e| Error::from(ErrorKind::ValidatorError(e.to_string())))?; - state_update.commit(); } let apply_state = ApplyState { From e7a2db07d4dea31d5f41cda4688656912417089a Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Fri, 23 Aug 2019 12:16:25 -0700 Subject: [PATCH 4/4] Better error message --- near/src/runtime.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/near/src/runtime.rs b/near/src/runtime.rs index 30aa1ecc30a..a8fb325365b 100644 --- a/near/src/runtime.rs +++ b/near/src/runtime.rs @@ -121,7 +121,9 @@ impl NightshadeRuntime { ); assert!( account.stake >= max_of_stakes, - "FATAL: staking invariant does not hold" + "FATAL: staking invariant does not hold. Account stake {} is less than maximum of stakes {} in the past three epochs", + account.stake, + max_of_stakes ); let return_stake = account.stake - max_of_stakes; account.stake -= return_stake;