Add pluggable BEEFY payload constructors (paritytech#12428)

* primitives/beefy: move Payload to its own file

* primitives/beefy: add Payload tests

* primitives/beefy: add MmrRootProvider as custom BEEFY payload provider

* client/beefy: use generic BEEFY 'PayloadProvider'

* primitives/beefy: rename Payload::new to Payload::from_single_entry for clarity

* fix visibility

* fix cargo doc

client/beefy/rpc/src/lib.rs

@@ -170,7 +170,7 @@ mod tests {
 		communication::notification::BeefyVersionedFinalityProofSender,
 		justification::BeefyVersionedFinalityProof,
 	};
-	use beefy_primitives::{known_payload_ids, Payload, SignedCommitment};
+	use beefy_primitives::{known_payloads, Payload, SignedCommitment};
 	use codec::{Decode, Encode};
 	use jsonrpsee::{types::EmptyParams, RpcModule};
 	use sp_runtime::traits::{BlakeTwo256, Hash};
@@ -266,7 +266,8 @@ mod tests {
 	}
 
 	fn create_finality_proof() -> BeefyVersionedFinalityProof<Block> {
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		BeefyVersionedFinalityProof::<Block>::V1(SignedCommitment {
 			commitment: beefy_primitives::Commitment {
 				payload,

client/beefy/src/communication/gossip.rs

@@ -237,8 +237,7 @@ mod tests {
 
 	use crate::keystore::{tests::Keyring, BeefyKeystore};
 	use beefy_primitives::{
-		crypto::Signature, known_payload_ids, Commitment, MmrRootHash, Payload, VoteMessage,
-		KEY_TYPE,
+		crypto::Signature, known_payloads, Commitment, MmrRootHash, Payload, VoteMessage, KEY_TYPE,
 	};
 
 	use super::*;
@@ -348,7 +347,10 @@ mod tests {
 	}
 
 	fn dummy_vote(block_number: u64) -> VoteMessage<u64, Public, Signature> {
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, MmrRootHash::default().encode());
+		let payload = Payload::from_single_entry(
+			known_payloads::MMR_ROOT_ID,
+			MmrRootHash::default().encode(),
+		);
 		let commitment = Commitment { payload, block_number, validator_set_id: 0 };
 		let signature = sign_commitment(&Keyring::Alice, &commitment);
 

client/beefy/src/justification.rs

@@ -81,7 +81,7 @@ fn verify_with_validator_set<Block: BlockT>(
 #[cfg(test)]
 pub(crate) mod tests {
 	use beefy_primitives::{
-		known_payload_ids, Commitment, Payload, SignedCommitment, VersionedFinalityProof,
+		known_payloads, Commitment, Payload, SignedCommitment, VersionedFinalityProof,
 	};
 	use substrate_test_runtime_client::runtime::Block;
 
@@ -94,7 +94,7 @@ pub(crate) mod tests {
 		keys: &[Keyring],
 	) -> BeefyVersionedFinalityProof<Block> {
 		let commitment = Commitment {
-			payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+			payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 			block_number: block_num,
 			validator_set_id: validator_set.id(),
 		};

client/beefy/src/lib.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use beefy_primitives::{BeefyApi, MmrRootHash};
+use beefy_primitives::{BeefyApi, MmrRootHash, PayloadProvider};
 use parking_lot::Mutex;
 use prometheus::Registry;
 use sc_client_api::{Backend, BlockBackend, BlockchainEvents, Finalizer};
@@ -167,11 +167,13 @@ pub struct BeefyNetworkParams<B: Block, N> {
 }
 
 /// BEEFY gadget initialization parameters.
-pub struct BeefyParams<B: Block, BE, C, N, R> {
+pub struct BeefyParams<B: Block, BE, C, N, P, R> {
 	/// BEEFY client
 	pub client: Arc<C>,
 	/// Client Backend
 	pub backend: Arc<BE>,
+	/// BEEFY Payload provider
+	pub payload_provider: P,
 	/// Runtime Api Provider
 	pub runtime: Arc<R>,
 	/// Local key store
@@ -191,18 +193,20 @@ pub struct BeefyParams<B: Block, BE, C, N, R> {
 /// Start the BEEFY gadget.
 ///
 /// This is a thin shim around running and awaiting a BEEFY worker.
-pub async fn start_beefy_gadget<B, BE, C, N, R>(beefy_params: BeefyParams<B, BE, C, N, R>)
+pub async fn start_beefy_gadget<B, BE, C, N, P, R>(beefy_params: BeefyParams<B, BE, C, N, P, R>)
 where
 	B: Block,
 	BE: Backend<B>,
 	C: Client<B, BE> + BlockBackend<B>,
+	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B>,
 	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
 	N: GossipNetwork<B> + NetworkRequest + SyncOracle + Send + Sync + 'static,
 {
 	let BeefyParams {
 		client,
 		backend,
+		payload_provider,
 		runtime,
 		key_store,
 		network_params,
@@ -249,6 +253,7 @@ where
 	let worker_params = worker::WorkerParams {
 		client,
 		backend,
+		payload_provider,
 		runtime,
 		network,
 		key_store: key_store.into(),
@@ -261,7 +266,7 @@ where
 		min_block_delta,
 	};
 
-	let worker = worker::BeefyWorker::<_, _, _, _, _>::new(worker_params);
+	let worker = worker::BeefyWorker::<_, _, _, _, _, _>::new(worker_params);
 
 	futures::future::join(worker.run(), on_demand_justifications_handler.run()).await;
 }

client/beefy/src/tests.rs

@@ -38,6 +38,7 @@ use sc_utils::notification::NotificationReceiver;
 
 use beefy_primitives::{
 	crypto::{AuthorityId, Signature},
+	mmr::MmrRootProvider,
 	BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, VersionedFinalityProof, BEEFY_ENGINE_ID,
 	KEY_TYPE as BeefyKeyType,
 };
@@ -372,10 +373,12 @@ where
 			justifications_protocol_name: on_demand_justif_handler.protocol_name(),
 			_phantom: PhantomData,
 		};
+		let payload_provider = MmrRootProvider::new(api.clone());
 
 		let beefy_params = crate::BeefyParams {
 			client: peer.client().as_client(),
 			backend: peer.client().as_backend(),
+			payload_provider,
 			runtime: api.clone(),
 			key_store: Some(keystore),
 			network_params,
@@ -384,7 +387,7 @@ where
 			prometheus_registry: None,
 			on_demand_justifications_handler: on_demand_justif_handler,
 		};
-		let task = crate::start_beefy_gadget::<_, _, _, _, _>(beefy_params);
+		let task = crate::start_beefy_gadget::<_, _, _, _, _, _>(beefy_params);
 
 		fn assert_send<T: Send>(_: &T) {}
 		assert_send(&task);

client/beefy/src/worker.rs

@@ -48,7 +48,7 @@ use sp_runtime::{
 
 use beefy_primitives::{
 	crypto::{AuthorityId, Signature},
-	known_payload_ids, BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, SignedCommitment,
+	BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, PayloadProvider, SignedCommitment,
 	ValidatorSet, VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
 };
 
@@ -194,9 +194,10 @@ impl<B: Block> VoterOracle<B> {
 	}
 }
 
-pub(crate) struct WorkerParams<B: Block, BE, C, R, N> {
+pub(crate) struct WorkerParams<B: Block, BE, C, P, R, N> {
 	pub client: Arc<C>,
 	pub backend: Arc<BE>,
+	pub payload_provider: P,
 	pub runtime: Arc<R>,
 	pub network: N,
 	pub key_store: BeefyKeystore,
@@ -210,10 +211,11 @@ pub(crate) struct WorkerParams<B: Block, BE, C, R, N> {
 }
 
 /// A BEEFY worker plays the BEEFY protocol
-pub(crate) struct BeefyWorker<B: Block, BE, C, R, N> {
+pub(crate) struct BeefyWorker<B: Block, BE, C, P, R, N> {
 	// utilities
 	client: Arc<C>,
 	backend: Arc<BE>,
+	payload_provider: P,
 	runtime: Arc<R>,
 	network: N,
 	key_store: BeefyKeystore,
@@ -243,11 +245,12 @@ pub(crate) struct BeefyWorker<B: Block, BE, C, R, N> {
 	voting_oracle: VoterOracle<B>,
 }
 
-impl<B, BE, C, R, N> BeefyWorker<B, BE, C, R, N>
+impl<B, BE, C, P, R, N> BeefyWorker<B, BE, C, P, R, N>
 where
 	B: Block + Codec,
 	BE: Backend<B>,
 	C: Client<B, BE>,
+	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B>,
 	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
 	N: NetworkEventStream + NetworkRequest + SyncOracle + Send + Sync + Clone + 'static,
@@ -258,10 +261,11 @@ where
 	/// BEEFY pallet has been deployed on-chain.
 	///
 	/// The BEEFY pallet is needed in order to keep track of the BEEFY authority set.
-	pub(crate) fn new(worker_params: WorkerParams<B, BE, C, R, N>) -> Self {
+	pub(crate) fn new(worker_params: WorkerParams<B, BE, C, P, R, N>) -> Self {
 		let WorkerParams {
 			client,
 			backend,
+			payload_provider,
 			runtime,
 			key_store,
 			network,
@@ -282,6 +286,7 @@ where
 		BeefyWorker {
 			client: client.clone(),
 			backend,
+			payload_provider,
 			runtime,
 			network,
 			known_peers,
@@ -299,17 +304,6 @@ where
 		}
 	}
 
-	/// Simple wrapper that gets MMR root from header digests or from client state.
-	fn get_mmr_root_digest(&self, header: &B::Header) -> Option<MmrRootHash> {
-		find_mmr_root_digest::<B>(header).or_else(|| {
-			self.runtime
-				.runtime_api()
-				.mmr_root(&BlockId::hash(header.hash()))
-				.ok()
-				.and_then(|r| r.ok())
-		})
-	}
-
 	/// Verify `active` validator set for `block` against the key store
 	///
 	/// We want to make sure that we have _at least one_ key in our keystore that
@@ -621,13 +615,12 @@ where
 		};
 		let target_hash = target_header.hash();
 
-		let mmr_root = if let Some(hash) = self.get_mmr_root_digest(&target_header) {
+		let payload = if let Some(hash) = self.payload_provider.payload(&target_header) {
 			hash
 		} else {
 			warn!(target: "beefy", "🥩 No MMR root digest found for: {:?}", target_hash);
 			return Ok(())
 		};
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, mmr_root.encode());
 
 		let rounds = self.voting_oracle.rounds_mut().ok_or(Error::UninitSession)?;
 		if !rounds.should_self_vote(&(payload.clone(), target_number)) {
@@ -917,20 +910,6 @@ where
 	}
 }
 
-/// Extract the MMR root hash from a digest in the given header, if it exists.
-fn find_mmr_root_digest<B>(header: &B::Header) -> Option<MmrRootHash>
-where
-	B: Block,
-{
-	let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID);
-
-	let filter = |log: ConsensusLog<AuthorityId>| match log {
-		ConsensusLog::MmrRoot(root) => Some(root),
-		_ => None,
-	};
-	header.digest().convert_first(|l| l.try_to(id).and_then(filter))
-}
-
 /// Scan the `header` digest log for a BEEFY validator set change. Return either the new
 /// validator set or `None` in case no validator set change has been signaled.
 fn find_authorities_change<B>(header: &B::Header) -> Option<ValidatorSet<AuthorityId>>
@@ -1016,8 +995,8 @@ pub(crate) mod tests {
 		BeefyRPCLinks,
 	};
 
+	use beefy_primitives::{known_payloads, mmr::MmrRootProvider};
 	use futures::{executor::block_on, future::poll_fn, task::Poll};
-
 	use sc_client_api::{Backend as BackendT, HeaderBackend};
 	use sc_network::NetworkService;
 	use sc_network_test::{PeersFullClient, TestNetFactory};
@@ -1032,7 +1011,14 @@ pub(crate) mod tests {
 		peer: &BeefyPeer,
 		key: &Keyring,
 		min_block_delta: u32,
-	) -> BeefyWorker<Block, Backend, PeersFullClient, TestApi, Arc<NetworkService<Block, H256>>> {
+	) -> BeefyWorker<
+		Block,
+		Backend,
+		PeersFullClient,
+		MmrRootProvider<Block, TestApi>,
+		TestApi,
+		Arc<NetworkService<Block, H256>>,
+	> {
 		let keystore = create_beefy_keystore(*key);
 
 		let (to_rpc_justif_sender, from_voter_justif_stream) =
@@ -1064,9 +1050,11 @@ pub(crate) mod tests {
 			"/beefy/justifs/1".into(),
 			known_peers.clone(),
 		);
+		let payload_provider = MmrRootProvider::new(api.clone());
 		let worker_params = crate::worker::WorkerParams {
 			client: peer.client().as_client(),
 			backend: peer.client().as_backend(),
+			payload_provider,
 			runtime: api,
 			key_store: Some(keystore).into(),
 			known_peers,
@@ -1078,7 +1066,7 @@ pub(crate) mod tests {
 			network,
 			on_demand_justifications,
 		};
-		BeefyWorker::<_, _, _, _, _>::new(worker_params)
+		BeefyWorker::<_, _, _, _, _, _>::new(worker_params)
 	}
 
 	#[test]
@@ -1300,30 +1288,6 @@ pub(crate) mod tests {
 		assert_eq!(extracted, Some(validator_set));
 	}
 
-	#[test]
-	fn extract_mmr_root_digest() {
-		let mut header = Header::new(
-			1u32.into(),
-			Default::default(),
-			Default::default(),
-			Default::default(),
-			Digest::default(),
-		);
-
-		// verify empty digest shows nothing
-		assert!(find_mmr_root_digest::<Block>(&header).is_none());
-
-		let mmr_root_hash = H256::random();
-		header.digest_mut().push(DigestItem::Consensus(
-			BEEFY_ENGINE_ID,
-			ConsensusLog::<AuthorityId>::MmrRoot(mmr_root_hash).encode(),
-		));
-
-		// verify validator set is correctly extracted from digest
-		let extracted = find_mmr_root_digest::<Block>(&header);
-		assert_eq!(extracted, Some(mmr_root_hash));
-	}
-
 	#[test]
 	fn keystore_vs_validator_set() {
 		let keys = &[Keyring::Alice];
@@ -1363,7 +1327,7 @@ pub(crate) mod tests {
 
 		let create_finality_proof = |block_num: NumberFor<Block>| {
 			let commitment = Commitment {
-				payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+				payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 				block_number: block_num,
 				validator_set_id: validator_set.id(),
 			};
@@ -1482,7 +1446,7 @@ pub(crate) mod tests {
 			block_number: NumberFor<Block>,
 		) -> VoteMessage<NumberFor<Block>, AuthorityId, Signature> {
 			let commitment = Commitment {
-				payload: Payload::new(*b"BF", vec![]),
+				payload: Payload::from_single_entry(*b"BF", vec![]),
 				block_number,
 				validator_set_id: 0,
 			};
@@ -1574,7 +1538,7 @@ pub(crate) mod tests {
 
 			// import/append BEEFY justification for session boundary block 10
 			let commitment = Commitment {
-				payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+				payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 				block_number: 10,
 				validator_set_id: validator_set.id(),
 			};
@@ -1608,7 +1572,7 @@ pub(crate) mod tests {
 
 			// import/append BEEFY justification for block 12
 			let commitment = Commitment {
-				payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+				payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 				block_number: 12,
 				validator_set_id: validator_set.id(),
 			};