Merge branch 'master' into ao-past-session-slashing-runtime

* master:
  Companion: wasm-builder support stable Rust (#6967)
  Fix feature (#6966)
  configuration: backport async backing parameters from the feature branch (#6961)
  Histogram support in runtime metrics (#6935)
  Bump openssl from 0.10.38 to 0.10.48 (#6955)
  Proxy for Nomination Pools (#6846)
  Nomination Pools migration v5: RewardPool fix (#6957)
  bump timestamp script to v0.2 (#6954)
  Subsystem channel tweaks (#6905)
  Companion for #13683 (#6944)
  inherent disputes: remove per block initializer and disputes timeout event (#6937)

.gitlab-ci.yml

@@ -202,7 +202,7 @@ include:
   - scripts/ci/gitlab/pipeline/zombienet.yml
   # timestamp handler
   - project: parity/infrastructure/ci_cd/shared
-    ref: v0.1
+    ref: v0.2
     file: /common/timestamp.yml
 
 #### stage:                        .post

node/core/backing/src/tests.rs

@@ -34,7 +34,7 @@ use polkadot_primitives::{
 	CandidateDescriptor, CollatorId, GroupRotationInfo, HeadData, PersistedValidationData,
 	PvfExecTimeoutKind, ScheduledCore,
 };
-use sp_application_crypto::AppKey;
+use sp_application_crypto::AppCrypto;
 use sp_keyring::Sr25519Keyring;
 use sp_keystore::Keystore;
 use sp_tracing as _;

node/core/dispute-coordinator/src/tests.rs

@@ -45,7 +45,7 @@ use polkadot_node_subsystem::{
 
 use polkadot_node_subsystem_util::TimeoutExt;
 use sc_keystore::LocalKeystore;
-use sp_application_crypto::AppKey;
+use sp_application_crypto::AppCrypto;
 use sp_core::{sr25519::Pair, testing::TaskExecutor, Pair as PairT};
 use sp_keyring::Sr25519Keyring;
 use sp_keystore::{Keystore, KeystorePtr};
@@ -562,18 +562,18 @@ impl TestState {
 		let validator_id = self.validators[index.0 as usize].public();
 
 		let payload = ApprovalVote(candidate_hash).signing_payload(session);
-		let signature =
-			Keystore::sign_with(&*keystore, ValidatorId::ID, &validator_id.into(), &payload[..])
-				.ok()
-				.flatten()
-				.unwrap();
+		let signature = keystore
+			.sr25519_sign(ValidatorId::ID, &validator_id, &payload)
+			.ok()
+			.flatten()
+			.unwrap();
 
 		SignedDisputeStatement::new_unchecked_from_trusted_source(
 			DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking),
 			candidate_hash,
 			session,
 			validator_id.into(),
-			signature.try_into().unwrap(),
+			signature.into(),
 		)
 	}
 

node/core/provisioner/src/tests.rs

@@ -57,7 +57,7 @@ pub fn scheduled_core(id: u32) -> ScheduledCore {
 mod select_availability_bitfields {
 	use super::{super::*, default_bitvec, occupied_core};
 	use polkadot_primitives::{ScheduledCore, SigningContext, ValidatorId, ValidatorIndex};
-	use sp_application_crypto::AppKey;
+	use sp_application_crypto::AppCrypto;
 	use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
 	use std::sync::Arc;
 

node/core/pvf-checker/src/tests.rs

@@ -29,7 +29,7 @@ use polkadot_primitives::{
 	BlockNumber, Hash, Header, PvfCheckStatement, SessionIndex, ValidationCode, ValidationCodeHash,
 	ValidatorId,
 };
-use sp_application_crypto::AppKey;
+use sp_application_crypto::AppCrypto;
 use sp_core::testing::TaskExecutor;
 use sp_keyring::Sr25519Keyring;
 use sp_keystore::Keystore;

node/metrics/src/runtime/mod.rs

@@ -20,30 +20,33 @@
 //! tracing support. This requires that the custom profiler (`TraceHandler`) to be
 //! registered in substrate via a `logger_hook()`. Events emitted from runtime are
 //! then captured/processed by the `TraceHandler` implementation.
+//!
+//! Don't add logs in this file because it gets executed before the logger is
+//! initialized and they won't be delivered. Add println! statements if you need
+//! to debug this code.
 
 #![cfg(feature = "runtime-metrics")]
 
 use codec::Decode;
 use primitives::{
-	metric_definitions::{CounterDefinition, CounterVecDefinition},
+	metric_definitions::{CounterDefinition, CounterVecDefinition, HistogramDefinition},
 	RuntimeMetricLabelValues, RuntimeMetricOp, RuntimeMetricUpdate,
 };
 use std::{
 	collections::hash_map::HashMap,
 	sync::{Arc, Mutex, MutexGuard},
 };
 use substrate_prometheus_endpoint::{
-	register, Counter, CounterVec, Opts, PrometheusError, Registry, U64,
+	register, Counter, CounterVec, Histogram, HistogramOpts, Opts, PrometheusError, Registry, U64,
 };
 mod parachain;
 
-const LOG_TARGET: &'static str = "metrics::runtime";
-
 /// Holds the registered Prometheus metric collections.
 #[derive(Clone, Default)]
 pub struct Metrics {
 	counter_vecs: Arc<Mutex<HashMap<String, CounterVec<U64>>>>,
 	counters: Arc<Mutex<HashMap<String, Counter<U64>>>>,
+	histograms: Arc<Mutex<HashMap<String, Histogram>>>,
 }
 
 /// Runtime metrics wrapper.
@@ -80,7 +83,20 @@ impl RuntimeMetricsProvider {
 		})
 	}
 
-	/// Increment a counter with labels by a value.
+	/// Register a histogram metric
+	pub fn register_histogram(&self, hist: HistogramDefinition) {
+		self.with_histograms_lock_held(|mut hashmap| {
+			hashmap.entry(hist.name.to_owned()).or_insert(register(
+				Histogram::with_opts(
+					HistogramOpts::new(hist.name, hist.description).buckets(hist.buckets.to_vec()),
+				)?,
+				&self.0,
+			)?);
+			return Ok(())
+		})
+	}
+
+	/// Increment a counter with labels by a value
 	pub fn inc_counter_vec_by(&self, name: &str, value: u64, labels: &RuntimeMetricLabelValues) {
 		self.with_counter_vecs_lock_held(|mut hashmap| {
 			hashmap.entry(name.to_owned()).and_modify(|counter_vec| {
@@ -101,28 +117,35 @@ impl RuntimeMetricsProvider {
 		})
 	}
 
+	/// Observe a histogram. `value` should be in `ns`.
+	pub fn observe_histogram(&self, name: &str, value: u128) {
+		self.with_histograms_lock_held(|mut hashmap| {
+			hashmap
+				.entry(name.to_owned())
+				.and_modify(|histogram| histogram.observe(value as f64 / 1_000_000_000.0)); // ns to sec
+			Ok(())
+		})
+	}
+
 	fn with_counters_lock_held<F>(&self, do_something: F)
 	where
 		F: FnOnce(MutexGuard<'_, HashMap<String, Counter<U64>>>) -> Result<(), PrometheusError>,
 	{
-		let _ = self.1.counters.lock().map(do_something).or_else(|error| {
-			gum::error!(target: LOG_TARGET, "Cannot acquire the counter hashmap lock: {:?}", error);
-			Err(error)
-		});
+		let _ = self.1.counters.lock().map(do_something).or_else(|error| Err(error));
 	}
 
 	fn with_counter_vecs_lock_held<F>(&self, do_something: F)
 	where
 		F: FnOnce(MutexGuard<'_, HashMap<String, CounterVec<U64>>>) -> Result<(), PrometheusError>,
 	{
-		let _ = self.1.counter_vecs.lock().map(do_something).or_else(|error| {
-			gum::error!(
-				target: LOG_TARGET,
-				"Cannot acquire the countervec hashmap lock: {:?}",
-				error
-			);
-			Err(error)
-		});
+		let _ = self.1.counter_vecs.lock().map(do_something).or_else(|error| Err(error));
+	}
+
+	fn with_histograms_lock_held<F>(&self, do_something: F)
+	where
+		F: FnOnce(MutexGuard<'_, HashMap<String, Histogram>>) -> Result<(), PrometheusError>,
+	{
+		let _ = self.1.histograms.lock().map(do_something).or_else(|error| Err(error));
 	}
 }
 
@@ -149,8 +172,8 @@ impl sc_tracing::TraceHandler for RuntimeMetricsProvider {
 				Ok(update_op) => {
 					self.parse_metric_update(update_op);
 				},
-				Err(e) => {
-					gum::error!(target: LOG_TARGET, "TraceEvent decode failed: {:?}", e);
+				Err(_) => {
+					// do nothing
 				},
 			}
 		}
@@ -165,6 +188,8 @@ impl RuntimeMetricsProvider {
 				self.inc_counter_vec_by(update.metric_name(), value, labels),
 			RuntimeMetricOp::IncrementCounter(value) =>
 				self.inc_counter_by(update.metric_name(), value),
+			RuntimeMetricOp::ObserveHistogram(value) =>
+				self.observe_histogram(update.metric_name(), value),
 		}
 	}
 
@@ -191,7 +216,6 @@ impl RuntimeMetricsProvider {
 pub fn logger_hook() -> impl FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration) -> () {
 	|logger_builder, config| {
 		if config.prometheus_registry().is_none() {
-			gum::debug!(target: LOG_TARGET, "Prometheus registry is not configured.",);
 			return
 		}
 		let registry = config.prometheus_registry().cloned().unwrap();

node/metrics/src/runtime/parachain.rs

@@ -22,7 +22,7 @@ use primitives::metric_definitions::{
 	PARACHAIN_CREATE_INHERENT_BITFIELDS_SIGNATURE_CHECKS,
 	PARACHAIN_INHERENT_DATA_BITFIELDS_PROCESSED, PARACHAIN_INHERENT_DATA_CANDIDATES_PROCESSED,
 	PARACHAIN_INHERENT_DATA_DISPUTE_SETS_INCLUDED, PARACHAIN_INHERENT_DATA_DISPUTE_SETS_PROCESSED,
-	PARACHAIN_INHERENT_DATA_WEIGHT,
+	PARACHAIN_INHERENT_DATA_WEIGHT, PARACHAIN_VERIFY_DISPUTE_SIGNATURE,
 };
 
 /// Register the parachain runtime metrics.
@@ -35,4 +35,5 @@ pub fn register_metrics(runtime_metrics_provider: &RuntimeMetricsProvider) {
 	runtime_metrics_provider.register_countervec(PARACHAIN_INHERENT_DATA_CANDIDATES_PROCESSED);
 	runtime_metrics_provider
 		.register_countervec(PARACHAIN_CREATE_INHERENT_BITFIELDS_SIGNATURE_CHECKS);
+	runtime_metrics_provider.register_histogram(PARACHAIN_VERIFY_DISPUTE_SIGNATURE);
 }

node/network/bitfield-distribution/src/tests.rs

@@ -33,7 +33,7 @@ use polkadot_node_subsystem_test_helpers::make_subsystem_context;
 use polkadot_node_subsystem_util::TimeoutExt;
 use polkadot_primitives::{AvailabilityBitfield, Signed, ValidatorIndex};
 use rand_chacha::ChaCha12Rng;
-use sp_application_crypto::AppKey;
+use sp_application_crypto::AppCrypto;
 use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair;
 use sp_core::Pair as PairT;
 use sp_keyring::Sr25519Keyring;

node/network/bridge/src/rx/mod.rs

@@ -652,7 +652,7 @@ where
 	)
 	.remote_handle();
 
-	ctx.spawn("network-bridge-in-network-worker", Box::pin(task))?;
+	ctx.spawn_blocking("network-bridge-in-network-worker", Box::pin(task))?;
 	futures::pin_mut!(network_event_handler);
 
 	let orchestra_signal_handler = run_incoming_orchestra_signals(

node/network/dispute-distribution/src/tests/mock.rs

@@ -28,7 +28,7 @@ use lazy_static::lazy_static;
 
 use polkadot_node_network_protocol::{authority_discovery::AuthorityDiscovery, PeerId};
 use sc_keystore::LocalKeystore;
-use sp_application_crypto::AppKey;
+use sp_application_crypto::AppCrypto;
 use sp_keyring::Sr25519Keyring;
 use sp_keystore::{Keystore, KeystorePtr};
 

node/network/gossip-support/src/lib.rs

@@ -36,7 +36,7 @@ use rand::{seq::SliceRandom as _, SeedableRng};
 use rand_chacha::ChaCha20Rng;
 
 use sc_network::Multiaddr;
-use sp_application_crypto::{AppKey, ByteArray};
+use sp_application_crypto::{AppCrypto, ByteArray};
 use sp_keystore::{Keystore, KeystorePtr};
 
 use polkadot_node_network_protocol::{

node/network/statement-distribution/src/tests.rs

@@ -42,7 +42,7 @@ use polkadot_primitives_test_helpers::{
 	dummy_committed_candidate_receipt, dummy_hash, AlwaysZeroRng,
 };
 use sc_keystore::LocalKeystore;
-use sp_application_crypto::{sr25519::Pair, AppKey, Pair as TraitPair};
+use sp_application_crypto::{sr25519::Pair, AppCrypto, Pair as TraitPair};
 use sp_authority_discovery::AuthorityPair;
 use sp_keyring::Sr25519Keyring;
 use sp_keystore::{Keystore, KeystorePtr};

node/overseer/Cargo.toml

@@ -15,7 +15,7 @@ polkadot-node-primitives = { path = "../primitives" }
 polkadot-node-subsystem-types = { path = "../subsystem-types" }
 polkadot-node-metrics = { path = "../metrics" }
 polkadot-primitives = { path = "../../primitives" }
-orchestra = "0.0.4"
+orchestra = "0.0.5"
 gum = { package = "tracing-gum", path = "../gum" }
 lru = "0.9"
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }

node/overseer/src/lib.rs

@@ -529,7 +529,7 @@ pub struct Overseer<SupportsParachains> {
 	])]
 	availability_store: AvailabilityStore,
 
-	#[subsystem(NetworkBridgeRxMessage, sends: [
+	#[subsystem(blocking, NetworkBridgeRxMessage, sends: [
 		BitfieldDistributionMessage,
 		StatementDistributionMessage,
 		ApprovalDistributionMessage,
@@ -540,7 +540,7 @@ pub struct Overseer<SupportsParachains> {
 	])]
 	network_bridge_rx: NetworkBridgeRx,
 
-	#[subsystem(NetworkBridgeTxMessage, sends: [])]
+	#[subsystem(blocking, NetworkBridgeTxMessage, sends: [])]
 	network_bridge_tx: NetworkBridgeTx,
 
 	#[subsystem(blocking, ChainApiMessage, sends: [])]
@@ -559,7 +559,7 @@ pub struct Overseer<SupportsParachains> {
 	])]
 	collator_protocol: CollatorProtocol,
 
-	#[subsystem(ApprovalDistributionMessage, sends: [
+	#[subsystem(blocking, message_capacity: 64000, ApprovalDistributionMessage, sends: [
 		NetworkBridgeTxMessage,
 		ApprovalVotingMessage,
 	])]
@@ -584,7 +584,7 @@ pub struct Overseer<SupportsParachains> {
 	])]
 	gossip_support: GossipSupport,
 
-	#[subsystem(blocking, DisputeCoordinatorMessage, sends: [
+	#[subsystem(blocking, message_capacity: 32000, DisputeCoordinatorMessage, sends: [
 		RuntimeApiMessage,
 		ChainApiMessage,
 		DisputeDistributionMessage,

node/primitives/src/disputes/mod.rs

@@ -21,8 +21,8 @@ use std::collections::{
 
 use parity_scale_codec::{Decode, Encode};
 
-use sp_application_crypto::AppKey;
-use sp_keystore::{Error as KeystoreError, Keystore, KeystorePtr};
+use sp_application_crypto::AppCrypto;
+use sp_keystore::{Error as KeystoreError, KeystorePtr};
 
 use super::{Statement, UncheckedSignedFullStatement};
 use polkadot_primitives::{
@@ -221,26 +221,16 @@ impl SignedDisputeStatement {
 		};
 
 		let data = dispute_statement.payload_data(candidate_hash, session_index);
-		let signature = Keystore::sign_with(
-			&**keystore,
-			ValidatorId::ID,
-			&validator_public.clone().into(),
-			&data,
-		)?;
-
-		let signature = match signature {
-			Some(sig) =>
-				sig.try_into().map_err(|_| KeystoreError::KeyNotSupported(ValidatorId::ID))?,
-			None => return Ok(None),
-		};
-
-		Ok(Some(Self {
-			dispute_statement,
-			candidate_hash,
-			validator_public,
-			validator_signature: signature,
-			session_index,
-		}))
+		let signature = keystore
+			.sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data)?
+			.map(|sig| Self {
+				dispute_statement,
+				candidate_hash,
+				validator_public,
+				validator_signature: sig.into(),
+				session_index,
+			});
+		Ok(signature)
 	}
 
 	/// Access the underlying dispute statement

node/subsystem-test-helpers/src/mock.rs

@@ -17,7 +17,7 @@
 use std::sync::Arc;
 
 use sc_keystore::LocalKeystore;
-use sp_application_crypto::AppKey;
+use sp_application_crypto::AppCrypto;
 use sp_keyring::Sr25519Keyring;
 use sp_keystore::{Keystore, KeystorePtr};
 

node/subsystem-types/Cargo.toml

@@ -13,7 +13,7 @@ polkadot-node-primitives = { path = "../primitives" }
 polkadot-node-network-protocol = { path = "../network/protocol" }
 polkadot-statement-table = { path = "../../statement-table" }
 polkadot-node-jaeger = { path = "../jaeger" }
-orchestra = "0.0.4"
+orchestra = "0.0.5"
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }