Update from upstream

Cargo.toml

@@ -121,7 +121,7 @@ reqwest = "0.11"
 rlp = "0.5"
 rocksdb = "0.21.0"
 rustc_version = "0.4.0"
-secp256k1 = "0.27.0"
+secp256k1 = { version = "0.27.0", features = ["recovery", "global-context"] }
 semver = "1"
 sentry = "0.31"
 serde = "1"

core/bin/external_node/Cargo.toml

@@ -15,6 +15,7 @@ zksync_core.workspace = true
 zksync_dal.workspace = true
 zksync_db_connection.workspace = true
 zksync_config.workspace = true
+zksync_protobuf_config.workspace = true
 zksync_eth_client.workspace = true
 zksync_storage.workspace = true
 zksync_utils.workspace = true

core/bin/external_node/src/config/mod.rs

@@ -8,15 +8,21 @@ use anyhow::Context;
 use serde::Deserialize;
 use url::Url;
 use zksync_basic_types::{Address, L1ChainId, L2ChainId};
-use zksync_config::{configs::chain::L1BatchCommitDataGeneratorMode, ObjectStoreConfig};
+use zksync_config::{
+    configs::{
+        chain::L1BatchCommitDataGeneratorMode,
+        consensus::{ConsensusConfig, ConsensusSecrets},
+    },
+    ObjectStoreConfig,
+};
 use zksync_core::{
     api_server::{
         tx_sender::TxSenderConfig,
         web3::{state::InternalApiConfig, Namespace},
     },
-    consensus,
-    temp_config_store::decode_yaml,
+    temp_config_store::decode_yaml_repr,
 };
+use zksync_protobuf_config::proto;
 use zksync_types::{api::BridgeAddresses, fee_model::FeeParams};
 use zksync_web3_decl::{
     client::L2Client,
@@ -564,20 +570,24 @@ impl PostgresConfig {
     }
 }
 
-pub(crate) fn read_consensus_secrets() -> anyhow::Result<Option<consensus::Secrets>> {
+pub(crate) fn read_consensus_secrets() -> anyhow::Result<Option<ConsensusSecrets>> {
     let Ok(path) = std::env::var("EN_CONSENSUS_SECRETS_PATH") else {
         return Ok(None);
     };
     let cfg = std::fs::read_to_string(&path).context(path)?;
-    Ok(Some(decode_yaml(&cfg).context("failed decoding YAML")?))
+    Ok(Some(
+        decode_yaml_repr::<proto::consensus::Secrets>(&cfg).context("failed decoding YAML")?,
+    ))
 }
 
-pub(crate) fn read_consensus_config() -> anyhow::Result<Option<consensus::Config>> {
+pub(crate) fn read_consensus_config() -> anyhow::Result<Option<ConsensusConfig>> {
     let Ok(path) = std::env::var("EN_CONSENSUS_CONFIG_PATH") else {
         return Ok(None);
     };
     let cfg = std::fs::read_to_string(&path).context(path)?;
-    Ok(Some(decode_yaml(&cfg).context("failed decoding YAML")?))
+    Ok(Some(
+        decode_yaml_repr::<proto::consensus::Config>(&cfg).context("failed decoding YAML")?,
+    ))
 }
 
 /// Configuration for snapshot recovery. Loaded optionally, only if the corresponding command-line argument
@@ -604,7 +614,7 @@ pub(crate) struct ExternalNodeConfig {
     pub postgres: PostgresConfig,
     pub optional: OptionalENConfig,
     pub remote: RemoteENConfig,
-    pub consensus: Option<consensus::Config>,
+    pub consensus: Option<ConsensusConfig>,
     pub api_component: ApiComponentConfig,
     pub tree_component: TreeComponentConfig,
 }

core/bin/zksync_server/src/config.rs

@@ -1,20 +1,26 @@
 use anyhow::Context as _;
-use zksync_core::{consensus, temp_config_store::decode_yaml};
+use zksync_config::configs::consensus::{ConsensusConfig, ConsensusSecrets};
+use zksync_core::temp_config_store::decode_yaml_repr;
+use zksync_protobuf_config::proto;
 
-pub(crate) fn read_consensus_secrets() -> anyhow::Result<Option<consensus::Secrets>> {
+pub(crate) fn read_consensus_secrets() -> anyhow::Result<Option<ConsensusSecrets>> {
     // Read public config.
     let Ok(path) = std::env::var("CONSENSUS_SECRETS_PATH") else {
         return Ok(None);
     };
     let secrets = std::fs::read_to_string(&path).context(path)?;
-    Ok(Some(decode_yaml(&secrets).context("failed decoding YAML")?))
+    Ok(Some(
+        decode_yaml_repr::<proto::consensus::Secrets>(&secrets).context("failed decoding YAML")?,
+    ))
 }
 
-pub(crate) fn read_consensus_config() -> anyhow::Result<Option<consensus::Config>> {
+pub(crate) fn read_consensus_config() -> anyhow::Result<Option<ConsensusConfig>> {
     // Read public config.
     let Ok(path) = std::env::var("CONSENSUS_CONFIG_PATH") else {
         return Ok(None);
     };
     let cfg = std::fs::read_to_string(&path).context(path)?;
-    Ok(Some(decode_yaml(&cfg).context("failed decoding YAML")?))
+    Ok(Some(
+        decode_yaml_repr::<proto::consensus::Config>(&cfg).context("failed decoding YAML")?,
+    ))
 }

core/lib/config/src/configs/consensus.rs

@@ -0,0 +1,69 @@
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    fmt,
+};
+
+/// Public key of the validator (consensus participant) of the form "validator:public:<signature scheme>:<hex encoded key material>"
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct ValidatorPublicKey(pub String);
+
+// Secret key of the validator (consensus participant) of the form "validator:secret:<signature scheme>:<hex encoded key material>"
+#[derive(PartialEq)]
+pub struct ValidatorSecretKey(pub String);
+
+/// Public key of the node (gossip network participant) of the form "node:public:<signature scheme>:<hex encoded key material>"
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct NodePublicKey(pub String);
+
+// Secret key of the node (gossip network participant) of the form "node:secret:<signature scheme>:<hex encoded key material>"
+#[derive(PartialEq)]
+pub struct NodeSecretKey(pub String);
+
+impl fmt::Debug for ValidatorSecretKey {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.write_str("<redacted>")
+    }
+}
+
+impl fmt::Debug for NodeSecretKey {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.write_str("<redacted>")
+    }
+}
+
+/// Network address in the `<domain/ip>:port` format.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Host(pub String);
+
+/// Config (shared between main node and external node).
+#[derive(Clone, Debug, PartialEq)]
+pub struct ConsensusConfig {
+    /// Local socket address to listen for the incoming connections.
+    pub server_addr: std::net::SocketAddr,
+    /// Public address of this node (should forward to `server_addr`)
+    /// that will be advertised to peers, so that they can connect to this
+    /// node.
+    pub public_addr: Host,
+
+    /// Validators participating in consensus.
+    pub validators: BTreeSet<ValidatorPublicKey>,
+
+    /// Maximal allowed size of the payload in bytes.
+    pub max_payload_size: usize,
+
+    /// Limit on the number of inbound connections outside
+    /// of the `static_inbound` set.
+    pub gossip_dynamic_inbound_limit: usize,
+    /// Inbound gossip connections that should be unconditionally accepted.
+    pub gossip_static_inbound: BTreeSet<NodePublicKey>,
+    /// Outbound gossip connections that the node should actively try to
+    /// establish and maintain.
+    pub gossip_static_outbound: BTreeMap<NodePublicKey, Host>,
+}
+
+/// Secrets need for consensus.
+#[derive(Debug, PartialEq)]
+pub struct ConsensusSecrets {
+    pub validator_key: Option<ValidatorSecretKey>,
+    pub node_key: Option<NodeSecretKey>,
+}

core/lib/config/src/configs/mod.rs

@@ -22,6 +22,7 @@ pub use self::{
 
 pub mod api;
 pub mod chain;
+pub mod consensus;
 pub mod contract_verifier;
 pub mod contracts;
 pub mod database;

core/lib/config/src/testonly.rs

@@ -673,3 +673,37 @@ impl Distribution<configs::SharedBridge> for EncodeDist {
         }
     }
 }
+
+impl Distribution<configs::consensus::ConsensusConfig> for EncodeDist {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::consensus::ConsensusConfig {
+        use configs::consensus::{ConsensusConfig, Host, NodePublicKey, ValidatorPublicKey};
+        ConsensusConfig {
+            server_addr: self.sample(rng),
+            public_addr: Host(self.sample(rng)),
+            validators: self
+                .sample_range(rng)
+                .map(|_| ValidatorPublicKey(self.sample(rng)))
+                .collect(),
+            max_payload_size: self.sample(rng),
+            gossip_dynamic_inbound_limit: self.sample(rng),
+            gossip_static_inbound: self
+                .sample_range(rng)
+                .map(|_| NodePublicKey(self.sample(rng)))
+                .collect(),
+            gossip_static_outbound: self
+                .sample_range(rng)
+                .map(|_| (NodePublicKey(self.sample(rng)), Host(self.sample(rng))))
+                .collect(),
+        }
+    }
+}
+
+impl Distribution<configs::consensus::ConsensusSecrets> for EncodeDist {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::consensus::ConsensusSecrets {
+        use configs::consensus::{ConsensusSecrets, NodeSecretKey, ValidatorSecretKey};
+        ConsensusSecrets {
+            validator_key: self.sample_opt(|| ValidatorSecretKey(self.sample(rng))),
+            node_key: self.sample_opt(|| NodeSecretKey(self.sample(rng))),
+        }
+    }
+}

core/lib/protobuf_config/build.rs

@@ -2,7 +2,7 @@
 fn main() {
     zksync_protobuf_build::Config {
         input_root: "src/proto".into(),
-        proto_root: "zksync/config".into(),
+        proto_root: "zksync".into(),
         dependencies: vec![],
         protobuf_crate: "::zksync_protobuf".parse().unwrap(),
         is_public: true,

core/lib/protobuf_config/src/consensus.rs

@@ -0,0 +1,92 @@
+use anyhow::Context as _;
+use zksync_config::configs::consensus::{
+    ConsensusConfig, ConsensusSecrets, Host, NodePublicKey, NodeSecretKey, ValidatorPublicKey,
+    ValidatorSecretKey,
+};
+use zksync_protobuf::{repr::ProtoRepr, required};
+
+use crate::proto::consensus as proto;
+
+impl ProtoRepr for proto::Config {
+    type Type = ConsensusConfig;
+    fn read(&self) -> anyhow::Result<Self::Type> {
+        let read_addr = |e: &proto::NodeAddr| {
+            let key = NodePublicKey(required(&e.key).context("key")?.clone());
+            let addr = Host(required(&e.addr).context("addr")?.clone());
+            anyhow::Ok((key, addr))
+        };
+        Ok(Self::Type {
+            server_addr: required(&self.server_addr)
+                .and_then(|x| Ok(x.parse()?))
+                .context("server_addr")?,
+            public_addr: Host(required(&self.public_addr).context("public_addr")?.clone()),
+            validators: self
+                .validators
+                .iter()
+                .map(|x| ValidatorPublicKey(x.clone()))
+                .collect(),
+            max_payload_size: required(&self.max_payload_size)
+                .and_then(|x| Ok((*x).try_into()?))
+                .context("max_payload_size")?,
+            gossip_dynamic_inbound_limit: required(&self.gossip_dynamic_inbound_limit)
+                .and_then(|x| Ok((*x).try_into()?))
+                .context("gossip_dynamic_inbound_limit")?,
+            gossip_static_inbound: self
+                .gossip_static_inbound
+                .iter()
+                .map(|x| NodePublicKey(x.clone()))
+                .collect(),
+            gossip_static_outbound: self
+                .gossip_static_outbound
+                .iter()
+                .enumerate()
+                .map(|(i, e)| read_addr(e).context(i))
+                .collect::<Result<_, _>>()?,
+        })
+    }
+
+    fn build(this: &Self::Type) -> Self {
+        Self {
+            server_addr: Some(this.server_addr.to_string()),
+            public_addr: Some(this.public_addr.0.clone()),
+            validators: this.validators.iter().map(|x| x.0.clone()).collect(),
+            max_payload_size: Some(this.max_payload_size.try_into().unwrap()),
+            gossip_dynamic_inbound_limit: Some(
+                this.gossip_dynamic_inbound_limit.try_into().unwrap(),
+            ),
+            gossip_static_inbound: this
+                .gossip_static_inbound
+                .iter()
+                .map(|x| x.0.clone())
+                .collect(),
+            gossip_static_outbound: this
+                .gossip_static_outbound
+                .iter()
+                .map(|x| proto::NodeAddr {
+                    key: Some(x.0 .0.clone()),
+                    addr: Some(x.1 .0.clone()),
+                })
+                .collect(),
+        }
+    }
+}
+
+impl ProtoRepr for proto::Secrets {
+    type Type = ConsensusSecrets;
+    fn read(&self) -> anyhow::Result<Self::Type> {
+        Ok(Self::Type {
+            validator_key: self
+                .validator_key
+                .as_ref()
+                .map(|x| ValidatorSecretKey(x.clone())),
+            node_key: self.node_key.as_ref().map(|x| NodeSecretKey(x.clone())),
+        })
+    }
+
+    fn build(this: &Self::Type) -> Self {
+        Self {
+            validator_key: this.validator_key.as_ref().map(|x| x.0.clone()),
+            node_key: this.node_key.as_ref().map(|x| x.0.clone()),
+        }
+    }
+}

core/lib/protobuf_config/src/lib.rs

@@ -7,6 +7,7 @@
 mod api;
 mod chain;
 mod circuit_breaker;
+mod consensus;
 mod contract_verifier;
 mod contracts;
 mod database;
@@ -39,6 +40,6 @@ fn parse_h160(bytes: &str) -> anyhow::Result<H160> {
     Ok(H160::from_str(bytes)?)
 }
 
-fn read_optional_repr<P: ProtoRepr>(field: &Option<P>) -> anyhow::Result<Option<P::Type>> {
+pub fn read_optional_repr<P: ProtoRepr>(field: &Option<P>) -> anyhow::Result<Option<P::Type>> {
     field.as_ref().map(|x| x.read()).transpose()
 }

core/lib/zksync_core/src/proto/consensus.proto → core/lib/protobuf_config/src/proto/core/consensus.proto

@@ -7,6 +7,8 @@
 //   (note that opening IPv6 ports may not work depending on the VM capabilities).
 //   examples: "203.0.113.7:3456", "[2001:DB8::1]:4567"
 //
+// Host - network address in the `<domain/ip>:port` format.
+// 
 // ValidatorPublicKey - public key of the validator (consensus participant) of the form "validator:public:<signature scheme>:<hex encoded key material>"
 //   Currently only bn254 signature scheme is supported for validators.
 //   example: "validator:public:bn254:4b0c4697f0a35eab30f63684ae4611f3c1d631eecfd97237e2345a9b3d0c472dbb16c49b793beceaab0cdd89cda6ff1099bd1aaf1ad6cabde9a15793cc09b407"
@@ -40,7 +42,7 @@ message Config {
 
   // Public IP:port to advertise, should forward to server_addr.
   // Can be `127.0.0.1:<port>` for local tests.
-  optional string public_addr = 2; // required; IpAddr
+  optional string public_addr = 2; // required; Host
 
   // Public keys of all validators.
   // Currently it has to be a singleton with a public key corresponding to secret key in CONSENSUS_VALIDATOR_KEY env var.
@@ -65,5 +67,3 @@ message Secrets {
   optional string validator_key = 1; // required for validator nodes; ValidatorSecretKey
   optional string node_key = 2; // required for any node; NodeSecretKey
 }
-
-

core/lib/protobuf_config/src/proto/mod.rs

@@ -1,3 +1,5 @@
 #![allow(warnings)]
 
+pub use self::{config::*, core::*};
+
 include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs"));