From bcda877b80e8e80841cd037ad519bfb1da0c9a61 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 14:39:09 -0700 Subject: [PATCH 01/21] Working local testing --- Cargo.lock | 1 + integration-tests/Cargo.toml | 3 +- integration-tests/build.rs | 3 + integration-tests/src/containers.rs | 203 +++++++++++- integration-tests/src/util.rs | 52 +++ integration-tests/tests/lib.rs | 24 +- mpc-recovery/build.rs | 3 + mpc-recovery/src/lib.rs | 469 +++++++++++++++++++++++++++- mpc-recovery/src/main.rs | 357 +-------------------- mpc-recovery/src/relayer/mod.rs | 2 +- 10 files changed, 744 insertions(+), 373 deletions(-) create mode 100644 integration-tests/build.rs create mode 100644 mpc-recovery/build.rs diff --git a/Cargo.lock b/Cargo.lock index 5e51173a9..185a8291c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2985,6 +2985,7 @@ version = "0.1.0" dependencies = [ "aes-gcm", "anyhow", + "async-process", "bollard", "clap 4.4.2", "curv-kzen", diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 278514309..22e1b16b6 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -7,9 +7,10 @@ publish = false [dependencies] aes-gcm = "0.10" anyhow = { version = "1.0", features = ["backtrace"] } +async-process = "1" bollard = "0.11" clap = { version = "4.2", features = ["derive", "env"] } -ed25519-dalek = {version = "1.0.1", features = ["serde"]} +ed25519-dalek = { version = "1.0.1", features = ["serde"] } futures = "0.3" hex = "0.4.3" hyper = { version = "0.14", features = ["full"] } diff --git a/integration-tests/build.rs b/integration-tests/build.rs new file mode 100644 index 000000000..8435ff092 --- /dev/null +++ b/integration-tests/build.rs @@ -0,0 +1,3 @@ +// HACK: need this build script so that env var OUT_DIR gets set: +// https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates +fn main() {} diff --git a/integration-tests/src/containers.rs b/integration-tests/src/containers.rs index 3a0c7db0f..51ef68fbb 100644 --- a/integration-tests/src/containers.rs +++ b/integration-tests/src/containers.rs @@ -1,7 +1,8 @@ #![allow(clippy::too_many_arguments)] use aes_gcm::{Aes256Gcm, KeyInit}; -use anyhow::{anyhow, Ok}; +use anyhow::{anyhow, Context, Ok}; +use async_process::{Child, Command, Stdio}; use bollard::{container::LogsOptions, network::CreateNetworkOptions, service::Ipam, Docker}; use ed25519_dalek::ed25519::signature::digest::{consts::U32, generic_array::GenericArray}; use ed25519_dalek::{PublicKey as PublicKeyEd25519, Verifier}; @@ -447,6 +448,32 @@ pub struct SignerNode<'a> { gcp_datastore_local_url: String, } +pub struct SignerNodeLocal { + pub address: String, + node_id: usize, + sk_share: ExpandedKeyPair, + cipher_key: GenericArray, + gcp_project_id: String, + gcp_datastore_url: String, + + // process held so it's not dropped. Once dropped, process will be killed. + #[allow(unused)] + process: Child, +} + +impl SignerNodeLocal { + pub fn api(&self) -> SignerNodeApi { + SignerNodeApi { + address: self.address.clone(), + node_id: self.node_id, + sk_share: self.sk_share.clone(), + cipher_key: self.cipher_key, + gcp_project_id: self.gcp_project_id.clone(), + gcp_datastore_local_url: self.gcp_datastore_url.clone(), + } + } +} + pub struct SignerNodeApi { pub address: String, pub node_id: usize, @@ -533,6 +560,83 @@ impl<'a> SignerNode<'a> { }) } + pub async fn run_local( + web_port: usize, + node_id: u64, + sk_share: &ExpandedKeyPair, + cipher_key: &GenericArray, + datastore_url: &str, + gcp_project_id: &str, + firebase_audience_id: &str, + release: bool, + ) -> anyhow::Result { + let executable = util::target_dir() + .context("could not find target dir while running signing node")? + .join(if release { "release" } else { "debug" }) + .join("mpc-recovery"); + let args = vec![ + "start-sign".to_string(), + "--node-id".to_string(), + node_id.to_string(), + "--sk-share".to_string(), + serde_json::to_string(&sk_share)?, + "--cipher-key".to_string(), + hex::encode(cipher_key), + "--web-port".to_string(), + web_port.to_string(), + "--oidc-providers".to_string(), + serde_json::json!([ + { + "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), + "audience": firebase_audience_id, + }, + ]) + .to_string(), + "--gcp-project-id".to_string(), + gcp_project_id.to_string(), + "--gcp-datastore-url".to_string(), + datastore_url.to_string(), + "--test".to_string(), + ]; + + let address = format!("http://localhost:{web_port}"); + let child = Command::new(&executable) + .args(&args) + .envs(std::env::vars()) + .env("RUST_LOG", "mpc_recovery=DEBUG") + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .with_context(|| { + format!( + "failed to run signing node: [node_id={node_id}, {}]", + executable.display() + ) + })?; + + tracing::info!("Signer node is start on {}", address); + loop { + let x: anyhow::Result = util::get(&address).await; + match x { + std::result::Result::Ok(status) if status == StatusCode::OK => break, + _err => {} + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + tracing::info!("Signer node started [node_id={node_id}, {address}]"); + + Ok(SignerNodeLocal { + address, + node_id: node_id as usize, + sk_share: sk_share.clone(), + cipher_key: *cipher_key, + gcp_project_id: gcp_project_id.to_string(), + gcp_datastore_url: datastore_url.to_string(), + process: child, + }) + } + pub fn api(&self) -> SignerNodeApi { SignerNodeApi { address: self.local_address.clone(), @@ -588,6 +692,24 @@ pub struct LeaderNode<'a> { local_address: String, } +pub struct LeaderNodeLocal { + pub address: String, + + // process held so it's not dropped. Once dropped, process will be killed. + #[allow(unused)] + process: Child, +} + +impl LeaderNodeLocal { + pub fn api(&self, near_rpc: &str, relayer: &DelegateActionRelayer) -> LeaderNodeApi { + LeaderNodeApi { + address: self.address.clone(), + client: NearRpcAndRelayerClient::connect(near_rpc), + relayer: relayer.clone(), + } + } +} + pub struct LeaderNodeApi { pub address: String, pub relayer: DelegateActionRelayer, @@ -674,6 +796,85 @@ impl<'a> LeaderNode<'a> { }) } + pub async fn run_local( + web_port: usize, + sign_nodes: Vec, + near_rpc: &str, + relayer_url: &str, + datastore_url: &str, + gcp_project_id: &str, + near_root_account: &AccountId, + account_creator_id: &AccountId, + account_creator_sk: &workspaces::types::SecretKey, + firebase_audience_id: &str, + release: bool, + ) -> anyhow::Result { + tracing::info!("Running leader node..."); + let executable = util::target_dir() + .context("could not find target dir while running leader node")? + .join(if release { "release" } else { "debug" }) + .join("mpc-recovery"); + let mut args = vec![ + "start-leader".to_string(), + "--web-port".to_string(), + web_port.to_string(), + "--near-rpc".to_string(), + near_rpc.to_string(), + "--near-root-account".to_string(), + near_root_account.to_string(), + "--account-creator-id".to_string(), + account_creator_id.to_string(), + "--account-creator-sk".to_string(), + account_creator_sk.to_string(), + "--fast-auth-partners".to_string(), + serde_json::json!([ + { + "oidc_provider": { + "issuer": format!("https://securetoken.google.com/{}", firebase_audience_id), + "audience": firebase_audience_id, + }, + "relayer": { + "url": relayer_url.to_string(), + "api_key": serde_json::Value::Null, + }, + }, + ]).to_string(), + "--gcp-project-id".to_string(), + gcp_project_id.to_string(), + "--gcp-datastore-url".to_string(), + datastore_url.to_string(), + "--test".to_string(), + ]; + for sign_node in sign_nodes { + args.push("--sign-nodes".to_string()); + args.push(sign_node); + } + + let child = Command::new(&executable) + .args(&args) + .envs(std::env::vars()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("failed to run leader node: {}", executable.display()))?; + + let address = format!("http://localhost:{web_port}"); + tracing::info!("Leader node container is starting at {}", address); + loop { + match util::get(&address).await { + std::result::Result::Ok(status) if status == StatusCode::OK => break, + _ => tokio::time::sleep(std::time::Duration::from_secs(1)).await, + } + } + + tracing::info!("Leader node container is running at {address}"); + Ok(LeaderNodeLocal { + address, + process: child, + }) + } + pub fn api(&self, near_rpc: &str, relayer: &DelegateActionRelayer) -> LeaderNodeApi { LeaderNodeApi { address: self.local_address.clone(), diff --git a/integration-tests/src/util.rs b/integration-tests/src/util.rs index d46d07b3a..3a46758dd 100644 --- a/integration-tests/src/util.rs +++ b/integration-tests/src/util.rs @@ -1,6 +1,7 @@ use std::{ fs::{self, File}, io::Write, + path::PathBuf, }; use anyhow::{Context, Ok}; @@ -45,6 +46,26 @@ where Ok((status, response)) } +pub async fn get(uri: U) -> anyhow::Result +where + Uri: TryFrom, + >::Error: Into, +{ + let req = Request::builder() + .method(Method::GET) + .uri(uri) + .header("content-type", "application/json") + .body(Body::empty()) + .context("failed to build the request")?; + + let client = Client::new(); + let response = client + .request(req) + .await + .context("failed to send the request")?; + Ok(response.status()) +} + #[derive(Deserialize, Serialize)] struct KeyFile { account_id: String, @@ -174,3 +195,34 @@ pub fn create_relayer_cofig_file( .expect("Failed to convert config file path to string") .to_string()) } + +/// Request an unused port from the OS. +pub async fn pick_unused_port() -> anyhow::Result { + // Port 0 means the OS gives us an unused port + // Important to use localhost as using 0.0.0.0 leads to users getting brief firewall popups to + // allow inbound connections on MacOS. + let addr = std::net::SocketAddrV4::new(std::net::Ipv4Addr::LOCALHOST, 0); + let listener = tokio::net::TcpListener::bind(addr).await?; + let port = listener.local_addr()?.port(); + Ok(port) +} + +pub fn target_dir() -> Option { + let mut out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap_or_else(|e| { + panic!( + "Failed to get OUT_DIR env variable: {e}.\n\ + Please run `cargo test` from the root of the repository.", + ) + })); + + loop { + if out_dir.ends_with("target") { + return Some(out_dir.to_path_buf()); + } + + match out_dir.parent() { + Some(parent) => out_dir = parent.to_owned(), + None => return None, // We've reached the root directory and didn't find "target" + } + } +} diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index b6e31ad21..6737a8641 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -10,7 +10,7 @@ use mpc_recovery::{ }, GenerateResult, }; -use mpc_recovery_integration_tests::containers; +use mpc_recovery_integration_tests::{containers, util}; use near_primitives::utils::generate_random_string; use workspaces::{network::Sandbox, Worker}; @@ -59,18 +59,16 @@ where let GenerateResult { pk_set, secrets } = mpc_recovery::generate(nodes); let mut signer_node_futures = Vec::new(); for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { - let signer_node = containers::SignerNode::run_signing_node( - &docker_client, - NETWORK, + signer_node_futures.push(containers::SignerNode::run_local( + util::pick_unused_port().await? as usize, i as u64, share, cipher_key, - &datastore.address, &datastore.local_address, GCP_PROJECT_ID, FIREBASE_AUDIENCE_ID, - ); - signer_node_futures.push(signer_node); + true, + )); } let signer_nodes = futures::future::join_all(signer_node_futures) .await @@ -79,18 +77,18 @@ where let signer_urls: &Vec<_> = &signer_nodes.iter().map(|n| n.address.clone()).collect(); let near_root_account = relayer_ctx.worker.root_account()?; - let leader_node = containers::LeaderNode::run( - &docker_client, - NETWORK, + let leader_node = containers::LeaderNode::run_local( + util::pick_unused_port().await? as usize, signer_urls.clone(), - &relayer_ctx.sandbox.address, - &relayer_ctx.relayer.address, - &datastore.address, + &relayer_ctx.sandbox.local_address, + &relayer_ctx.relayer.local_address, + &datastore.local_address, GCP_PROJECT_ID, near_root_account.id(), relayer_ctx.creator_account.id(), relayer_ctx.creator_account.secret_key(), FIREBASE_AUDIENCE_ID, + true, ) .await?; diff --git a/mpc-recovery/build.rs b/mpc-recovery/build.rs new file mode 100644 index 000000000..8435ff092 --- /dev/null +++ b/mpc-recovery/build.rs @@ -0,0 +1,3 @@ +// HACK: need this build script so that env var OUT_DIR gets set: +// https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates +fn main() {} diff --git a/mpc-recovery/src/lib.rs b/mpc-recovery/src/lib.rs index 5aa11f347..995921434 100644 --- a/mpc-recovery/src/lib.rs +++ b/mpc-recovery/src/lib.rs @@ -1,14 +1,25 @@ // TODO: FIXME: Remove this once we have a better way to handle these large errors #![allow(clippy::result_large_err)] +use std::path::PathBuf; + use aes_gcm::aead::consts::U32; use aes_gcm::aead::generic_array::GenericArray; use aes_gcm::aead::OsRng; -use aes_gcm::Aes256Gcm; -use aes_gcm::KeyInit; +use aes_gcm::{Aes256Gcm, KeyInit}; +use clap::Parser; use curv::elliptic::curves::Ed25519; use curv::elliptic::curves::Point; use multi_party_eddsa::protocols::ExpandedKeyPair; +use serde::de::DeserializeOwned; +use tracing_subscriber::EnvFilter; + +use near_primitives::types::AccountId; + +use crate::firewall::allowed::{OidcProviderList, PartnerList}; +use crate::gcp::GcpService; +use crate::oauth::{PagodaFirebaseTokenVerifier, UniversalTokenVerifier}; +use crate::sign_node::migration; pub mod error; pub mod firewall; @@ -52,3 +63,457 @@ pub fn generate(n: usize) -> GenerateResult { secrets: sk_set.into_iter().zip(cipher_keys.into_iter()).collect(), } } + +#[derive(Parser, Debug)] +pub enum Cli { + Generate { + n: usize, + }, + StartLeader { + /// Environment to run in (`dev` or `prod`) + #[arg(long, env("MPC_RECOVERY_ENV"), default_value("dev"))] + env: String, + /// The web port for this server + #[arg(long, env("MPC_RECOVERY_WEB_PORT"))] + web_port: u16, + /// The compute nodes to connect to + #[arg(long, value_parser, num_args = 1.., value_delimiter = ',', env("MPC_RECOVERY_SIGN_NODES"))] + sign_nodes: Vec, + /// NEAR RPC address + #[arg( + long, + env("MPC_RECOVERY_NEAR_RPC"), + default_value("https://rpc.testnet.near.org") + )] + near_rpc: String, + /// NEAR root account that has linkdrop contract deployed on it + #[arg(long, env("MPC_RECOVERY_NEAR_ROOT_ACCOUNT"), default_value("testnet"))] + near_root_account: String, + /// Account creator ID + #[arg(long, env("MPC_RECOVERY_ACCOUNT_CREATOR_ID"))] + account_creator_id: AccountId, + /// TEMPORARY - Account creator ed25519 secret key + #[arg(long, env("MPC_RECOVERY_ACCOUNT_CREATOR_SK"))] + account_creator_sk: Option, + /// JSON list of related items to be used to verify OIDC tokens. + #[arg(long, env("FAST_AUTH_PARTNERS"))] + fast_auth_partners: Option, + /// Filepath to a JSON list of related items to be used to verify OIDC tokens. + #[arg(long, value_parser, env("FAST_AUTH_PARTNERS_FILEPATH"))] + fast_auth_partners_filepath: Option, + /// GCP project ID + #[arg(long, env("MPC_RECOVERY_GCP_PROJECT_ID"))] + gcp_project_id: String, + /// GCP datastore URL + #[arg(long, env("MPC_RECOVERY_GCP_DATASTORE_URL"))] + gcp_datastore_url: Option, + /// Whether to accept test tokens + #[arg(long, env("MPC_RECOVERY_TEST"), default_value("false"))] + test: bool, + }, + StartSign { + /// Environment to run in (`dev` or `prod`) + #[arg(long, env("MPC_RECOVERY_ENV"), default_value("dev"))] + env: String, + /// Node ID + #[arg(long, env("MPC_RECOVERY_NODE_ID"))] + node_id: u64, + /// Cipher key to encrypt stored user credentials, will be pulled from GCP Secret Manager if omitted + #[arg(long, env("MPC_RECOVERY_CIPHER_KEY"))] + cipher_key: Option, + /// Secret key share, will be pulled from GCP Secret Manager if omitted + #[arg(long, env("MPC_RECOVERY_SK_SHARE"))] + sk_share: Option, + /// The web port for this server + #[arg(long, env("MPC_RECOVERY_WEB_PORT"))] + web_port: u16, + /// JSON list of related items to be used to verify OIDC tokens. + #[arg(long, env("OIDC_PROVIDERS"))] + oidc_providers: Option, + /// Filepath to a JSON list of related items to be used to verify OIDC tokens. + #[arg(long, value_parser, env("OIDC_PROVIDERS_FILEPATH"))] + oidc_providers_filepath: Option, + /// GCP project ID + #[arg(long, env("MPC_RECOVERY_GCP_PROJECT_ID"))] + gcp_project_id: String, + /// GCP datastore URL + #[arg(long, env("MPC_RECOVERY_GCP_DATASTORE_URL"))] + gcp_datastore_url: Option, + /// Whether to accept test tokens + #[arg(long, env("MPC_RECOVERY_TEST"), default_value("false"))] + test: bool, + }, + RotateSignNodeCipher { + /// Environment to run in (`dev` or `prod`) + #[arg(long, env("MPC_RECOVERY_ENV"), default_value("dev"))] + env: String, + /// If no `new_env` is specified, the rotation will be done inplace in the current `env`. + #[arg(long, env("MPC_RECOVERY_ROTATE_INPLACE"))] + new_env: Option, + /// Node ID + #[arg(long, env("MPC_RECOVERY_NODE_ID"))] + node_id: u64, + /// Old cipher key, will be pulled from GCP Secret Manager if omitted + #[arg(long, env("MPC_RECOVERY_OLD_CIPHER_KEY"))] + old_cipher_key: Option, + /// The new cipher key to replace each encrypted record with. + #[arg(long, env("MPC_RECOVERY_NEW_CIPHER_KEY"))] + new_cipher_key: Option, + /// GCP project ID + #[arg(long, env("MPC_RECOVERY_GCP_PROJECT_ID"))] + gcp_project_id: String, + /// GCP datastore URL + #[arg(long, env("MPC_RECOVERY_GCP_DATASTORE_URL"))] + gcp_datastore_url: Option, + }, +} + +pub async fn run(cmd: Cli) -> anyhow::Result<()> { + // Install global collector configured based on RUST_LOG env var. + let mut subscriber = tracing_subscriber::fmt() + .with_thread_ids(true) + .with_env_filter(EnvFilter::from_default_env()); + // Check if running in Google Cloud Run: https://cloud.google.com/run/docs/container-contract#services-env-vars + if std::env::var("K_SERVICE").is_ok() { + // Disable colored logging as it messes up Google's log formatting + subscriber = subscriber.with_ansi(false); + } + subscriber.init(); + let _span = tracing::trace_span!("cli").entered(); + + match cmd { + Cli::Generate { n } => { + let GenerateResult { pk_set, secrets } = generate(n); + tracing::info!("Public key set: {}", serde_json::to_string(&pk_set)?); + for (i, (sk_share, cipher_key)) in secrets.iter().enumerate() { + tracing::info!( + "Secret key share {}: {}", + i, + serde_json::to_string(sk_share)? + ); + tracing::info!("Cipher {}: {}", i, hex::encode(cipher_key)); + } + } + Cli::StartLeader { + env, + web_port, + sign_nodes, + near_rpc, + near_root_account, + account_creator_id, + account_creator_sk, + fast_auth_partners: partners, + fast_auth_partners_filepath: partners_filepath, + gcp_project_id, + gcp_datastore_url, + test, + } => { + let gcp_service = + GcpService::new(env.clone(), gcp_project_id, gcp_datastore_url).await?; + let account_creator_sk = + load_account_creator_sk(&gcp_service, &env, account_creator_sk).await?; + let partners = PartnerList { + entries: load_entries(&gcp_service, &env, "leader", partners, partners_filepath) + .await?, + }; + + let account_creator_sk = account_creator_sk.parse()?; + + let config = LeaderConfig { + env, + port: web_port, + sign_nodes, + near_rpc, + near_root_account, + // TODO: Create such an account for testnet and mainnet in a secure way + account_creator_id, + account_creator_sk, + partners, + }; + + if test { + run_leader_node::(config).await; + } else { + run_leader_node::(config).await; + } + } + Cli::StartSign { + env, + node_id, + sk_share, + cipher_key, + web_port, + oidc_providers, + oidc_providers_filepath, + gcp_project_id, + gcp_datastore_url, + test, + } => { + let gcp_service = + GcpService::new(env.clone(), gcp_project_id, gcp_datastore_url).await?; + let oidc_providers = OidcProviderList { + entries: load_entries( + &gcp_service, + &env, + node_id.to_string().as_str(), + oidc_providers, + oidc_providers_filepath, + ) + .await?, + }; + let cipher_key = load_cipher_key(&gcp_service, &env, node_id, cipher_key).await?; + let cipher_key = hex::decode(cipher_key)?; + let cipher_key = GenericArray::::clone_from_slice(&cipher_key); + let cipher = Aes256Gcm::new(&cipher_key); + + let sk_share = load_sh_skare(&gcp_service, &env, node_id, sk_share).await?; + + // TODO Import just the private key and derive the rest + let sk_share: ExpandedKeyPair = serde_json::from_str(&sk_share).unwrap(); + + let config = SignerConfig { + gcp_service, + our_index: node_id, + node_key: sk_share, + cipher, + port: web_port, + oidc_providers, + }; + if test { + run_sign_node::(config).await; + } else { + run_sign_node::(config).await; + } + } + Cli::RotateSignNodeCipher { + env, + new_env, + node_id, + old_cipher_key, + new_cipher_key, + gcp_project_id, + gcp_datastore_url, + } => { + let gcp_service = GcpService::new( + env.clone(), + gcp_project_id.clone(), + gcp_datastore_url.clone(), + ) + .await?; + + let dest_gcp_service = if let Some(new_env) = new_env { + GcpService::new(new_env, gcp_project_id, gcp_datastore_url).await? + } else { + gcp_service.clone() + }; + + let old_cipher_key = + load_cipher_key(&gcp_service, &env, node_id, old_cipher_key).await?; + let old_cipher_key = hex::decode(old_cipher_key)?; + let old_cipher_key = GenericArray::::clone_from_slice(&old_cipher_key); + let old_cipher = Aes256Gcm::new(&old_cipher_key); + + let new_cipher_key = + load_cipher_key(&gcp_service, &env, node_id, new_cipher_key).await?; + let new_cipher_key = hex::decode(new_cipher_key)?; + let new_cipher_key = GenericArray::::clone_from_slice(&new_cipher_key); + let new_cipher = Aes256Gcm::new(&new_cipher_key); + + migration::rotate_cipher( + node_id as usize, + &old_cipher, + &new_cipher, + &gcp_service, + &dest_gcp_service, + ) + .await?; + } + } + + Ok(()) +} + +async fn load_sh_skare( + gcp_service: &GcpService, + env: &str, + node_id: u64, + sk_share_arg: Option, +) -> anyhow::Result { + match sk_share_arg { + Some(sk_share) => Ok(sk_share), + None => { + let name = format!("mpc-recovery-secret-share-{node_id}-{env}/versions/latest"); + Ok(std::str::from_utf8(&gcp_service.load_secret(name).await?)?.to_string()) + } + } +} + +async fn load_cipher_key( + gcp_service: &GcpService, + env: &str, + node_id: u64, + cipher_key_arg: Option, +) -> anyhow::Result { + match cipher_key_arg { + Some(cipher_key) => Ok(cipher_key), + None => { + let name = format!("mpc-recovery-encryption-cipher-{node_id}-{env}/versions/latest"); + Ok(std::str::from_utf8(&gcp_service.load_secret(name).await?)?.to_string()) + } + } +} + +async fn load_account_creator_sk( + gcp_service: &GcpService, + env: &str, + account_creator_sk_arg: Option, +) -> anyhow::Result { + match account_creator_sk_arg { + Some(account_creator_sk) => Ok(account_creator_sk), + None => { + let name = format!("mpc-recovery-account-creator-sk-{env}/versions/latest"); + Ok(std::str::from_utf8(&gcp_service.load_secret(name).await?)?.to_string()) + } + } +} + +async fn load_entries( + gcp_service: &GcpService, + env: &str, + node_id: &str, + data: Option, + path: Option, +) -> anyhow::Result +where + T: DeserializeOwned, +{ + let entries = match (data, path) { + (Some(data), None) => serde_json::from_str(&data)?, + (None, Some(path)) => { + let file = std::fs::File::open(path)?; + let reader = std::io::BufReader::new(file); + serde_json::from_reader(reader)? + } + (None, None) => { + let name = + format!("mpc-recovery-allowed-oidc-providers-{node_id}-{env}/versions/latest"); + let data = gcp_service.load_secret(name).await?; + serde_json::from_str(std::str::from_utf8(&data)?)? + } + _ => return Err(anyhow::anyhow!("Invalid combination of data and path")), + }; + + Ok(entries) +} + +// impl Cli { + +// fn generate_command_args(self) -> Vec { +// match self { +// Cli::Generate { n } => { +// command_args.push("--n".to_string()); +// command_args.push(n.to_string()); +// } +// Cli::StartLeader { +// env, +// web_port, +// sign_nodes, +// near_rpc, +// near_root_account, +// account_creator_id, +// account_creator_sk, +// fast_auth_partners, +// fast_auth_partners_filepath, +// gcp_project_id, +// gcp_datastore_url, +// test, +// } => { +// // command_args.push("--env".to_string()); +// // command_args.push(env); +// // command_args.push("--web_port".to_string()); +// // command_args.push(web_port.to_string()); + +// // for sign_node in sign_nodes { +// // command_args.push("--sign_nodes".to_string()); +// // command_args.push(sign_node); +// // } + +// // command_args.push("--near_rpc".to_string()); +// // command_args.push(near_rpc); +// // command_args.push("--near_root_account".to_string()); +// // command_args.push(near_root_account); +// // // ... + +// vec![ +// "--env".into(), env, +// "--web_port".into(), web_port.to_string(), +// ] + +// } +// Cli::StartSign { +// env, +// node_id, +// cipher_key, +// sk_share, +// web_port, +// oidc_providers, +// // ... +// } => { +// command_args.push("--env".to_string()); +// command_args.push(env); +// command_args.push("--node_id".to_string()); +// command_args.push(node_id.to_string(); + +// if let Some(key) = cipher_key { +// command_args.push("--cipher_key".to_string()); +// command_args.push(key); +// } + +// if let Some(share) = sk_share { +// command_args.push("--sk_share".to_string()); +// command_args.push(share); +// } + +// command_args.push("--web_port".to_string()); +// command_args.push(web_port.to_string()); + +// if let Some(providers) = oidc_providers { +// command_args.push("--oidc_providers".to_string()); +// command_args.push(providers); +// } +// // ... +// } +// Cli::RotateSignNodeCipher { +// env, +// new_env, +// node_id, +// old_cipher_key, +// new_cipher_key, +// gcp_project_id, +// gcp_datastore_url, +// } => { +// command_args.push("--env".to_string()); +// command_args.push(env); + +// if let Some(new_env) = new_env { +// command_args.push("--new_env".to_string()); +// command_args.push(new_env); +// } + +// command_args.push("--node_id".to_string()); +// command_args.push(node_id.to_string(); + +// if let Some(old_key) = old_cipher_key { +// command_args.push("--old_cipher_key".to_string()); +// command_args.push(old_key); +// } + +// if let Some(new_key) = new_cipher_key { +// command_args.push("--new_cipher_key".to_string()); +// command_args.push(new_key); +// } +// } +// } +// } + +// } diff --git a/mpc-recovery/src/main.rs b/mpc-recovery/src/main.rs index 7f3d4161a..2193a5134 100644 --- a/mpc-recovery/src/main.rs +++ b/mpc-recovery/src/main.rs @@ -1,361 +1,8 @@ -use std::path::PathBuf; - -use aes_gcm::{ - aead::{consts::U32, generic_array::GenericArray, KeyInit}, - Aes256Gcm, -}; use clap::Parser; -use mpc_recovery::{ - firewall::allowed::{OidcProviderList, PartnerList}, - gcp::GcpService, - oauth::{PagodaFirebaseTokenVerifier, UniversalTokenVerifier}, - sign_node::migration, - GenerateResult, LeaderConfig, SignerConfig, -}; -use multi_party_eddsa::protocols::ExpandedKeyPair; -use near_primitives::types::AccountId; -use serde::de::DeserializeOwned; -use tracing_subscriber::EnvFilter; - -#[derive(Parser, Debug)] -enum Cli { - Generate { - n: usize, - }, - StartLeader { - /// Environment to run in (`dev` or `prod`) - #[arg(long, env("MPC_RECOVERY_ENV"), default_value("dev"))] - env: String, - /// The web port for this server - #[arg(long, env("MPC_RECOVERY_WEB_PORT"))] - web_port: u16, - /// The compute nodes to connect to - #[arg(long, value_parser, num_args = 1.., value_delimiter = ',', env("MPC_RECOVERY_SIGN_NODES"))] - sign_nodes: Vec, - /// NEAR RPC address - #[arg( - long, - env("MPC_RECOVERY_NEAR_RPC"), - default_value("https://rpc.testnet.near.org") - )] - near_rpc: String, - /// NEAR root account that has linkdrop contract deployed on it - #[arg(long, env("MPC_RECOVERY_NEAR_ROOT_ACCOUNT"), default_value("testnet"))] - near_root_account: String, - /// Account creator ID - #[arg(long, env("MPC_RECOVERY_ACCOUNT_CREATOR_ID"))] - account_creator_id: AccountId, - /// TEMPORARY - Account creator ed25519 secret key - #[arg(long, env("MPC_RECOVERY_ACCOUNT_CREATOR_SK"))] - account_creator_sk: Option, - /// JSON list of related items to be used to verify OIDC tokens. - #[arg(long, env("FAST_AUTH_PARTNERS"))] - fast_auth_partners: Option, - /// Filepath to a JSON list of related items to be used to verify OIDC tokens. - #[arg(long, value_parser, env("FAST_AUTH_PARTNERS_FILEPATH"))] - fast_auth_partners_filepath: Option, - /// GCP project ID - #[arg(long, env("MPC_RECOVERY_GCP_PROJECT_ID"))] - gcp_project_id: String, - /// GCP datastore URL - #[arg(long, env("MPC_RECOVERY_GCP_DATASTORE_URL"))] - gcp_datastore_url: Option, - /// Whether to accept test tokens - #[arg(long, env("MPC_RECOVERY_TEST"), default_value("false"))] - test: bool, - }, - StartSign { - /// Environment to run in (`dev` or `prod`) - #[arg(long, env("MPC_RECOVERY_ENV"), default_value("dev"))] - env: String, - /// Node ID - #[arg(long, env("MPC_RECOVERY_NODE_ID"))] - node_id: u64, - /// Cipher key to encrypt stored user credentials, will be pulled from GCP Secret Manager if omitted - #[arg(long, env("MPC_RECOVERY_CIPHER_KEY"))] - cipher_key: Option, - /// Secret key share, will be pulled from GCP Secret Manager if omitted - #[arg(long, env("MPC_RECOVERY_SK_SHARE"))] - sk_share: Option, - /// The web port for this server - #[arg(long, env("MPC_RECOVERY_WEB_PORT"))] - web_port: u16, - /// JSON list of related items to be used to verify OIDC tokens. - #[arg(long, env("OIDC_PROVIDERS"))] - oidc_providers: Option, - /// Filepath to a JSON list of related items to be used to verify OIDC tokens. - #[arg(long, value_parser, env("OIDC_PROVIDERS_FILEPATH"))] - oidc_providers_filepath: Option, - /// GCP project ID - #[arg(long, env("MPC_RECOVERY_GCP_PROJECT_ID"))] - gcp_project_id: String, - /// GCP datastore URL - #[arg(long, env("MPC_RECOVERY_GCP_DATASTORE_URL"))] - gcp_datastore_url: Option, - /// Whether to accept test tokens - #[arg(long, env("MPC_RECOVERY_TEST"), default_value("false"))] - test: bool, - }, - RotateSignNodeCipher { - /// Environment to run in (`dev` or `prod`) - #[arg(long, env("MPC_RECOVERY_ENV"), default_value("dev"))] - env: String, - /// If no `new_env` is specified, the rotation will be done inplace in the current `env`. - #[arg(long, env("MPC_RECOVERY_ROTATE_INPLACE"))] - new_env: Option, - /// Node ID - #[arg(long, env("MPC_RECOVERY_NODE_ID"))] - node_id: u64, - /// Old cipher key, will be pulled from GCP Secret Manager if omitted - #[arg(long, env("MPC_RECOVERY_OLD_CIPHER_KEY"))] - old_cipher_key: Option, - /// The new cipher key to replace each encrypted record with. - #[arg(long, env("MPC_RECOVERY_NEW_CIPHER_KEY"))] - new_cipher_key: Option, - /// GCP project ID - #[arg(long, env("MPC_RECOVERY_GCP_PROJECT_ID"))] - gcp_project_id: String, - /// GCP datastore URL - #[arg(long, env("MPC_RECOVERY_GCP_DATASTORE_URL"))] - gcp_datastore_url: Option, - }, -} - -async fn load_sh_skare( - gcp_service: &GcpService, - env: &str, - node_id: u64, - sk_share_arg: Option, -) -> anyhow::Result { - match sk_share_arg { - Some(sk_share) => Ok(sk_share), - None => { - let name = format!("mpc-recovery-secret-share-{node_id}-{env}/versions/latest"); - Ok(std::str::from_utf8(&gcp_service.load_secret(name).await?)?.to_string()) - } - } -} - -async fn load_cipher_key( - gcp_service: &GcpService, - env: &str, - node_id: u64, - cipher_key_arg: Option, -) -> anyhow::Result { - match cipher_key_arg { - Some(cipher_key) => Ok(cipher_key), - None => { - let name = format!("mpc-recovery-encryption-cipher-{node_id}-{env}/versions/latest"); - Ok(std::str::from_utf8(&gcp_service.load_secret(name).await?)?.to_string()) - } - } -} - -async fn load_account_creator_sk( - gcp_service: &GcpService, - env: &str, - account_creator_sk_arg: Option, -) -> anyhow::Result { - match account_creator_sk_arg { - Some(account_creator_sk) => Ok(account_creator_sk), - None => { - let name = format!("mpc-recovery-account-creator-sk-{env}/versions/latest"); - Ok(std::str::from_utf8(&gcp_service.load_secret(name).await?)?.to_string()) - } - } -} - -async fn load_entries( - gcp_service: &GcpService, - env: &str, - node_id: &str, - data: Option, - path: Option, -) -> anyhow::Result -where - T: DeserializeOwned, -{ - let entries = match (data, path) { - (Some(data), None) => serde_json::from_str(&data)?, - (None, Some(path)) => { - let file = std::fs::File::open(path)?; - let reader = std::io::BufReader::new(file); - serde_json::from_reader(reader)? - } - (None, None) => { - let name = - format!("mpc-recovery-allowed-oidc-providers-{node_id}-{env}/versions/latest"); - let data = gcp_service.load_secret(name).await?; - serde_json::from_str(std::str::from_utf8(&data)?)? - } - _ => return Err(anyhow::anyhow!("Invalid combination of data and path")), - }; - - Ok(entries) -} +use mpc_recovery::Cli; #[tokio::main] async fn main() -> anyhow::Result<()> { - // Install global collector configured based on RUST_LOG env var. - let mut subscriber = tracing_subscriber::fmt() - .with_thread_ids(true) - .with_env_filter(EnvFilter::from_default_env()); - // Check if running in Google Cloud Run: https://cloud.google.com/run/docs/container-contract#services-env-vars - if std::env::var("K_SERVICE").is_ok() { - // Disable colored logging as it messes up Google's log formatting - subscriber = subscriber.with_ansi(false); - } - subscriber.init(); - let _span = tracing::trace_span!("cli").entered(); - - match Cli::parse() { - Cli::Generate { n } => { - let GenerateResult { pk_set, secrets } = mpc_recovery::generate(n); - tracing::info!("Public key set: {}", serde_json::to_string(&pk_set)?); - for (i, (sk_share, cipher_key)) in secrets.iter().enumerate() { - tracing::info!( - "Secret key share {}: {}", - i, - serde_json::to_string(sk_share)? - ); - tracing::info!("Cipher {}: {}", i, hex::encode(cipher_key)); - } - } - Cli::StartLeader { - env, - web_port, - sign_nodes, - near_rpc, - near_root_account, - account_creator_id, - account_creator_sk, - fast_auth_partners: partners, - fast_auth_partners_filepath: partners_filepath, - gcp_project_id, - gcp_datastore_url, - test, - } => { - let gcp_service = - GcpService::new(env.clone(), gcp_project_id, gcp_datastore_url).await?; - let account_creator_sk = - load_account_creator_sk(&gcp_service, &env, account_creator_sk).await?; - let partners = PartnerList { - entries: load_entries(&gcp_service, &env, "leader", partners, partners_filepath) - .await?, - }; - - let account_creator_sk = account_creator_sk.parse()?; - - let config = LeaderConfig { - env, - port: web_port, - sign_nodes, - near_rpc, - near_root_account, - // TODO: Create such an account for testnet and mainnet in a secure way - account_creator_id, - account_creator_sk, - partners, - }; - - if test { - mpc_recovery::run_leader_node::(config).await; - } else { - mpc_recovery::run_leader_node::(config).await; - } - } - Cli::StartSign { - env, - node_id, - sk_share, - cipher_key, - web_port, - oidc_providers, - oidc_providers_filepath, - gcp_project_id, - gcp_datastore_url, - test, - } => { - let gcp_service = - GcpService::new(env.clone(), gcp_project_id, gcp_datastore_url).await?; - let oidc_providers = OidcProviderList { - entries: load_entries( - &gcp_service, - &env, - node_id.to_string().as_str(), - oidc_providers, - oidc_providers_filepath, - ) - .await?, - }; - let cipher_key = load_cipher_key(&gcp_service, &env, node_id, cipher_key).await?; - let cipher_key = hex::decode(cipher_key)?; - let cipher_key = GenericArray::::clone_from_slice(&cipher_key); - let cipher = Aes256Gcm::new(&cipher_key); - - let sk_share = load_sh_skare(&gcp_service, &env, node_id, sk_share).await?; - - // TODO Import just the private key and derive the rest - let sk_share: ExpandedKeyPair = serde_json::from_str(&sk_share).unwrap(); - - let config = SignerConfig { - gcp_service, - our_index: node_id, - node_key: sk_share, - cipher, - port: web_port, - oidc_providers, - }; - if test { - mpc_recovery::run_sign_node::(config).await; - } else { - mpc_recovery::run_sign_node::(config).await; - } - } - Cli::RotateSignNodeCipher { - env, - new_env, - node_id, - old_cipher_key, - new_cipher_key, - gcp_project_id, - gcp_datastore_url, - } => { - let gcp_service = GcpService::new( - env.clone(), - gcp_project_id.clone(), - gcp_datastore_url.clone(), - ) - .await?; - - let dest_gcp_service = if let Some(new_env) = new_env { - GcpService::new(new_env, gcp_project_id, gcp_datastore_url).await? - } else { - gcp_service.clone() - }; - - let old_cipher_key = - load_cipher_key(&gcp_service, &env, node_id, old_cipher_key).await?; - let old_cipher_key = hex::decode(old_cipher_key)?; - let old_cipher_key = GenericArray::::clone_from_slice(&old_cipher_key); - let old_cipher = Aes256Gcm::new(&old_cipher_key); - - let new_cipher_key = - load_cipher_key(&gcp_service, &env, node_id, new_cipher_key).await?; - let new_cipher_key = hex::decode(new_cipher_key)?; - let new_cipher_key = GenericArray::::clone_from_slice(&new_cipher_key); - let new_cipher = Aes256Gcm::new(&new_cipher_key); - - migration::rotate_cipher( - node_id as usize, - &old_cipher, - &new_cipher, - &gcp_service, - &dest_gcp_service, - ) - .await?; - } - } - + mpc_recovery::run(Cli::parse()).await?; Ok(()) } diff --git a/mpc-recovery/src/relayer/mod.rs b/mpc-recovery/src/relayer/mod.rs index 6c5d0dc55..654eb8adc 100644 --- a/mpc-recovery/src/relayer/mod.rs +++ b/mpc-recovery/src/relayer/mod.rs @@ -51,7 +51,7 @@ impl NearRpcAndRelayerClient { ) -> Result<(), RelayerError> { let mut req = Request::builder() .method(Method::POST) - .uri(format!("{}/register_account", relayer.url)) + .uri(format!("{}/register_account_and_allowance", relayer.url)) .header("content-type", "application/json"); if let Some(api_key) = relayer.api_key { From 97eceaf44a0fa0c646e1f9f4d6cdb96594f56f75 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 14:59:49 -0700 Subject: [PATCH 02/21] Moved running local related testing functions over to local.rs --- integration-tests/src/containers.rs | 205 +-------------------------- integration-tests/src/lib.rs | 1 + integration-tests/src/local.rs | 212 ++++++++++++++++++++++++++++ integration-tests/tests/lib.rs | 6 +- 4 files changed, 218 insertions(+), 206 deletions(-) create mode 100644 integration-tests/src/local.rs diff --git a/integration-tests/src/containers.rs b/integration-tests/src/containers.rs index 51ef68fbb..893914a67 100644 --- a/integration-tests/src/containers.rs +++ b/integration-tests/src/containers.rs @@ -1,8 +1,7 @@ #![allow(clippy::too_many_arguments)] use aes_gcm::{Aes256Gcm, KeyInit}; -use anyhow::{anyhow, Context, Ok}; -use async_process::{Child, Command, Stdio}; +use anyhow::{anyhow, Ok}; use bollard::{container::LogsOptions, network::CreateNetworkOptions, service::Ipam, Docker}; use ed25519_dalek::ed25519::signature::digest::{consts::U32, generic_array::GenericArray}; use ed25519_dalek::{PublicKey as PublicKeyEd25519, Verifier}; @@ -448,32 +447,6 @@ pub struct SignerNode<'a> { gcp_datastore_local_url: String, } -pub struct SignerNodeLocal { - pub address: String, - node_id: usize, - sk_share: ExpandedKeyPair, - cipher_key: GenericArray, - gcp_project_id: String, - gcp_datastore_url: String, - - // process held so it's not dropped. Once dropped, process will be killed. - #[allow(unused)] - process: Child, -} - -impl SignerNodeLocal { - pub fn api(&self) -> SignerNodeApi { - SignerNodeApi { - address: self.address.clone(), - node_id: self.node_id, - sk_share: self.sk_share.clone(), - cipher_key: self.cipher_key, - gcp_project_id: self.gcp_project_id.clone(), - gcp_datastore_local_url: self.gcp_datastore_url.clone(), - } - } -} - pub struct SignerNodeApi { pub address: String, pub node_id: usize, @@ -560,83 +533,6 @@ impl<'a> SignerNode<'a> { }) } - pub async fn run_local( - web_port: usize, - node_id: u64, - sk_share: &ExpandedKeyPair, - cipher_key: &GenericArray, - datastore_url: &str, - gcp_project_id: &str, - firebase_audience_id: &str, - release: bool, - ) -> anyhow::Result { - let executable = util::target_dir() - .context("could not find target dir while running signing node")? - .join(if release { "release" } else { "debug" }) - .join("mpc-recovery"); - let args = vec![ - "start-sign".to_string(), - "--node-id".to_string(), - node_id.to_string(), - "--sk-share".to_string(), - serde_json::to_string(&sk_share)?, - "--cipher-key".to_string(), - hex::encode(cipher_key), - "--web-port".to_string(), - web_port.to_string(), - "--oidc-providers".to_string(), - serde_json::json!([ - { - "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), - "audience": firebase_audience_id, - }, - ]) - .to_string(), - "--gcp-project-id".to_string(), - gcp_project_id.to_string(), - "--gcp-datastore-url".to_string(), - datastore_url.to_string(), - "--test".to_string(), - ]; - - let address = format!("http://localhost:{web_port}"); - let child = Command::new(&executable) - .args(&args) - .envs(std::env::vars()) - .env("RUST_LOG", "mpc_recovery=DEBUG") - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn() - .with_context(|| { - format!( - "failed to run signing node: [node_id={node_id}, {}]", - executable.display() - ) - })?; - - tracing::info!("Signer node is start on {}", address); - loop { - let x: anyhow::Result = util::get(&address).await; - match x { - std::result::Result::Ok(status) if status == StatusCode::OK => break, - _err => {} - } - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - } - tracing::info!("Signer node started [node_id={node_id}, {address}]"); - - Ok(SignerNodeLocal { - address, - node_id: node_id as usize, - sk_share: sk_share.clone(), - cipher_key: *cipher_key, - gcp_project_id: gcp_project_id.to_string(), - gcp_datastore_url: datastore_url.to_string(), - process: child, - }) - } - pub fn api(&self) -> SignerNodeApi { SignerNodeApi { address: self.local_address.clone(), @@ -692,28 +588,10 @@ pub struct LeaderNode<'a> { local_address: String, } -pub struct LeaderNodeLocal { - pub address: String, - - // process held so it's not dropped. Once dropped, process will be killed. - #[allow(unused)] - process: Child, -} - -impl LeaderNodeLocal { - pub fn api(&self, near_rpc: &str, relayer: &DelegateActionRelayer) -> LeaderNodeApi { - LeaderNodeApi { - address: self.address.clone(), - client: NearRpcAndRelayerClient::connect(near_rpc), - relayer: relayer.clone(), - } - } -} - pub struct LeaderNodeApi { pub address: String, pub relayer: DelegateActionRelayer, - client: NearRpcAndRelayerClient, + pub(crate) client: NearRpcAndRelayerClient, } impl<'a> LeaderNode<'a> { @@ -796,85 +674,6 @@ impl<'a> LeaderNode<'a> { }) } - pub async fn run_local( - web_port: usize, - sign_nodes: Vec, - near_rpc: &str, - relayer_url: &str, - datastore_url: &str, - gcp_project_id: &str, - near_root_account: &AccountId, - account_creator_id: &AccountId, - account_creator_sk: &workspaces::types::SecretKey, - firebase_audience_id: &str, - release: bool, - ) -> anyhow::Result { - tracing::info!("Running leader node..."); - let executable = util::target_dir() - .context("could not find target dir while running leader node")? - .join(if release { "release" } else { "debug" }) - .join("mpc-recovery"); - let mut args = vec![ - "start-leader".to_string(), - "--web-port".to_string(), - web_port.to_string(), - "--near-rpc".to_string(), - near_rpc.to_string(), - "--near-root-account".to_string(), - near_root_account.to_string(), - "--account-creator-id".to_string(), - account_creator_id.to_string(), - "--account-creator-sk".to_string(), - account_creator_sk.to_string(), - "--fast-auth-partners".to_string(), - serde_json::json!([ - { - "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{}", firebase_audience_id), - "audience": firebase_audience_id, - }, - "relayer": { - "url": relayer_url.to_string(), - "api_key": serde_json::Value::Null, - }, - }, - ]).to_string(), - "--gcp-project-id".to_string(), - gcp_project_id.to_string(), - "--gcp-datastore-url".to_string(), - datastore_url.to_string(), - "--test".to_string(), - ]; - for sign_node in sign_nodes { - args.push("--sign-nodes".to_string()); - args.push(sign_node); - } - - let child = Command::new(&executable) - .args(&args) - .envs(std::env::vars()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn() - .with_context(|| format!("failed to run leader node: {}", executable.display()))?; - - let address = format!("http://localhost:{web_port}"); - tracing::info!("Leader node container is starting at {}", address); - loop { - match util::get(&address).await { - std::result::Result::Ok(status) if status == StatusCode::OK => break, - _ => tokio::time::sleep(std::time::Duration::from_secs(1)).await, - } - } - - tracing::info!("Leader node container is running at {address}"); - Ok(LeaderNodeLocal { - address, - process: child, - }) - } - pub fn api(&self, near_rpc: &str, relayer: &DelegateActionRelayer) -> LeaderNodeApi { LeaderNodeApi { address: self.local_address.clone(), diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs index 319e0d0f0..afb03e173 100644 --- a/integration-tests/src/lib.rs +++ b/integration-tests/src/lib.rs @@ -10,6 +10,7 @@ use workspaces::{ pub mod containers; pub mod sandbox; pub mod util; +pub mod local; async fn fetch_validator_keys( docker_client: &containers::DockerClient, diff --git a/integration-tests/src/local.rs b/integration-tests/src/local.rs new file mode 100644 index 000000000..62b38ee7f --- /dev/null +++ b/integration-tests/src/local.rs @@ -0,0 +1,212 @@ +#![allow(clippy::too_many_arguments)] + +use aes_gcm::aead::consts::U32; +use aes_gcm::aead::generic_array::GenericArray; +use anyhow::Context; +use async_process::{Child, Command, Stdio}; +use hyper::StatusCode; +use mpc_recovery::firewall::allowed::DelegateActionRelayer; +use mpc_recovery::relayer::NearRpcAndRelayerClient; +use multi_party_eddsa::protocols::ExpandedKeyPair; + +use crate::containers::{LeaderNodeApi, SignerNodeApi}; +use crate::util; + +pub struct SignerNode { + pub address: String, + node_id: usize, + sk_share: ExpandedKeyPair, + cipher_key: GenericArray, + gcp_project_id: String, + gcp_datastore_url: String, + + // process held so it's not dropped. Once dropped, process will be killed. + #[allow(unused)] + process: Child, +} + +impl SignerNode { + pub async fn run( + web_port: usize, + node_id: u64, + sk_share: &ExpandedKeyPair, + cipher_key: &GenericArray, + datastore_url: &str, + gcp_project_id: &str, + firebase_audience_id: &str, + release: bool, + ) -> anyhow::Result { + let executable = util::target_dir() + .context("could not find target dir while running signing node")? + .join(if release { "release" } else { "debug" }) + .join("mpc-recovery"); + let args = vec![ + "start-sign".to_string(), + "--node-id".to_string(), + node_id.to_string(), + "--sk-share".to_string(), + serde_json::to_string(&sk_share)?, + "--cipher-key".to_string(), + hex::encode(cipher_key), + "--web-port".to_string(), + web_port.to_string(), + "--oidc-providers".to_string(), + serde_json::json!([ + { + "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), + "audience": firebase_audience_id, + }, + ]) + .to_string(), + "--gcp-project-id".to_string(), + gcp_project_id.to_string(), + "--gcp-datastore-url".to_string(), + datastore_url.to_string(), + "--test".to_string(), + ]; + + let address = format!("http://localhost:{web_port}"); + let child = Command::new(&executable) + .args(&args) + .envs(std::env::vars()) + .env("RUST_LOG", "mpc_recovery=DEBUG") + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .with_context(|| { + format!( + "failed to run signing node: [node_id={node_id}, {}]", + executable.display() + ) + })?; + + tracing::info!("Signer node is start on {}", address); + loop { + let x: anyhow::Result = util::get(&address).await; + match x { + std::result::Result::Ok(status) if status == StatusCode::OK => break, + _err => {} + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + tracing::info!("Signer node started [node_id={node_id}, {address}]"); + + Ok(Self { + address, + node_id: node_id as usize, + sk_share: sk_share.clone(), + cipher_key: *cipher_key, + gcp_project_id: gcp_project_id.to_string(), + gcp_datastore_url: datastore_url.to_string(), + process: child, + }) + } + + pub fn api(&self) -> SignerNodeApi { + SignerNodeApi { + address: self.address.clone(), + node_id: self.node_id, + sk_share: self.sk_share.clone(), + cipher_key: self.cipher_key, + gcp_project_id: self.gcp_project_id.clone(), + gcp_datastore_local_url: self.gcp_datastore_url.clone(), + } + } +} + +pub struct LeaderNode { + pub address: String, + + // process held so it's not dropped. Once dropped, process will be killed. + #[allow(unused)] + process: Child, +} +impl LeaderNode { + pub fn api(&self, near_rpc: &str, relayer: &DelegateActionRelayer) -> LeaderNodeApi { + LeaderNodeApi { + address: self.address.clone(), + client: NearRpcAndRelayerClient::connect(near_rpc), + relayer: relayer.clone(), + } + } + + pub async fn run( + web_port: usize, + sign_nodes: Vec, + near_rpc: &str, + relayer_url: &str, + datastore_url: &str, + gcp_project_id: &str, + near_root_account: &workspaces::AccountId, + account_creator_id: &workspaces::AccountId, + account_creator_sk: &workspaces::types::SecretKey, + firebase_audience_id: &str, + release: bool, + ) -> anyhow::Result { + tracing::info!("Running leader node..."); + let executable = util::target_dir() + .context("could not find target dir while running leader node")? + .join(if release { "release" } else { "debug" }) + .join("mpc-recovery"); + let mut args = vec![ + "start-leader".to_string(), + "--web-port".to_string(), + web_port.to_string(), + "--near-rpc".to_string(), + near_rpc.to_string(), + "--near-root-account".to_string(), + near_root_account.to_string(), + "--account-creator-id".to_string(), + account_creator_id.to_string(), + "--account-creator-sk".to_string(), + account_creator_sk.to_string(), + "--fast-auth-partners".to_string(), + serde_json::json!([ + { + "oidc_provider": { + "issuer": format!("https://securetoken.google.com/{}", firebase_audience_id), + "audience": firebase_audience_id, + }, + "relayer": { + "url": relayer_url.to_string(), + "api_key": serde_json::Value::Null, + }, + }, + ]).to_string(), + "--gcp-project-id".to_string(), + gcp_project_id.to_string(), + "--gcp-datastore-url".to_string(), + datastore_url.to_string(), + "--test".to_string(), + ]; + for sign_node in sign_nodes { + args.push("--sign-nodes".to_string()); + args.push(sign_node); + } + + let child = Command::new(&executable) + .args(&args) + .envs(std::env::vars()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("failed to run leader node: {}", executable.display()))?; + + let address = format!("http://localhost:{web_port}"); + tracing::info!("Leader node container is starting at {}", address); + loop { + match util::get(&address).await { + std::result::Result::Ok(status) if status == StatusCode::OK => break, + _ => tokio::time::sleep(std::time::Duration::from_secs(1)).await, + } + } + + tracing::info!("Leader node container is running at {address}"); + Ok(Self { + address, + process: child, + }) + } +} diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index 6737a8641..0ebec4d2f 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -10,7 +10,7 @@ use mpc_recovery::{ }, GenerateResult, }; -use mpc_recovery_integration_tests::{containers, util}; +use mpc_recovery_integration_tests::{containers, local, util}; use near_primitives::utils::generate_random_string; use workspaces::{network::Sandbox, Worker}; @@ -59,7 +59,7 @@ where let GenerateResult { pk_set, secrets } = mpc_recovery::generate(nodes); let mut signer_node_futures = Vec::new(); for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { - signer_node_futures.push(containers::SignerNode::run_local( + signer_node_futures.push(local::SignerNode::run( util::pick_unused_port().await? as usize, i as u64, share, @@ -77,7 +77,7 @@ where let signer_urls: &Vec<_> = &signer_nodes.iter().map(|n| n.address.clone()).collect(); let near_root_account = relayer_ctx.worker.root_account()?; - let leader_node = containers::LeaderNode::run_local( + let leader_node = local::LeaderNode::run( util::pick_unused_port().await? as usize, signer_urls.clone(), &relayer_ctx.sandbox.local_address, From fa5f204da649d272a8d8ea050a31b751c3428062 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 15:16:48 -0700 Subject: [PATCH 03/21] Reduced LeaderNode::api to not require arguments --- integration-tests/src/containers.rs | 15 +++++++++++---- integration-tests/src/local.rs | 27 +++++++++++++++++---------- integration-tests/src/util.rs | 16 +++++----------- integration-tests/tests/lib.rs | 13 +++---------- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/integration-tests/src/containers.rs b/integration-tests/src/containers.rs index 893914a67..6a2778485 100644 --- a/integration-tests/src/containers.rs +++ b/integration-tests/src/containers.rs @@ -586,6 +586,8 @@ pub struct LeaderNode<'a> { pub container: Container<'a, GenericImage>, pub address: String, local_address: String, + near_rpc: String, + relayer_url: String, } pub struct LeaderNodeApi { @@ -671,14 +673,19 @@ impl<'a> LeaderNode<'a> { container, address: full_address, local_address: format!("http://localhost:{host_port}"), + near_rpc: near_rpc.to_string(), + relayer_url: relayer_url.to_string(), }) } - pub fn api(&self, near_rpc: &str, relayer: &DelegateActionRelayer) -> LeaderNodeApi { + pub fn api(&self) -> LeaderNodeApi { LeaderNodeApi { - address: self.local_address.clone(), - client: NearRpcAndRelayerClient::connect(near_rpc), - relayer: relayer.clone(), + address: self.address.clone(), + client: NearRpcAndRelayerClient::connect(&self.near_rpc), + relayer: DelegateActionRelayer { + url: self.relayer_url.clone(), + api_key: None, + }, } } } diff --git a/integration-tests/src/local.rs b/integration-tests/src/local.rs index 62b38ee7f..133001223 100644 --- a/integration-tests/src/local.rs +++ b/integration-tests/src/local.rs @@ -27,7 +27,7 @@ pub struct SignerNode { impl SignerNode { pub async fn run( - web_port: usize, + web_port: u16, node_id: u64, sk_share: &ExpandedKeyPair, cipher_key: &GenericArray, @@ -117,22 +117,16 @@ impl SignerNode { pub struct LeaderNode { pub address: String, + near_rpc: String, + relayer_url: String, // process held so it's not dropped. Once dropped, process will be killed. #[allow(unused)] process: Child, } impl LeaderNode { - pub fn api(&self, near_rpc: &str, relayer: &DelegateActionRelayer) -> LeaderNodeApi { - LeaderNodeApi { - address: self.address.clone(), - client: NearRpcAndRelayerClient::connect(near_rpc), - relayer: relayer.clone(), - } - } - pub async fn run( - web_port: usize, + web_port: u16, sign_nodes: Vec, near_rpc: &str, relayer_url: &str, @@ -206,7 +200,20 @@ impl LeaderNode { tracing::info!("Leader node container is running at {address}"); Ok(Self { address, + near_rpc: near_rpc.to_string(), + relayer_url: relayer_url.to_string(), process: child, }) } + + pub fn api(&self) -> LeaderNodeApi { + LeaderNodeApi { + address: self.address.clone(), + client: NearRpcAndRelayerClient::connect(&self.near_rpc), + relayer: DelegateActionRelayer { + url: self.relayer_url.clone(), + api_key: None, + }, + } + } } diff --git a/integration-tests/src/util.rs b/integration-tests/src/util.rs index 3a46758dd..6c11a0fe8 100644 --- a/integration-tests/src/util.rs +++ b/integration-tests/src/util.rs @@ -1,7 +1,7 @@ use std::{ fs::{self, File}, io::Write, - path::PathBuf, + path::{Path, PathBuf}, }; use anyhow::{Context, Ok}; @@ -208,21 +208,15 @@ pub async fn pick_unused_port() -> anyhow::Result { } pub fn target_dir() -> Option { - let mut out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap_or_else(|e| { - panic!( - "Failed to get OUT_DIR env variable: {e}.\n\ - Please run `cargo test` from the root of the repository.", - ) - })); - + let mut out_dir = Path::new(std::env!("OUT_DIR")); loop { if out_dir.ends_with("target") { - return Some(out_dir.to_path_buf()); + break Some(out_dir.to_path_buf()); } match out_dir.parent() { - Some(parent) => out_dir = parent.to_owned(), - None => return None, // We've reached the root directory and didn't find "target" + Some(parent) => out_dir = parent, + None => break None, // We've reached the root directory and didn't find "target" } } } diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index 0ebec4d2f..4fb2bc52b 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -3,7 +3,6 @@ mod mpc; use curv::elliptic::curves::{Ed25519, Point}; use hyper::StatusCode; use mpc_recovery::{ - firewall::allowed::DelegateActionRelayer, gcp::GcpService, msg::{ ClaimOidcResponse, MpcPkResponse, NewAccountResponse, SignResponse, UserCredentialsResponse, @@ -60,7 +59,7 @@ where let mut signer_node_futures = Vec::new(); for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { signer_node_futures.push(local::SignerNode::run( - util::pick_unused_port().await? as usize, + util::pick_unused_port().await?, i as u64, share, cipher_key, @@ -78,7 +77,7 @@ where let near_root_account = relayer_ctx.worker.root_account()?; let leader_node = local::LeaderNode::run( - util::pick_unused_port().await? as usize, + util::pick_unused_port().await?, signer_urls.clone(), &relayer_ctx.sandbox.local_address, &relayer_ctx.relayer.local_address, @@ -93,14 +92,8 @@ where .await?; f(TestContext { - leader_node: leader_node.api( - &relayer_ctx.sandbox.local_address, - &DelegateActionRelayer { - url: relayer_ctx.relayer.local_address.clone(), - api_key: None, - }, - ), pk_set, + leader_node: leader_node.api(), signer_nodes: signer_nodes.iter().map(|n| n.api()).collect(), worker: relayer_ctx.worker.clone(), gcp_datastore_url: datastore.local_address, From 3fff52abeae4e855dcf5b13956302ef5c4b327c7 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 21:00:54 -0700 Subject: [PATCH 04/21] Moved all containers init into env/ --- integration-tests/Cargo.toml | 5 +- integration-tests/src/env/mod.rs | 180 +++++++++++++++++++++++++++++++ integration-tests/src/lib.rs | 3 +- integration-tests/src/local.rs | 6 +- integration-tests/tests/lib.rs | 76 +++---------- 5 files changed, 204 insertions(+), 66 deletions(-) create mode 100644 integration-tests/src/env/mod.rs diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 22e1b16b6..f6386b25b 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -10,6 +10,7 @@ anyhow = { version = "1.0", features = ["backtrace"] } async-process = "1" bollard = "0.11" clap = { version = "4.2", features = ["derive", "env"] } +curv = { package = "curv-kzen", version = "0.9", default-features = false } ed25519-dalek = { version = "1.0.1", features = ["serde"] } futures = "0.3" hex = "0.4.3" @@ -37,9 +38,9 @@ test-log = { version = "0.2.12", features = ["log", "trace"] } env_logger = "0.10.0" tracing-log = "0.1.3" tokio-util = { version = "0.7", features = ["full"] } -curv = { package = "curv-kzen", version = "0.9", default-features = false } reqwest = "0.11.16" [features] +default = ["drop-containers", "test-local"] drop-containers = [] -default = ["drop-containers"] +test-local = [] diff --git a/integration-tests/src/env/mod.rs b/integration-tests/src/env/mod.rs new file mode 100644 index 000000000..34b93e261 --- /dev/null +++ b/integration-tests/src/env/mod.rs @@ -0,0 +1,180 @@ +use curv::elliptic::curves::{Ed25519, Point}; +use mpc_recovery::GenerateResult; +use near_primitives::utils::generate_random_string; + +use crate::containers::{DockerClient, LeaderNodeApi, SignerNodeApi}; +use crate::{containers, initialize_relayer, local, util, RelayerCtx}; + +pub const NETWORK: &str = "mpc_it_network"; +pub const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; +// TODO: figure out how to instantiate and use a local firebase deployment +pub const FIREBASE_AUDIENCE_ID: &str = "test_audience"; + +pub enum Nodes<'a> { + Local { + ctx: Context<'a>, + pk_set: Vec>, + leader_node: local::LeaderNode, + signer_nodes: Vec, + }, + Docker { + ctx: Context<'a>, + pk_set: Vec>, + leader_node: containers::LeaderNode<'a>, + signer_nodes: Vec>, + }, +} + +impl Nodes<'_> { + pub fn ctx(&self) -> &Context { + match self { + Nodes::Local { ctx, .. } => ctx, + Nodes::Docker { ctx, .. } => ctx, + } + } + + pub fn pk_set(&self) -> Vec> { + match self { + Nodes::Local { pk_set, .. } => pk_set.clone(), + Nodes::Docker { pk_set, .. } => pk_set.clone(), + } + } + + pub fn leader_api(&self) -> LeaderNodeApi { + match self { + Nodes::Local { leader_node, .. } => leader_node.api(), + Nodes::Docker { leader_node, .. } => leader_node.api(), + } + } + + pub fn signer_apis(&self) -> Vec { + match self { + Nodes::Local { signer_nodes, .. } => signer_nodes.iter().map(|n| n.api()).collect(), + Nodes::Docker { signer_nodes, .. } => signer_nodes.iter().map(|n| n.api()).collect(), + } + } + + pub fn datastore_addr(&self) -> String { + // this is different per env: + match self { + Nodes::Local { ctx, .. } => ctx.datastore.local_address.clone(), + Nodes::Docker { ctx, .. } => ctx.datastore.address.clone(), + } + } +} + +pub struct Context<'a> { + pub relayer_ctx: RelayerCtx<'a>, + pub datastore: containers::Datastore<'a>, +} + +pub async fn setup<'a>(docker_client: &'a DockerClient) -> anyhow::Result> { + docker_client.create_network(NETWORK).await?; + + let relayer_id = generate_random_string(7); // used to distinguish relayer tmp files in multiple tests + let relayer_ctx_future = initialize_relayer(&docker_client, NETWORK, &relayer_id); + let datastore_future = containers::Datastore::run(&docker_client, NETWORK, GCP_PROJECT_ID); + + let (relayer_ctx, datastore) = + futures::future::join(relayer_ctx_future, datastore_future).await; + let relayer_ctx = relayer_ctx?; + let datastore = datastore?; + + Ok(Context { + relayer_ctx, + datastore, + }) +} + +pub async fn docker(nodes: usize, docker_client: &DockerClient) -> anyhow::Result { + let ctx = setup(docker_client).await?; + + let GenerateResult { pk_set, secrets } = mpc_recovery::generate(nodes); + let mut signer_node_futures = Vec::with_capacity(nodes); + for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { + signer_node_futures.push(containers::SignerNode::run_signing_node( + &docker_client, + NETWORK, + i as u64, + share, + cipher_key, + &ctx.datastore.address, + &ctx.datastore.local_address, + GCP_PROJECT_ID, + FIREBASE_AUDIENCE_ID, + )); + } + let signer_nodes = futures::future::join_all(signer_node_futures) + .await + .into_iter() + .collect::, _>>()?; + let signer_urls: &Vec<_> = &signer_nodes.iter().map(|n| n.address.clone()).collect(); + + let near_root_account = ctx.relayer_ctx.worker.root_account()?; + let leader_node = containers::LeaderNode::run( + &docker_client, + NETWORK, + signer_urls.clone(), + &ctx.relayer_ctx.sandbox.address, + &ctx.relayer_ctx.relayer.address, + &ctx.datastore.address, + GCP_PROJECT_ID, + near_root_account.id(), + ctx.relayer_ctx.creator_account.id(), + ctx.relayer_ctx.creator_account.secret_key(), + FIREBASE_AUDIENCE_ID, + ) + .await?; + + Ok(Nodes::Docker { + ctx, + pk_set, + leader_node, + signer_nodes, + }) +} + +pub async fn host(nodes: usize, docker_client: &DockerClient) -> anyhow::Result { + let ctx = setup(docker_client).await?; + let GenerateResult { pk_set, secrets } = mpc_recovery::generate(nodes); + let mut signer_node_futures = Vec::with_capacity(nodes); + for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { + signer_node_futures.push(local::SignerNode::run( + util::pick_unused_port().await?, + i as u64, + share, + cipher_key, + &ctx.datastore.local_address, + GCP_PROJECT_ID, + FIREBASE_AUDIENCE_ID, + true, + )); + } + let signer_nodes = futures::future::join_all(signer_node_futures) + .await + .into_iter() + .collect::, _>>()?; + + let near_root_account = ctx.relayer_ctx.worker.root_account()?; + let leader_node = local::LeaderNode::run( + util::pick_unused_port().await?, + signer_nodes.iter().map(|n| n.address.clone()).collect(), + &ctx.relayer_ctx.sandbox.local_address, + &ctx.relayer_ctx.relayer.local_address, + &ctx.datastore.local_address, + GCP_PROJECT_ID, + near_root_account.id(), + ctx.relayer_ctx.creator_account.id(), + ctx.relayer_ctx.creator_account.secret_key(), + FIREBASE_AUDIENCE_ID, + true, + ) + .await?; + + Ok(Nodes::Local { + ctx, + pk_set, + leader_node, + signer_nodes, + }) +} diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs index afb03e173..b5669f5a2 100644 --- a/integration-tests/src/lib.rs +++ b/integration-tests/src/lib.rs @@ -8,9 +8,10 @@ use workspaces::{ }; pub mod containers; +pub mod env; +pub mod local; pub mod sandbox; pub mod util; -pub mod local; async fn fetch_validator_keys( docker_client: &containers::DockerClient, diff --git a/integration-tests/src/local.rs b/integration-tests/src/local.rs index 133001223..37f735d2b 100644 --- a/integration-tests/src/local.rs +++ b/integration-tests/src/local.rs @@ -12,6 +12,8 @@ use multi_party_eddsa::protocols::ExpandedKeyPair; use crate::containers::{LeaderNodeApi, SignerNodeApi}; use crate::util; +const EXECUTABLE: &str = "mpc-recovery"; + pub struct SignerNode { pub address: String, node_id: usize, @@ -39,7 +41,7 @@ impl SignerNode { let executable = util::target_dir() .context("could not find target dir while running signing node")? .join(if release { "release" } else { "debug" }) - .join("mpc-recovery"); + .join(EXECUTABLE); let args = vec![ "start-sign".to_string(), "--node-id".to_string(), @@ -142,7 +144,7 @@ impl LeaderNode { let executable = util::target_dir() .context("could not find target dir while running leader node")? .join(if release { "release" } else { "debug" }) - .join("mpc-recovery"); + .join(EXECUTABLE); let mut args = vec![ "start-leader".to_string(), "--web-port".to_string(), diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index 4fb2bc52b..1f107289f 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -7,17 +7,13 @@ use mpc_recovery::{ msg::{ ClaimOidcResponse, MpcPkResponse, NewAccountResponse, SignResponse, UserCredentialsResponse, }, - GenerateResult, }; -use mpc_recovery_integration_tests::{containers, local, util}; -use near_primitives::utils::generate_random_string; +use mpc_recovery_integration_tests::{ + containers, + env::{self, GCP_PROJECT_ID}, +}; use workspaces::{network::Sandbox, Worker}; -const NETWORK: &str = "mpc_it_network"; -const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; -// TODO: figure out how to instantiate and use a local firebase deployment -pub const FIREBASE_AUDIENCE_ID: &str = "test_audience"; - pub struct TestContext { leader_node: containers::LeaderNodeApi, pk_set: Vec>, @@ -43,64 +39,22 @@ where Fut: core::future::Future>, { let docker_client = containers::DockerClient::default(); - docker_client.create_network(NETWORK).await?; - - let relayer_id = generate_random_string(7); // used to distinguish relayer tmp files in multiple tests - let relayer_ctx_future = - mpc_recovery_integration_tests::initialize_relayer(&docker_client, NETWORK, &relayer_id); - let datastore_future = containers::Datastore::run(&docker_client, NETWORK, GCP_PROJECT_ID); - - let (relayer_ctx, datastore) = - futures::future::join(relayer_ctx_future, datastore_future).await; - let relayer_ctx = relayer_ctx?; - let datastore = datastore?; - - let GenerateResult { pk_set, secrets } = mpc_recovery::generate(nodes); - let mut signer_node_futures = Vec::new(); - for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { - signer_node_futures.push(local::SignerNode::run( - util::pick_unused_port().await?, - i as u64, - share, - cipher_key, - &datastore.local_address, - GCP_PROJECT_ID, - FIREBASE_AUDIENCE_ID, - true, - )); - } - let signer_nodes = futures::future::join_all(signer_node_futures) - .await - .into_iter() - .collect::, _>>()?; - let signer_urls: &Vec<_> = &signer_nodes.iter().map(|n| n.address.clone()).collect(); - - let near_root_account = relayer_ctx.worker.root_account()?; - let leader_node = local::LeaderNode::run( - util::pick_unused_port().await?, - signer_urls.clone(), - &relayer_ctx.sandbox.local_address, - &relayer_ctx.relayer.local_address, - &datastore.local_address, - GCP_PROJECT_ID, - near_root_account.id(), - relayer_ctx.creator_account.id(), - relayer_ctx.creator_account.secret_key(), - FIREBASE_AUDIENCE_ID, - true, - ) - .await?; + let nodes = if cfg!(feature = "test-local") { + env::host(nodes, &docker_client).await? + } else { + env::docker(nodes, &docker_client).await? + }; f(TestContext { - pk_set, - leader_node: leader_node.api(), - signer_nodes: signer_nodes.iter().map(|n| n.api()).collect(), - worker: relayer_ctx.worker.clone(), - gcp_datastore_url: datastore.local_address, + pk_set: nodes.pk_set(), + leader_node: nodes.leader_api(), + signer_nodes: nodes.signer_apis(), + worker: nodes.ctx().relayer_ctx.worker.clone(), + gcp_datastore_url: nodes.datastore_addr(), }) .await?; - relayer_ctx.relayer.clean_tmp_files()?; + nodes.ctx().relayer_ctx.relayer.clean_tmp_files()?; Ok(()) } From 869c39ae7f4032181aca8eb336ac5f2f3e445d7f Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 21:05:13 -0700 Subject: [PATCH 05/21] Moved containers.rs and local.rs into env/ --- integration-tests/src/{ => env}/containers.rs | 0 integration-tests/src/{ => env}/local.rs | 0 integration-tests/src/env/mod.rs | 7 +++++-- integration-tests/src/lib.rs | 4 ++-- integration-tests/src/main.rs | 2 +- integration-tests/tests/lib.rs | 5 +---- 6 files changed, 9 insertions(+), 9 deletions(-) rename integration-tests/src/{ => env}/containers.rs (100%) rename integration-tests/src/{ => env}/local.rs (100%) diff --git a/integration-tests/src/containers.rs b/integration-tests/src/env/containers.rs similarity index 100% rename from integration-tests/src/containers.rs rename to integration-tests/src/env/containers.rs diff --git a/integration-tests/src/local.rs b/integration-tests/src/env/local.rs similarity index 100% rename from integration-tests/src/local.rs rename to integration-tests/src/env/local.rs diff --git a/integration-tests/src/env/mod.rs b/integration-tests/src/env/mod.rs index 34b93e261..8623e5832 100644 --- a/integration-tests/src/env/mod.rs +++ b/integration-tests/src/env/mod.rs @@ -1,9 +1,12 @@ +pub mod containers; +pub mod local; + use curv::elliptic::curves::{Ed25519, Point}; use mpc_recovery::GenerateResult; use near_primitives::utils::generate_random_string; -use crate::containers::{DockerClient, LeaderNodeApi, SignerNodeApi}; -use crate::{containers, initialize_relayer, local, util, RelayerCtx}; +use crate::env::containers::{DockerClient, LeaderNodeApi, SignerNodeApi}; +use crate::{initialize_relayer, util, RelayerCtx}; pub const NETWORK: &str = "mpc_it_network"; pub const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs index b5669f5a2..649094248 100644 --- a/integration-tests/src/lib.rs +++ b/integration-tests/src/lib.rs @@ -7,9 +7,9 @@ use workspaces::{ Account, Worker, }; -pub mod containers; +use crate::env::containers; + pub mod env; -pub mod local; pub mod sandbox; pub mod util; diff --git a/integration-tests/src/main.rs b/integration-tests/src/main.rs index 0592ab4b3..51f935aaf 100644 --- a/integration-tests/src/main.rs +++ b/integration-tests/src/main.rs @@ -1,6 +1,6 @@ use clap::Parser; use mpc_recovery::GenerateResult; -use mpc_recovery_integration_tests::containers; +use mpc_recovery_integration_tests::env::containers; use near_primitives::utils::generate_random_string; use tokio::io::{stdin, AsyncReadExt}; use tracing_subscriber::EnvFilter; diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index 1f107289f..179a1147c 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -8,10 +8,7 @@ use mpc_recovery::{ ClaimOidcResponse, MpcPkResponse, NewAccountResponse, SignResponse, UserCredentialsResponse, }, }; -use mpc_recovery_integration_tests::{ - containers, - env::{self, GCP_PROJECT_ID}, -}; +use mpc_recovery_integration_tests::env::{self, containers, GCP_PROJECT_ID}; use workspaces::{network::Sandbox, Worker}; pub struct TestContext { From 9d967f3aa79bd017a085402e92f5d38fef6154b6 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 21:33:04 -0700 Subject: [PATCH 06/21] Change feature flag to docker-test --- integration-tests/Cargo.toml | 4 ++-- integration-tests/tests/lib.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index f6386b25b..38c6ab60f 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -41,6 +41,6 @@ tokio-util = { version = "0.7", features = ["full"] } reqwest = "0.11.16" [features] -default = ["drop-containers", "test-local"] +default = ["drop-containers"] drop-containers = [] -test-local = [] +docker-test = [] diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index 179a1147c..0225cefe2 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -36,10 +36,10 @@ where Fut: core::future::Future>, { let docker_client = containers::DockerClient::default(); - let nodes = if cfg!(feature = "test-local") { - env::host(nodes, &docker_client).await? - } else { + let nodes = if cfg!(feature = "docker-test") { env::docker(nodes, &docker_client).await? + } else { + env::host(nodes, &docker_client).await? }; f(TestContext { From 4895ad5600867d8718c33189de1b812b024aa2b7 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 21:38:29 -0700 Subject: [PATCH 07/21] Fix docker test --- integration-tests/src/env/containers.rs | 26 ++++++++++++------------- integration-tests/src/env/local.rs | 14 ++++++------- integration-tests/src/env/mod.rs | 24 ++++++++--------------- 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index 6a2778485..a330525aa 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -44,6 +44,8 @@ use std::fs; use crate::util::{self, create_key_file, create_relayer_cofig_file}; +use super::Context; + static NETWORK_MUTEX: Lazy> = Lazy::new(|| Mutex::new(0)); pub struct DockerClient { @@ -586,8 +588,8 @@ pub struct LeaderNode<'a> { pub container: Container<'a, GenericImage>, pub address: String, local_address: String, - near_rpc: String, - relayer_url: String, + local_rpc_url: String, + local_relayer_url: String, } pub struct LeaderNodeApi { @@ -604,9 +606,7 @@ impl<'a> LeaderNode<'a> { docker_client: &'a DockerClient, network: &str, sign_nodes: Vec, - near_rpc: &str, - relayer_url: &str, - datastore_url: &str, + ctx: &Context<'a>, gcp_project_id: &str, near_root_account: &AccountId, account_creator_id: &AccountId, @@ -624,7 +624,7 @@ impl<'a> LeaderNode<'a> { "--web-port".to_string(), Self::CONTAINER_PORT.to_string(), "--near-rpc".to_string(), - near_rpc.to_string(), + ctx.relayer_ctx.sandbox.address.to_string(), "--near-root-account".to_string(), near_root_account.to_string(), "--account-creator-id".to_string(), @@ -639,7 +639,7 @@ impl<'a> LeaderNode<'a> { "audience": firebase_audience_id, }, "relayer": { - "url": relayer_url.to_string(), + "url": &ctx.relayer_ctx.relayer.address, "api_key": serde_json::Value::Null, }, }, @@ -647,7 +647,7 @@ impl<'a> LeaderNode<'a> { "--gcp-project-id".to_string(), gcp_project_id.to_string(), "--gcp-datastore-url".to_string(), - datastore_url.to_string(), + ctx.datastore.address.to_string(), "--test".to_string(), ]; for sign_node in sign_nodes { @@ -673,17 +673,17 @@ impl<'a> LeaderNode<'a> { container, address: full_address, local_address: format!("http://localhost:{host_port}"), - near_rpc: near_rpc.to_string(), - relayer_url: relayer_url.to_string(), + local_rpc_url: ctx.relayer_ctx.sandbox.local_address.clone(), + local_relayer_url: ctx.relayer_ctx.relayer.local_address.clone(), }) } pub fn api(&self) -> LeaderNodeApi { LeaderNodeApi { - address: self.address.clone(), - client: NearRpcAndRelayerClient::connect(&self.near_rpc), + address: self.local_address.clone(), + client: NearRpcAndRelayerClient::connect(&self.local_rpc_url), relayer: DelegateActionRelayer { - url: self.relayer_url.clone(), + url: self.local_relayer_url.clone(), api_key: None, }, } diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index 37f735d2b..bd2b28490 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -130,9 +130,7 @@ impl LeaderNode { pub async fn run( web_port: u16, sign_nodes: Vec, - near_rpc: &str, - relayer_url: &str, - datastore_url: &str, + ctx: &super::Context<'_>, gcp_project_id: &str, near_root_account: &workspaces::AccountId, account_creator_id: &workspaces::AccountId, @@ -150,7 +148,7 @@ impl LeaderNode { "--web-port".to_string(), web_port.to_string(), "--near-rpc".to_string(), - near_rpc.to_string(), + ctx.relayer_ctx.sandbox.local_address.clone(), "--near-root-account".to_string(), near_root_account.to_string(), "--account-creator-id".to_string(), @@ -165,7 +163,7 @@ impl LeaderNode { "audience": firebase_audience_id, }, "relayer": { - "url": relayer_url.to_string(), + "url": &ctx.relayer_ctx.relayer.local_address, "api_key": serde_json::Value::Null, }, }, @@ -173,7 +171,7 @@ impl LeaderNode { "--gcp-project-id".to_string(), gcp_project_id.to_string(), "--gcp-datastore-url".to_string(), - datastore_url.to_string(), + ctx.datastore.local_address.to_string(), "--test".to_string(), ]; for sign_node in sign_nodes { @@ -202,8 +200,8 @@ impl LeaderNode { tracing::info!("Leader node container is running at {address}"); Ok(Self { address, - near_rpc: near_rpc.to_string(), - relayer_url: relayer_url.to_string(), + near_rpc: ctx.relayer_ctx.sandbox.local_address.clone(), + relayer_url: ctx.relayer_ctx.relayer.local_address.clone(), process: child, }) } diff --git a/integration-tests/src/env/mod.rs b/integration-tests/src/env/mod.rs index 8623e5832..9b2bfb788 100644 --- a/integration-tests/src/env/mod.rs +++ b/integration-tests/src/env/mod.rs @@ -58,11 +58,7 @@ impl Nodes<'_> { } pub fn datastore_addr(&self) -> String { - // this is different per env: - match self { - Nodes::Local { ctx, .. } => ctx.datastore.local_address.clone(), - Nodes::Docker { ctx, .. } => ctx.datastore.address.clone(), - } + self.ctx().datastore.local_address.clone() } } @@ -71,12 +67,12 @@ pub struct Context<'a> { pub datastore: containers::Datastore<'a>, } -pub async fn setup<'a>(docker_client: &'a DockerClient) -> anyhow::Result> { +pub async fn setup(docker_client: &DockerClient) -> anyhow::Result> { docker_client.create_network(NETWORK).await?; let relayer_id = generate_random_string(7); // used to distinguish relayer tmp files in multiple tests - let relayer_ctx_future = initialize_relayer(&docker_client, NETWORK, &relayer_id); - let datastore_future = containers::Datastore::run(&docker_client, NETWORK, GCP_PROJECT_ID); + let relayer_ctx_future = initialize_relayer(docker_client, NETWORK, &relayer_id); + let datastore_future = containers::Datastore::run(docker_client, NETWORK, GCP_PROJECT_ID); let (relayer_ctx, datastore) = futures::future::join(relayer_ctx_future, datastore_future).await; @@ -96,7 +92,7 @@ pub async fn docker(nodes: usize, docker_client: &DockerClient) -> anyhow::Resul let mut signer_node_futures = Vec::with_capacity(nodes); for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { signer_node_futures.push(containers::SignerNode::run_signing_node( - &docker_client, + docker_client, NETWORK, i as u64, share, @@ -115,12 +111,10 @@ pub async fn docker(nodes: usize, docker_client: &DockerClient) -> anyhow::Resul let near_root_account = ctx.relayer_ctx.worker.root_account()?; let leader_node = containers::LeaderNode::run( - &docker_client, + docker_client, NETWORK, signer_urls.clone(), - &ctx.relayer_ctx.sandbox.address, - &ctx.relayer_ctx.relayer.address, - &ctx.datastore.address, + &ctx, GCP_PROJECT_ID, near_root_account.id(), ctx.relayer_ctx.creator_account.id(), @@ -162,9 +156,7 @@ pub async fn host(nodes: usize, docker_client: &DockerClient) -> anyhow::Result< let leader_node = local::LeaderNode::run( util::pick_unused_port().await?, signer_nodes.iter().map(|n| n.address.clone()).collect(), - &ctx.relayer_ctx.sandbox.local_address, - &ctx.relayer_ctx.relayer.local_address, - &ctx.datastore.local_address, + &ctx, GCP_PROJECT_ID, near_root_account.id(), ctx.relayer_ctx.creator_account.id(), From 2d496cf100b6cd0496cd81f75d50efff1c78daa3 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 22:24:22 -0700 Subject: [PATCH 08/21] Consistent way to create CLI args --- integration-tests/src/env/containers.rs | 88 ++++---- integration-tests/src/env/local.rs | 86 ++++---- mpc-recovery/src/lib.rs | 261 ++++++++++++++---------- 3 files changed, 227 insertions(+), 208 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index a330525aa..e42249351 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -478,33 +478,30 @@ impl<'a> SignerNode<'a> { .with_wait_for(WaitFor::Nothing) .with_exposed_port(Self::CONTAINER_PORT) .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); - let image: RunnableImage = ( - image, - vec![ - "start-sign".to_string(), - "--node-id".to_string(), - node_id.to_string(), - "--sk-share".to_string(), - serde_json::to_string(&sk_share)?, - "--cipher-key".to_string(), - hex::encode(cipher_key), - "--web-port".to_string(), - Self::CONTAINER_PORT.to_string(), - "--oidc-providers".to_string(), + + let args = mpc_recovery::Cli::StartSign { + env: "dev".to_string(), + node_id, + web_port: Self::CONTAINER_PORT, + sk_share: Some(serde_json::to_string(&sk_share)?), + cipher_key: Some(hex::encode(cipher_key)), + oidc_providers_filepath: None, + oidc_providers: Some( serde_json::json!([ { - "issuer": format!("https://securetoken.google.com/{}", firebase_audience_id), + "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), "audience": firebase_audience_id, }, - ]).to_string(), - "--gcp-project-id".to_string(), - gcp_project_id.to_string(), - "--gcp-datastore-url".to_string(), - datastore_url.to_string(), - "--test".to_string(), - ], - ) - .into(); + ]) + .to_string(), + ), + gcp_project_id: gcp_project_id.to_string(), + gcp_datastore_url: Some(datastore_url.to_string()), + test: true, + } + .into_str_args(); + + let image: RunnableImage = (image, args).into(); let image = image.with_network(network); let container = docker_client.cli.run(image); let ip_address = docker_client @@ -619,23 +616,19 @@ impl<'a> LeaderNode<'a> { .with_wait_for(WaitFor::Nothing) .with_exposed_port(Self::CONTAINER_PORT) .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); - let mut cmd = vec![ - "start-leader".to_string(), - "--web-port".to_string(), - Self::CONTAINER_PORT.to_string(), - "--near-rpc".to_string(), - ctx.relayer_ctx.sandbox.address.to_string(), - "--near-root-account".to_string(), - near_root_account.to_string(), - "--account-creator-id".to_string(), - account_creator_id.to_string(), - "--account-creator-sk".to_string(), - account_creator_sk.to_string(), - "--fast-auth-partners".to_string(), - serde_json::json!([ + + let args = mpc_recovery::Cli::StartLeader { + env: "dev".to_string(), + web_port: Self::CONTAINER_PORT, + sign_nodes, + near_rpc: ctx.relayer_ctx.sandbox.address.clone(), + near_root_account: near_root_account.to_string(), + account_creator_id: account_creator_id.clone(), + account_creator_sk: Some(account_creator_sk.to_string()), + fast_auth_partners: Some(serde_json::json!([ { "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{}", firebase_audience_id), + "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), "audience": firebase_audience_id, }, "relayer": { @@ -643,18 +636,15 @@ impl<'a> LeaderNode<'a> { "api_key": serde_json::Value::Null, }, }, - ]).to_string(), - "--gcp-project-id".to_string(), - gcp_project_id.to_string(), - "--gcp-datastore-url".to_string(), - ctx.datastore.address.to_string(), - "--test".to_string(), - ]; - for sign_node in sign_nodes { - cmd.push("--sign-nodes".to_string()); - cmd.push(sign_node); + ]).to_string()), + fast_auth_partners_filepath: None, + gcp_project_id: gcp_project_id.to_string(), + gcp_datastore_url: Some(ctx.datastore.address.to_string()), + test: true, } - let image: RunnableImage = (image, cmd).into(); + .into_str_args(); + + let image: RunnableImage = (image, args).into(); let image = image.with_network(network); let container = docker_client.cli.run(image); let ip_address = docker_client diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index bd2b28490..8756b4692 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -42,30 +42,28 @@ impl SignerNode { .context("could not find target dir while running signing node")? .join(if release { "release" } else { "debug" }) .join(EXECUTABLE); - let args = vec![ - "start-sign".to_string(), - "--node-id".to_string(), - node_id.to_string(), - "--sk-share".to_string(), - serde_json::to_string(&sk_share)?, - "--cipher-key".to_string(), - hex::encode(cipher_key), - "--web-port".to_string(), - web_port.to_string(), - "--oidc-providers".to_string(), - serde_json::json!([ - { - "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), - "audience": firebase_audience_id, - }, - ]) - .to_string(), - "--gcp-project-id".to_string(), - gcp_project_id.to_string(), - "--gcp-datastore-url".to_string(), - datastore_url.to_string(), - "--test".to_string(), - ]; + + let args = mpc_recovery::Cli::StartSign { + env: "dev".to_string(), + node_id, + web_port, + sk_share: Some(serde_json::to_string(&sk_share)?), + cipher_key: Some(hex::encode(cipher_key)), + oidc_providers_filepath: None, + oidc_providers: Some( + serde_json::json!([ + { + "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), + "audience": firebase_audience_id, + }, + ]) + .to_string(), + ), + gcp_project_id: gcp_project_id.to_string(), + gcp_datastore_url: Some(datastore_url.to_string()), + test: true, + } + .into_str_args(); let address = format!("http://localhost:{web_port}"); let child = Command::new(&executable) @@ -143,20 +141,16 @@ impl LeaderNode { .context("could not find target dir while running leader node")? .join(if release { "release" } else { "debug" }) .join(EXECUTABLE); - let mut args = vec![ - "start-leader".to_string(), - "--web-port".to_string(), - web_port.to_string(), - "--near-rpc".to_string(), - ctx.relayer_ctx.sandbox.local_address.clone(), - "--near-root-account".to_string(), - near_root_account.to_string(), - "--account-creator-id".to_string(), - account_creator_id.to_string(), - "--account-creator-sk".to_string(), - account_creator_sk.to_string(), - "--fast-auth-partners".to_string(), - serde_json::json!([ + + let args = mpc_recovery::Cli::StartLeader { + env: "dev".to_string(), + web_port, + sign_nodes, + near_rpc: ctx.relayer_ctx.sandbox.local_address.clone(), + near_root_account: near_root_account.to_string(), + account_creator_id: account_creator_id.clone(), + account_creator_sk: Some(account_creator_sk.to_string()), + fast_auth_partners: Some(serde_json::json!([ { "oidc_provider": { "issuer": format!("https://securetoken.google.com/{}", firebase_audience_id), @@ -167,17 +161,13 @@ impl LeaderNode { "api_key": serde_json::Value::Null, }, }, - ]).to_string(), - "--gcp-project-id".to_string(), - gcp_project_id.to_string(), - "--gcp-datastore-url".to_string(), - ctx.datastore.local_address.to_string(), - "--test".to_string(), - ]; - for sign_node in sign_nodes { - args.push("--sign-nodes".to_string()); - args.push(sign_node); + ]).to_string()), + fast_auth_partners_filepath: None, + gcp_project_id: gcp_project_id.to_string(), + gcp_datastore_url: Some(ctx.datastore.local_address.to_string()), + test: true, } + .into_str_args(); let child = Command::new(&executable) .args(&args) diff --git a/mpc-recovery/src/lib.rs b/mpc-recovery/src/lib.rs index 995921434..36337eea3 100644 --- a/mpc-recovery/src/lib.rs +++ b/mpc-recovery/src/lib.rs @@ -406,114 +406,153 @@ where Ok(entries) } -// impl Cli { - -// fn generate_command_args(self) -> Vec { -// match self { -// Cli::Generate { n } => { -// command_args.push("--n".to_string()); -// command_args.push(n.to_string()); -// } -// Cli::StartLeader { -// env, -// web_port, -// sign_nodes, -// near_rpc, -// near_root_account, -// account_creator_id, -// account_creator_sk, -// fast_auth_partners, -// fast_auth_partners_filepath, -// gcp_project_id, -// gcp_datastore_url, -// test, -// } => { -// // command_args.push("--env".to_string()); -// // command_args.push(env); -// // command_args.push("--web_port".to_string()); -// // command_args.push(web_port.to_string()); - -// // for sign_node in sign_nodes { -// // command_args.push("--sign_nodes".to_string()); -// // command_args.push(sign_node); -// // } - -// // command_args.push("--near_rpc".to_string()); -// // command_args.push(near_rpc); -// // command_args.push("--near_root_account".to_string()); -// // command_args.push(near_root_account); -// // // ... - -// vec![ -// "--env".into(), env, -// "--web_port".into(), web_port.to_string(), -// ] - -// } -// Cli::StartSign { -// env, -// node_id, -// cipher_key, -// sk_share, -// web_port, -// oidc_providers, -// // ... -// } => { -// command_args.push("--env".to_string()); -// command_args.push(env); -// command_args.push("--node_id".to_string()); -// command_args.push(node_id.to_string(); - -// if let Some(key) = cipher_key { -// command_args.push("--cipher_key".to_string()); -// command_args.push(key); -// } - -// if let Some(share) = sk_share { -// command_args.push("--sk_share".to_string()); -// command_args.push(share); -// } - -// command_args.push("--web_port".to_string()); -// command_args.push(web_port.to_string()); - -// if let Some(providers) = oidc_providers { -// command_args.push("--oidc_providers".to_string()); -// command_args.push(providers); -// } -// // ... -// } -// Cli::RotateSignNodeCipher { -// env, -// new_env, -// node_id, -// old_cipher_key, -// new_cipher_key, -// gcp_project_id, -// gcp_datastore_url, -// } => { -// command_args.push("--env".to_string()); -// command_args.push(env); - -// if let Some(new_env) = new_env { -// command_args.push("--new_env".to_string()); -// command_args.push(new_env); -// } - -// command_args.push("--node_id".to_string()); -// command_args.push(node_id.to_string(); - -// if let Some(old_key) = old_cipher_key { -// command_args.push("--old_cipher_key".to_string()); -// command_args.push(old_key); -// } - -// if let Some(new_key) = new_cipher_key { -// command_args.push("--new_cipher_key".to_string()); -// command_args.push(new_key); -// } -// } -// } -// } - -// } +impl Cli { + pub fn into_str_args(self) -> Vec { + match self { + Cli::Generate { n } => { + vec!["generate".to_string(), n.to_string()] + } + Cli::StartLeader { + env, + web_port, + sign_nodes, + near_rpc, + near_root_account, + account_creator_id, + account_creator_sk, + fast_auth_partners, + fast_auth_partners_filepath, + gcp_project_id, + gcp_datastore_url, + test, + } => { + let mut buf = vec![ + "start-leader".to_string(), + "--env".to_string(), + env.to_string(), + "--web-port".to_string(), + web_port.to_string(), + "--near-rpc".to_string(), + near_rpc, + "--near-root-account".to_string(), + near_root_account, + "--account-creator-id".to_string(), + account_creator_id.to_string(), + "--gcp-project-id".to_string(), + gcp_project_id, + ]; + + if let Some(key) = account_creator_sk { + buf.push("--account-creator-sk".to_string()); + buf.push(key); + } + if let Some(partners) = fast_auth_partners { + buf.push("--fast-auth-partners".to_string()); + buf.push(partners); + } + if let Some(partners_filepath) = fast_auth_partners_filepath { + buf.push("--fast-auth-partners-filepath".to_string()); + buf.push(partners_filepath.to_str().unwrap().to_string()); + } + if let Some(gcp_datastore_url) = gcp_datastore_url { + buf.push("--gcp-datastore-url".to_string()); + buf.push(gcp_datastore_url); + } + if test { + buf.push("--test".to_string()); + } + for sign_node in sign_nodes { + buf.push("--sign-nodes".to_string()); + buf.push(sign_node); + } + buf + } + Cli::StartSign { + env, + node_id, + web_port, + cipher_key, + sk_share, + oidc_providers, + oidc_providers_filepath, + gcp_project_id, + gcp_datastore_url, + test, + } => { + let mut buf = vec![ + "start-sign".to_string(), + "--env".to_string(), + env.to_string(), + "--node-id".to_string(), + node_id.to_string(), + "--web-port".to_string(), + web_port.to_string(), + "--gcp-project-id".to_string(), + gcp_project_id, + ]; + if let Some(key) = cipher_key { + buf.push("--cipher-key".to_string()); + buf.push(key); + } + if let Some(share) = sk_share { + buf.push("--sk-share".to_string()); + buf.push(share); + } + if let Some(providers) = oidc_providers { + buf.push("--oidc-providers".to_string()); + buf.push(providers); + } + if let Some(providers_filepath) = oidc_providers_filepath { + buf.push("--oidc-providers-filepath".to_string()); + buf.push(providers_filepath.to_str().unwrap().to_string()); + } + if let Some(gcp_datastore_url) = gcp_datastore_url { + buf.push("--gcp-datastore-url".to_string()); + buf.push(gcp_datastore_url); + } + if test { + buf.push("--test".to_string()); + } + + buf + } + Cli::RotateSignNodeCipher { + env, + new_env, + node_id, + old_cipher_key, + new_cipher_key, + gcp_project_id, + gcp_datastore_url, + } => { + let mut buf = vec![ + "rotate-sign-node-cipher".to_string(), + "--env".to_string(), + env.to_string(), + "--node-id".to_string(), + node_id.to_string(), + "--gcp-project-id".to_string(), + gcp_project_id, + ]; + if let Some(new_env) = new_env { + buf.push("--new-env".to_string()); + buf.push(new_env); + } + if let Some(old_cipher_key) = old_cipher_key { + buf.push("--old-cipher-key".to_string()); + buf.push(old_cipher_key); + } + if let Some(new_cipher_key) = new_cipher_key { + buf.push("--new-cipher-key".to_string()); + buf.push(new_cipher_key); + } + if let Some(gcp_datastore_url) = gcp_datastore_url { + buf.push("--gcp-datastore-url".to_string()); + buf.push(gcp_datastore_url); + } + + buf + } + } + } +} From 920b895e4d00619e4d9d0f1fc7a63842239e4eb7 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Wed, 11 Oct 2023 22:26:09 -0700 Subject: [PATCH 09/21] Removed need for integration-tests/main.rs --- integration-tests/src/main.rs | 162 ---------------------------------- 1 file changed, 162 deletions(-) delete mode 100644 integration-tests/src/main.rs diff --git a/integration-tests/src/main.rs b/integration-tests/src/main.rs deleted file mode 100644 index 51f935aaf..000000000 --- a/integration-tests/src/main.rs +++ /dev/null @@ -1,162 +0,0 @@ -use clap::Parser; -use mpc_recovery::GenerateResult; -use mpc_recovery_integration_tests::env::containers; -use near_primitives::utils::generate_random_string; -use tokio::io::{stdin, AsyncReadExt}; -use tracing_subscriber::EnvFilter; - -const NETWORK: &str = "mpc_recovery_dev_network"; -const GCP_PROJECT_ID: &str = "mpc-recovery-dev-gcp-project"; -pub const FIREBASE_AUDIENCE_ID: &str = "test_audience"; - -#[derive(Parser, Debug)] -enum Cli { - TestLeader { nodes: usize }, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let subscriber = tracing_subscriber::fmt() - .with_thread_ids(true) - .with_env_filter(EnvFilter::from_default_env()); - subscriber.init(); - match Cli::parse() { - Cli::TestLeader { nodes } => { - tracing::info!("Setting up an environment with {} nodes", nodes); - let docker_client = containers::DockerClient::default(); - - let relayer_id = generate_random_string(7); - let relayer_ctx_future = mpc_recovery_integration_tests::initialize_relayer( - &docker_client, - NETWORK, - &relayer_id, - ); - let datastore_future = - containers::Datastore::run(&docker_client, NETWORK, GCP_PROJECT_ID); - - let (relayer_ctx, datastore) = - futures::future::join(relayer_ctx_future, datastore_future).await; - let relayer_ctx = relayer_ctx?; - let datastore = datastore?; - - tracing::info!("Generating secrets"); - let GenerateResult { secrets, .. } = mpc_recovery::generate(nodes); - tracing::info!("Running signer nodes..."); - let mut signer_node_futures = Vec::new(); - for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { - let signer_node = containers::SignerNode::run_signing_node( - &docker_client, - NETWORK, - i as u64, - share, - cipher_key, - &datastore.address, - &datastore.local_address, - GCP_PROJECT_ID, - FIREBASE_AUDIENCE_ID, - ); - signer_node_futures.push(signer_node); - } - let signer_nodes = futures::future::join_all(signer_node_futures) - .await - .into_iter() - .collect::, _>>()?; - tracing::info!("Signer nodes initialized"); - let signer_urls: &Vec<_> = &signer_nodes.iter().map(|n| n.address.clone()).collect(); - - let near_root_account = relayer_ctx.worker.root_account()?; - tracing::info!("Root account_id: {}", near_root_account.id()); - - let mut cmd = vec![ - "start-leader".to_string(), - "--web-port".to_string(), - "3000".to_string(), - "--near-rpc".to_string(), - format!( - "http://localhost:{}", - relayer_ctx - .sandbox - .container - .get_host_port_ipv4(containers::Sandbox::CONTAINER_RPC_PORT) - ), - "--near-root-account".to_string(), - near_root_account.id().to_string(), - "--account-creator-id".to_string(), - relayer_ctx.creator_account.id().to_string(), - "--account-creator-sk".to_string(), - relayer_ctx.creator_account.secret_key().to_string(), - "--gcp-project-id".to_string(), - GCP_PROJECT_ID.to_string(), - "--gcp-datastore-url".to_string(), - format!( - "http://localhost:{}", - datastore - .container - .get_host_port_ipv4(containers::Datastore::CONTAINER_PORT) - ), - "--fast-auth-partners".to_string(), - escape_json_string(&serde_json::json!([ - { - "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{}", FIREBASE_AUDIENCE_ID.to_string()), - "audience": FIREBASE_AUDIENCE_ID.to_string(), - }, - "relayer": { - "url": format!( - "http://localhost:{}", - relayer_ctx - .relayer - .container - .get_host_port_ipv4(containers::Relayer::CONTAINER_PORT) - ), - "api_key": serde_json::Value::Null, - }, - }, - ]).to_string()), - "--test".to_string(), - ]; - for sign_node in signer_urls { - cmd.push("--sign-nodes".to_string()); - cmd.push(sign_node.clone()); - } - - tracing::info!("Please run the command below to start a leader node:"); - tracing::info!( - "RUST_LOG=mpc_recovery=debug cargo run --bin mpc-recovery -- {}", - cmd.join(" ") - ); - tracing::info!("===================================="); - tracing::info!("You can now interact with your local service manually. For example:"); - tracing::info!( - r#"curl -X POST -H "Content-Type: application/json" -d '{{"oidc_token": "validToken:1", "near_account_id": "abc45436676.near", "create_account_options": {{"full_access_keys": ["ed25519:4fnCz9NTEMhkfwAHDhFDkPS1mD58QHdRyago5n4vtCS2"]}}}}' http://localhost:3000/new_account"# - ); - - tracing::info!("Press any button to exit and destroy all containers..."); - - while stdin().read(&mut [0]).await? == 0 {} - } - }; - - Ok(()) -} - -fn escape_json_string(input: &str) -> String { - let mut result = String::with_capacity(input.len() + 2); - result.push('"'); - - for c in input.chars() { - match c { - '"' => result.push_str(r"\\"), - '\\' => result.push_str(r"\\"), - '\n' => result.push_str(r"\n"), - '\r' => result.push_str(r"\r"), - '\t' => result.push_str(r"\t"), - '\u{08}' => result.push_str(r"\b"), // Backspace - '\u{0C}' => result.push_str(r"\f"), // Form feed - _ => result.push(c), - } - } - - result.push('"'); - result -} From 46b615adb9b82e1f48ff7996dd20a0de26665d69 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Thu, 12 Oct 2023 15:14:33 -0700 Subject: [PATCH 10/21] Update docs and cleanup --- integration-tests/README.md | 26 +++++++++++++++++++++----- integration-tests/src/env/local.rs | 17 ++++++----------- integration-tests/src/util.rs | 9 +++++++++ 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index c7b634d23..3eda17194 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -5,23 +5,38 @@ Running integration tests requires you to have relayer and sandbox docker images present on your machine: ```BASH -docker pull ghcr.io/near/pagoda-relayer-rs-fastauth +docker pull ghcr.io/near/os-relayer docker pull ghcr.io/near/sandbox ``` + In case of authorization issues make sure you have logged into docker using your [access token](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-with-a-personal-access-token-classic). Now, build mpc-recovery from the project's root: +```BASH +cargo build --release +``` + +Then run the integration tests: + +```BASH +cargo test -p mpc-recovery-integration-tests +``` + +### Alternative: Docker Builds/Tests + +If instead, we need to run docker build/tests: + ```BASH docker build . -t near/mpc-recovery ``` **Note**. You will need to re-build the Docker image each time you make a code change and want to run the integration tests. -Finally, run the integration tests: +Finally, run the integration tests with the built docker image: ```BASH -cargo test -p mpc-recovery-integration-tests +cargo test -p mpc-recovery-integration-tests --features docker-test ``` ## FAQ @@ -50,7 +65,7 @@ b2724d0c9530 near/mpc-recovery:latest "mpc-recovery st 67308ab06c5d near/mpc-recovery:latest "mpc-recovery start-…" 5 minutes ago Up 5 minutes 0.0.0.0:32791->3000/tcp, :::32791->3000/tcp upbeat_volhard 65ec65384af4 near/mpc-recovery:latest "mpc-recovery start-…" 5 minutes ago Up 5 minutes 0.0.0.0:32790->3000/tcp, :::32790->3000/tcp friendly_easley b4f90b1546ec near/mpc-recovery:latest "mpc-recovery start-…" 5 minutes ago Up 5 minutes 0.0.0.0:32789->3000/tcp, :::32789->3000/tcp vibrant_allen -934ec13d9146 ghcr.io/near/pagoda-relayer-rs-fastauth:latest "/usr/local/bin/entr…" 5 minutes ago Up 5 minutes 0.0.0.0:32788->16581/tcp, :::32788->16581/tcp sleepy_grothendieck +934ec13d9146 ghcr.io/near/os-relayer:latest "/usr/local/bin/entr…" 5 minutes ago Up 5 minutes 0.0.0.0:32788->16581/tcp, :::32788->16581/tcp sleepy_grothendieck c505ead6eb18 redis:latest "docker-entrypoint.s…" 5 minutes ago Up 5 minutes 0.0.0.0:32787->6379/tcp, :::32787->6379/tcp trusting_lederberg 2843226b16a9 google/cloud-sdk:latest "gcloud beta emulato…" 5 minutes ago Up 5 minutes 0.0.0.0:32786->15805/tcp, :::32786->15805/tcp hungry_pasteur 3f4c70020a4c ghcr.io/near/sandbox:latest "near-sandbox --home…" 5 minutes ago Up 5 minutes practical_elbakyan @@ -67,8 +82,9 @@ $ cargo run -p mpc-recovery-integration-tests -- test-leader 3 ``` ### I'm getting "Error: error trying to connect: No such file or directory (os error 2)" + It's a known issue on MacOS. Try executiong the following command: ```bash sudo ln -s $HOME/.docker/run/docker.sock /var/run/docker.sock -``` \ No newline at end of file +``` diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index 1dd25cca6..c66cce937 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -12,8 +12,6 @@ use multi_party_eddsa::protocols::ExpandedKeyPair; use crate::containers::{LeaderNodeApi, SignerNodeApi}; use crate::util; -const EXECUTABLE: &str = "mpc-recovery"; - pub struct SignerNode { pub address: String, node_id: usize, @@ -38,10 +36,8 @@ impl SignerNode { firebase_audience_id: &str, release: bool, ) -> anyhow::Result { - let executable = util::target_dir() - .context("could not find target dir while running signing node")? - .join(if release { "release" } else { "debug" }) - .join(EXECUTABLE); + let executable = util::executable(release) + .context("could not find target dir while running signing node")?; let args = mpc_recovery::Cli::StartSign { env: "dev".to_string(), @@ -86,7 +82,7 @@ impl SignerNode { let x: anyhow::Result = util::get(&address).await; match x { std::result::Result::Ok(status) if status == StatusCode::OK => break, - _err => {} + _ => (), } tokio::time::sleep(std::time::Duration::from_secs(1)).await; } @@ -124,6 +120,7 @@ pub struct LeaderNode { #[allow(unused)] process: Child, } + impl LeaderNode { pub async fn run( web_port: u16, @@ -137,10 +134,8 @@ impl LeaderNode { release: bool, ) -> anyhow::Result { tracing::info!("Running leader node..."); - let executable = util::target_dir() - .context("could not find target dir while running leader node")? - .join(if release { "release" } else { "debug" }) - .join(EXECUTABLE); + let executable = util::executable(release) + .context("could not find target dir while running leader node")?; let args = mpc_recovery::Cli::StartLeader { env: "dev".to_string(), diff --git a/integration-tests/src/util.rs b/integration-tests/src/util.rs index 6c11a0fe8..3ae82bb78 100644 --- a/integration-tests/src/util.rs +++ b/integration-tests/src/util.rs @@ -12,6 +12,8 @@ use workspaces::{types::SecretKey, AccountId}; use crate::containers::RelayerConfig; +const EXECUTABLE: &str = "mpc-recovery"; + pub async fn post( uri: U, request: Req, @@ -220,3 +222,10 @@ pub fn target_dir() -> Option { } } } + +pub fn executable(release: bool) -> Option { + let executable = target_dir()? + .join(if release { "release" } else { "debug" }) + .join(EXECUTABLE); + Some(executable) +} From d68e11f2e6450e749e44f9cd88a0ce774c3bb9a5 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Oct 2023 13:55:14 -0700 Subject: [PATCH 11/21] Cleanup --- integration-tests/src/env/containers.rs | 101 ++++++++++-------------- integration-tests/src/env/local.rs | 52 ++++++------ integration-tests/src/env/mod.rs | 73 ++++++++++------- integration-tests/tests/lib.rs | 13 +-- 4 files changed, 117 insertions(+), 122 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index 8cc58b770..5d54b7b20 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -42,10 +42,9 @@ use workspaces::AccountId; use std::fs; +use crate::env::{Context, LeaderNodeApi, SignerNodeApi}; use crate::util::{self, create_key_file, create_relayer_cofig_file}; -use super::Context; - static NETWORK_MUTEX: Lazy> = Lazy::new(|| Mutex::new(0)); pub struct DockerClient { @@ -492,30 +491,15 @@ pub struct SignerNode<'a> { gcp_datastore_local_url: String, } -pub struct SignerNodeApi { - pub address: String, - pub node_id: usize, - pub sk_share: ExpandedKeyPair, - pub cipher_key: GenericArray, - pub gcp_project_id: String, - pub gcp_datastore_local_url: String, -} - -impl<'a> SignerNode<'a> { +impl SignerNode<'_> { // Container port used for the docker network, does not have to be unique const CONTAINER_PORT: u16 = 3000; - pub async fn run_signing_node( - docker_client: &'a DockerClient, - network: &str, - node_id: u64, + pub async fn run<'a>( + ctx: &super::Context<'a>, + node_id: usize, sk_share: &ExpandedKeyPair, cipher_key: &GenericArray, - datastore_url: &str, - datastore_local_url: &str, - gcp_project_id: &str, - firebase_audience_id: &str, - oidc_provider_url: &str, ) -> anyhow::Result> { tracing::info!("Running signer node container {}...", node_id); let image: GenericImage = GenericImage::new("near/mpc-recovery", "latest") @@ -525,7 +509,7 @@ impl<'a> SignerNode<'a> { let args = mpc_recovery::Cli::StartSign { env: "dev".to_string(), - node_id, + node_id: node_id as u64, web_port: Self::CONTAINER_PORT, sk_share: Some(serde_json::to_string(&sk_share)?), cipher_key: Some(hex::encode(cipher_key)), @@ -533,23 +517,24 @@ impl<'a> SignerNode<'a> { oidc_providers: Some( serde_json::json!([ { - "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), - "audience": firebase_audience_id, + "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "audience": ctx.audience_id, }, ]) .to_string(), ), - gcp_project_id: gcp_project_id.to_string(), - gcp_datastore_url: Some(datastore_url.to_string()), - jwt_signature_pk_url: oidc_provider_url.to_string(), + gcp_project_id: ctx.gcp_project_id.clone(), + gcp_datastore_url: Some(ctx.datastore.address.clone()), + jwt_signature_pk_url: ctx.oidc_provider.jwt_pk_url.clone(), } .into_str_args(); let image: RunnableImage = (image, args).into(); - let image = image.with_network(network); - let container = docker_client.cli.run(image); - let ip_address = docker_client - .get_network_ip_address(&container, network) + let image = image.with_network(&ctx.docker_network); + let container = ctx.docker_client.cli.run(image); + let ip_address = ctx + .docker_client + .get_network_ip_address(&container, &ctx.docker_network) .await?; let host_port = container.get_host_port_ipv4(Self::CONTAINER_PORT); @@ -568,11 +553,11 @@ impl<'a> SignerNode<'a> { container, address: full_address, local_address: format!("http://localhost:{host_port}"), - node_id: node_id as usize, + node_id, sk_share: sk_share.clone(), cipher_key: *cipher_key, - gcp_project_id: gcp_project_id.to_string(), - gcp_datastore_local_url: datastore_local_url.to_string(), + gcp_project_id: ctx.gcp_project_id.clone(), + gcp_datastore_local_url: ctx.datastore.local_address.clone(), }) } @@ -633,26 +618,16 @@ pub struct LeaderNode<'a> { local_relayer_url: String, } -pub struct LeaderNodeApi { - pub address: String, - pub relayer: DelegateActionRelayer, - pub(crate) client: NearRpcAndRelayerClient, -} - impl<'a> LeaderNode<'a> { // Container port used for the docker network, does not have to be unique const CONTAINER_PORT: u16 = 3000; pub async fn run( - docker_client: &'a DockerClient, - network: &str, - sign_nodes: Vec, ctx: &Context<'a>, - gcp_project_id: &str, + sign_nodes: Vec, near_root_account: &AccountId, account_creator_id: &AccountId, account_creator_sk: &workspaces::types::SecretKey, - firebase_audience_id: &str, ) -> anyhow::Result> { tracing::info!("Running leader node container..."); @@ -669,30 +644,34 @@ impl<'a> LeaderNode<'a> { near_root_account: near_root_account.to_string(), account_creator_id: account_creator_id.clone(), account_creator_sk: Some(account_creator_sk.to_string()), - fast_auth_partners: Some(serde_json::json!([ - { - "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), - "audience": firebase_audience_id, - }, - "relayer": { - "url": &ctx.relayer_ctx.relayer.address, - "api_key": serde_json::Value::Null, + fast_auth_partners: Some( + serde_json::json!([ + { + "oidc_provider": { + "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "audience": ctx.audience_id, + }, + "relayer": { + "url": &ctx.relayer_ctx.relayer.address, + "api_key": serde_json::Value::Null, + }, }, - }, - ]).to_string()), + ]) + .to_string(), + ), fast_auth_partners_filepath: None, - gcp_project_id: gcp_project_id.to_string(), + gcp_project_id: ctx.gcp_project_id.clone(), gcp_datastore_url: Some(ctx.datastore.address.to_string()), jwt_signature_pk_url: ctx.oidc_provider.jwt_pk_url.to_string(), } .into_str_args(); let image: RunnableImage = (image, args).into(); - let image = image.with_network(network); - let container = docker_client.cli.run(image); - let ip_address = docker_client - .get_network_ip_address(&container, network) + let image = image.with_network(&ctx.docker_network); + let container = ctx.docker_client.cli.run(image); + let ip_address = ctx + .docker_client + .get_network_ip_address(&container, &ctx.docker_network) .await?; let host_port = container.get_host_port_ipv4(Self::CONTAINER_PORT); diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index bb4049a14..d286531d1 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -9,7 +9,7 @@ use mpc_recovery::firewall::allowed::DelegateActionRelayer; use mpc_recovery::relayer::NearRpcAndRelayerClient; use multi_party_eddsa::protocols::ExpandedKeyPair; -use crate::containers::{LeaderNodeApi, SignerNodeApi}; +use crate::env::{LeaderNodeApi, SignerNodeApi}; use crate::util; pub struct SignerNode { @@ -32,9 +32,6 @@ impl SignerNode { sk_share: &ExpandedKeyPair, cipher_key: &GenericArray, ctx: &super::Context<'_>, - datastore_url: &str, - gcp_project_id: &str, - firebase_audience_id: &str, release: bool, ) -> anyhow::Result { let executable = util::executable(release) @@ -50,14 +47,14 @@ impl SignerNode { oidc_providers: Some( serde_json::json!([ { - "issuer": format!("https://securetoken.google.com/{firebase_audience_id}"), - "audience": firebase_audience_id, + "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "audience": ctx.audience_id, }, ]) .to_string(), ), - gcp_project_id: gcp_project_id.to_string(), - gcp_datastore_url: Some(datastore_url.to_string()), + gcp_project_id: ctx.gcp_project_id.clone(), + gcp_datastore_url: Some(ctx.datastore.local_address.clone()), jwt_signature_pk_url: ctx.oidc_provider.jwt_local_url.clone(), } .into_str_args(); @@ -65,8 +62,8 @@ impl SignerNode { let address = format!("http://localhost:{web_port}"); let child = Command::new(&executable) .args(&args) - .envs(std::env::vars()) .env("RUST_LOG", "mpc_recovery=DEBUG") + .envs(std::env::vars()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) .kill_on_drop(true) @@ -94,8 +91,8 @@ impl SignerNode { node_id: node_id as usize, sk_share: sk_share.clone(), cipher_key: *cipher_key, - gcp_project_id: gcp_project_id.to_string(), - gcp_datastore_url: datastore_url.to_string(), + gcp_project_id: ctx.gcp_project_id.clone(), + gcp_datastore_url: ctx.datastore.local_address.clone(), process: child, }) } @@ -124,14 +121,12 @@ pub struct LeaderNode { impl LeaderNode { pub async fn run( + ctx: &super::Context<'_>, web_port: u16, sign_nodes: Vec, - ctx: &super::Context<'_>, - gcp_project_id: &str, near_root_account: &workspaces::AccountId, account_creator_id: &workspaces::AccountId, account_creator_sk: &workspaces::types::SecretKey, - firebase_audience_id: &str, release: bool, ) -> anyhow::Result { tracing::info!("Running leader node..."); @@ -146,21 +141,24 @@ impl LeaderNode { near_root_account: near_root_account.to_string(), account_creator_id: account_creator_id.clone(), account_creator_sk: Some(account_creator_sk.to_string()), - fast_auth_partners: Some(serde_json::json!([ - { - "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{}", firebase_audience_id), - "audience": firebase_audience_id, - }, - "relayer": { - "url": &ctx.relayer_ctx.relayer.local_address, - "api_key": serde_json::Value::Null, + fast_auth_partners: Some( + serde_json::json!([ + { + "oidc_provider": { + "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "audience": ctx.audience_id, + }, + "relayer": { + "url": &ctx.relayer_ctx.relayer.local_address, + "api_key": serde_json::Value::Null, + }, }, - }, - ]).to_string()), + ]) + .to_string(), + ), fast_auth_partners_filepath: None, - gcp_project_id: gcp_project_id.to_string(), - gcp_datastore_url: Some(ctx.datastore.local_address.to_string()), + gcp_project_id: ctx.gcp_project_id.clone(), + gcp_datastore_url: Some(ctx.datastore.local_address.clone()), jwt_signature_pk_url: ctx.oidc_provider.jwt_local_url.clone(), } .into_str_args(); diff --git a/integration-tests/src/env/mod.rs b/integration-tests/src/env/mod.rs index d0cce6754..29893e995 100644 --- a/integration-tests/src/env/mod.rs +++ b/integration-tests/src/env/mod.rs @@ -1,11 +1,17 @@ pub mod containers; pub mod local; +use aes_gcm::aead::consts::U32; +use aes_gcm::aead::generic_array::GenericArray; use curv::elliptic::curves::{Ed25519, Point}; -use mpc_recovery::GenerateResult; +use multi_party_eddsa::protocols::ExpandedKeyPair; use near_primitives::utils::generate_random_string; -use crate::env::containers::{DockerClient, LeaderNodeApi, SignerNodeApi}; +use mpc_recovery::firewall::allowed::DelegateActionRelayer; +use mpc_recovery::relayer::NearRpcAndRelayerClient; +use mpc_recovery::GenerateResult; + +use crate::env::containers::DockerClient; use crate::{initialize_relayer, util, RelayerCtx}; pub const NETWORK: &str = "mpc_it_network"; @@ -13,6 +19,21 @@ pub const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; // TODO: figure out how to instantiate and use a local firebase deployment pub const FIREBASE_AUDIENCE_ID: &str = "test_audience"; +pub struct SignerNodeApi { + pub address: String, + pub node_id: usize, + pub sk_share: ExpandedKeyPair, + pub cipher_key: GenericArray, + pub gcp_project_id: String, + pub gcp_datastore_local_url: String, +} + +pub struct LeaderNodeApi { + pub address: String, + pub relayer: DelegateActionRelayer, + pub client: NearRpcAndRelayerClient, +} + pub enum Nodes<'a> { Local { ctx: Context<'a>, @@ -63,18 +84,26 @@ impl Nodes<'_> { } pub struct Context<'a> { + pub docker_client: &'a DockerClient, + pub docker_network: String, + pub gcp_project_id: String, + pub audience_id: String, + pub relayer_ctx: RelayerCtx<'a>, pub datastore: containers::Datastore<'a>, pub oidc_provider: containers::OidcProvider<'a>, } pub async fn setup(docker_client: &DockerClient) -> anyhow::Result> { - docker_client.create_network(NETWORK).await?; + let gcp_project_id = GCP_PROJECT_ID; + let docker_network = NETWORK; + docker_client.create_network(docker_network).await?; let relayer_id = generate_random_string(7); // used to distinguish relayer tmp files in multiple tests - let relayer_ctx_future = initialize_relayer(docker_client, NETWORK, &relayer_id); - let datastore_future = containers::Datastore::run(docker_client, NETWORK, GCP_PROJECT_ID); - let oidc_provider_future = containers::OidcProvider::run(&docker_client, NETWORK); + let relayer_ctx_future = initialize_relayer(docker_client, docker_network, &relayer_id); + let datastore_future = + containers::Datastore::run(docker_client, docker_network, gcp_project_id); + let oidc_provider_future = containers::OidcProvider::run(docker_client, docker_network); let (relayer_ctx, datastore, oidc_provider) = futures::future::join3(relayer_ctx_future, datastore_future, oidc_provider_future).await; @@ -83,6 +112,10 @@ pub async fn setup(docker_client: &DockerClient) -> anyhow::Result> let oidc_provider = oidc_provider?; Ok(Context { + docker_client, + docker_network: docker_network.to_string(), + gcp_project_id: gcp_project_id.to_string(), + audience_id: FIREBASE_AUDIENCE_ID.to_string(), relayer_ctx, datastore, oidc_provider, @@ -94,18 +127,9 @@ pub async fn docker(nodes: usize, docker_client: &DockerClient) -> anyhow::Resul let GenerateResult { pk_set, secrets } = mpc_recovery::generate(nodes); let mut signer_node_futures = Vec::with_capacity(nodes); - for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { - signer_node_futures.push(containers::SignerNode::run_signing_node( - docker_client, - NETWORK, - i as u64, - share, - cipher_key, - &ctx.datastore.address, - &ctx.datastore.local_address, - GCP_PROJECT_ID, - FIREBASE_AUDIENCE_ID, - &ctx.oidc_provider.jwt_pk_url, + for (node_id, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { + signer_node_futures.push(containers::SignerNode::run( + &ctx, node_id, share, cipher_key, )); } let signer_nodes = futures::future::join_all(signer_node_futures) @@ -116,15 +140,11 @@ pub async fn docker(nodes: usize, docker_client: &DockerClient) -> anyhow::Resul let near_root_account = ctx.relayer_ctx.worker.root_account()?; let leader_node = containers::LeaderNode::run( - docker_client, - NETWORK, - signer_urls.clone(), &ctx, - GCP_PROJECT_ID, + signer_urls.clone(), near_root_account.id(), ctx.relayer_ctx.creator_account.id(), ctx.relayer_ctx.creator_account.secret_key(), - FIREBASE_AUDIENCE_ID, ) .await?; @@ -147,9 +167,6 @@ pub async fn host(nodes: usize, docker_client: &DockerClient) -> anyhow::Result< share, cipher_key, &ctx, - &ctx.datastore.local_address, - GCP_PROJECT_ID, - FIREBASE_AUDIENCE_ID, true, )); } @@ -160,14 +177,12 @@ pub async fn host(nodes: usize, docker_client: &DockerClient) -> anyhow::Result< let near_root_account = ctx.relayer_ctx.worker.root_account()?; let leader_node = local::LeaderNode::run( + &ctx, util::pick_unused_port().await?, signer_nodes.iter().map(|n| n.address.clone()).collect(), - &ctx, - GCP_PROJECT_ID, near_root_account.id(), ctx.relayer_ctx.creator_account.id(), ctx.relayer_ctx.creator_account.secret_key(), - FIREBASE_AUDIENCE_ID, true, ) .await?; diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index 0225cefe2..a6415b3ed 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -8,14 +8,16 @@ use mpc_recovery::{ ClaimOidcResponse, MpcPkResponse, NewAccountResponse, SignResponse, UserCredentialsResponse, }, }; -use mpc_recovery_integration_tests::env::{self, containers, GCP_PROJECT_ID}; +use mpc_recovery_integration_tests::env; +use mpc_recovery_integration_tests::env::containers::DockerClient; use workspaces::{network::Sandbox, Worker}; pub struct TestContext { - leader_node: containers::LeaderNodeApi, + leader_node: env::LeaderNodeApi, pk_set: Vec>, worker: Worker, - signer_nodes: Vec, + signer_nodes: Vec, + gcp_project_id: String, gcp_datastore_url: String, } @@ -23,7 +25,7 @@ impl TestContext { pub async fn gcp_service(&self) -> anyhow::Result { GcpService::new( "dev".into(), - GCP_PROJECT_ID.into(), + self.gcp_project_id.clone(), Some(self.gcp_datastore_url.clone()), ) .await @@ -35,7 +37,7 @@ where Task: FnOnce(TestContext) -> Fut, Fut: core::future::Future>, { - let docker_client = containers::DockerClient::default(); + let docker_client = DockerClient::default(); let nodes = if cfg!(feature = "docker-test") { env::docker(nodes, &docker_client).await? } else { @@ -47,6 +49,7 @@ where leader_node: nodes.leader_api(), signer_nodes: nodes.signer_apis(), worker: nodes.ctx().relayer_ctx.worker.clone(), + gcp_project_id: nodes.ctx().gcp_project_id.clone(), gcp_datastore_url: nodes.datastore_addr(), }) .await?; From 3f2fc62c2d706331d089647678bd8b9de5d61cca Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Oct 2023 14:22:34 -0700 Subject: [PATCH 12/21] setup-env cmd --- integration-tests/README.md | 3 +- integration-tests/src/env/local.rs | 3 +- integration-tests/src/main.rs | 176 ++++++----------------------- 3 files changed, 38 insertions(+), 144 deletions(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index bb0f34a27..4ed6d1611 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -83,7 +83,8 @@ Now, you can inspect each container's logs according to your needs using `docker We have a CLI tool that can instantiate a short-lived development environment that has everything except for the leader node set up. You can then seamlessly plug in your own leader node instance that you have set up manually (the tool gives you a CLI command to use as a starting point, but you can attach debugger, enable extra logs etc). Try it out now (sets up 3 signer nodes): ```bash -$ cargo run -p mpc-recovery-integration-tests -- test-leader 3 +$ export RUST_LOG=info +$ cargo run -p mpc-recovery-integration-tests -- setup-env 3 ``` ### I'm getting "Error: error trying to connect: No such file or directory (os error 2)" diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index d286531d1..5aea52155 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -62,7 +62,7 @@ impl SignerNode { let address = format!("http://localhost:{web_port}"); let child = Command::new(&executable) .args(&args) - .env("RUST_LOG", "mpc_recovery=DEBUG") + .env("RUST_LOG", "mpc_recovery=INFO") .envs(std::env::vars()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -165,6 +165,7 @@ impl LeaderNode { let child = Command::new(&executable) .args(&args) + .env("RUST_LOG", "mpc_recovery=INFO") .envs(std::env::vars()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) diff --git a/integration-tests/src/main.rs b/integration-tests/src/main.rs index 41684202d..47f0ff912 100644 --- a/integration-tests/src/main.rs +++ b/integration-tests/src/main.rs @@ -1,17 +1,12 @@ use clap::Parser; -use mpc_recovery::GenerateResult; -use mpc_recovery_integration_tests::env::containers; -use near_primitives::utils::generate_random_string; +use mpc_recovery_integration_tests::env; +use mpc_recovery_integration_tests::env::containers::DockerClient; use tokio::io::{stdin, AsyncReadExt}; use tracing_subscriber::EnvFilter; -const NETWORK: &str = "mpc_recovery_dev_network"; -const GCP_PROJECT_ID: &str = "mpc-recovery-dev-gcp-project"; -pub const FIREBASE_AUDIENCE_ID: &str = "test_audience"; - #[derive(Parser, Debug)] enum Cli { - TestLeader { nodes: usize }, + SetupEnv { nodes: usize }, } #[tokio::main] @@ -21,150 +16,47 @@ async fn main() -> anyhow::Result<()> { .with_env_filter(EnvFilter::from_default_env()); subscriber.init(); match Cli::parse() { - Cli::TestLeader { nodes } => { + Cli::SetupEnv { nodes } => { tracing::info!("Setting up an environment with {} nodes", nodes); - let docker_client = containers::DockerClient::default(); - - let relayer_id = generate_random_string(7); - let relayer_ctx_future = mpc_recovery_integration_tests::initialize_relayer( - &docker_client, - NETWORK, - &relayer_id, + let docker_client = DockerClient::default(); + let nodes = env::host(nodes, &docker_client).await?; + let ctx = nodes.ctx(); + + tracing::info!(""); + tracing::info!("Environment is ready"); + tracing::info!(" Docker network: {}", ctx.docker_network); + tracing::info!(" GCP project id: {}", ctx.gcp_project_id); + tracing::info!(" Audience id: {}", ctx.audience_id); + + tracing::info!("Datastore address: {}", nodes.datastore_addr()); + tracing::info!("Sandbox address: {}", ctx.relayer_ctx.sandbox.local_address); + tracing::info!( + "Sandbox root account: {}", + ctx.relayer_ctx.worker.root_account()?.id() ); - let datastore_future = - containers::Datastore::run(&docker_client, NETWORK, GCP_PROJECT_ID); - - let oidc_provider_future = containers::OidcProvider::run(&docker_client, NETWORK); - - let (relayer_ctx, datastore, oidc_provider) = - futures::future::join3(relayer_ctx_future, datastore_future, oidc_provider_future) - .await; - - let relayer_ctx = relayer_ctx?; - let datastore = datastore?; - let oidc_provider = oidc_provider?; - - tracing::info!("Generating secrets"); - let GenerateResult { secrets, .. } = mpc_recovery::generate(nodes); - tracing::info!("Running signer nodes..."); - let mut signer_node_futures = Vec::new(); - for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { - let signer_node = containers::SignerNode::run_signing_node( - &docker_client, - NETWORK, - i as u64, - share, - cipher_key, - &datastore.address, - &datastore.local_address, - GCP_PROJECT_ID, - FIREBASE_AUDIENCE_ID, - &oidc_provider.jwt_pk_url, - ); - signer_node_futures.push(signer_node); - } - let signer_nodes = futures::future::join_all(signer_node_futures) - .await - .into_iter() - .collect::, _>>()?; - tracing::info!("Signer nodes initialized"); - let signer_urls: &Vec<_> = &signer_nodes.iter().map(|n| n.address.clone()).collect(); - - let near_root_account = relayer_ctx.worker.root_account()?; - tracing::info!("Root account_id: {}", near_root_account.id()); - - let mut cmd = vec![ - "start-leader".to_string(), - "--web-port".to_string(), - "3000".to_string(), - "--near-rpc".to_string(), - format!( - "http://localhost:{}", - relayer_ctx - .sandbox - .container - .get_host_port_ipv4(containers::Sandbox::CONTAINER_RPC_PORT) - ), - "--near-root-account".to_string(), - near_root_account.id().to_string(), - "--account-creator-id".to_string(), - relayer_ctx.creator_account.id().to_string(), - "--account-creator-sk".to_string(), - relayer_ctx.creator_account.secret_key().to_string(), - "--gcp-project-id".to_string(), - GCP_PROJECT_ID.to_string(), - "--gcp-datastore-url".to_string(), - format!( - "http://localhost:{}", - datastore - .container - .get_host_port_ipv4(containers::Datastore::CONTAINER_PORT) - ), - "--fast-auth-partners".to_string(), - escape_json_string(&serde_json::json!([ - { - "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{}", FIREBASE_AUDIENCE_ID.to_string()), - "audience": FIREBASE_AUDIENCE_ID.to_string(), - }, - "relayer": { - "url": format!( - "http://localhost:{}", - relayer_ctx - .relayer - .container - .get_host_port_ipv4(containers::Relayer::CONTAINER_PORT) - ), - "api_key": serde_json::Value::Null, - }, - }, - ]).to_string()), - "--jwt-signature-pk-url".to_string(), - oidc_provider.jwt_pk_url, - - ]; - for sign_node in signer_urls { - cmd.push("--sign-nodes".to_string()); - cmd.push(sign_node.clone()); - } - - tracing::info!("Please run the command below to start a leader node:"); + tracing::info!("Relayer address: {}", ctx.relayer_ctx.relayer.local_address); tracing::info!( - "RUST_LOG=mpc_recovery=debug cargo run --bin mpc-recovery -- {}", - cmd.join(" ") + "Relayer creator account: {}", + ctx.relayer_ctx.creator_account.id() ); - tracing::info!("===================================="); - tracing::info!("You can now interact with your local service manually. For example:"); + tracing::info!("OidcProvider address: {}", ctx.oidc_provider.jwt_local_url); tracing::info!( - r#"curl -X POST -H "Content-Type: application/json" -d '{{"oidc_token": , "near_account_id": "abc45436676.near", "create_account_options": {{"full_access_keys": ["ed25519:4fnCz9NTEMhkfwAHDhFDkPS1mD58QHdRyago5n4vtCS2"]}}}}' http://localhost:3000/new_account"# + "Signer node URLs:\n{:#?}", + nodes + .signer_apis() + .iter() + .map(|n| n.address.as_str()) + .collect::>() ); - + tracing::info!("pk set: {:?}", nodes.pk_set()); + tracing::info!("Leader node address: {}", nodes.leader_api().address); tracing::info!("Press any button to exit and destroy all containers..."); - while stdin().read(&mut [0]).await? == 0 {} + while stdin().read(&mut [0]).await? == 0 { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } } }; Ok(()) } - -fn escape_json_string(input: &str) -> String { - let mut result = String::with_capacity(input.len() + 2); - result.push('"'); - - for c in input.chars() { - match c { - '"' => result.push_str(r"\\"), - '\\' => result.push_str(r"\\"), - '\n' => result.push_str(r"\n"), - '\r' => result.push_str(r"\r"), - '\t' => result.push_str(r"\t"), - '\u{08}' => result.push_str(r"\b"), // Backspace - '\u{0C}' => result.push_str(r"\f"), // Form feed - _ => result.push(c), - } - } - - result.push('"'); - result -} From 8c7288e36343bdb06919bd63f1650f895b2be6c3 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Oct 2023 14:25:53 -0700 Subject: [PATCH 13/21] Update GA pipeline --- .github/workflows/build-and-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index eb8d8443c..cf42c4f2e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -76,10 +76,9 @@ jobs: env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} - - name: Pull MPC Recovery Docker Image + - name: Build MPC Recovery Binary Locally run: | - docker pull us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} - docker tag us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} near/mpc-recovery:latest + cargo build --release - name: Install stable toolchain uses: actions-rs/toolchain@v1 From 2b0495598b44991b1470f61bca024a6fb82b52e1 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Oct 2023 15:04:24 -0700 Subject: [PATCH 14/21] More cleanup --- integration-tests/src/env/containers.rs | 27 ++++++------ integration-tests/src/env/local.rs | 34 +++++++-------- integration-tests/src/env/mod.rs | 57 ++++++++++--------------- integration-tests/tests/lib.rs | 10 ++--- 4 files changed, 55 insertions(+), 73 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index 5d54b7b20..e9e340da4 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -484,6 +484,8 @@ pub struct SignerNode<'a> { pub container: Container<'a, GenericImage>, pub address: String, pub local_address: String, + + env: String, node_id: usize, sk_share: ExpandedKeyPair, cipher_key: GenericArray, @@ -508,7 +510,7 @@ impl SignerNode<'_> { .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); let args = mpc_recovery::Cli::StartSign { - env: "dev".to_string(), + env: ctx.env.clone(), node_id: node_id as u64, web_port: Self::CONTAINER_PORT, sk_share: Some(serde_json::to_string(&sk_share)?), @@ -553,6 +555,8 @@ impl SignerNode<'_> { container, address: full_address, local_address: format!("http://localhost:{host_port}"), + + env: ctx.env.clone(), node_id, sk_share: sk_share.clone(), cipher_key: *cipher_key, @@ -563,6 +567,7 @@ impl SignerNode<'_> { pub fn api(&self) -> SignerNodeApi { SignerNodeApi { + env: self.env.clone(), address: self.local_address.clone(), node_id: self.node_id, sk_share: self.sk_share.clone(), @@ -585,9 +590,8 @@ impl SignerNodeApi { &self, new_cipher_key: &GenericArray, ) -> anyhow::Result<(Aes256Gcm, Aes256Gcm)> { - let env = "dev".to_string(); let gcp_service = mpc_recovery::gcp::GcpService::new( - env, + self.env.clone(), self.gcp_project_id.clone(), Some(self.gcp_datastore_local_url.clone()), ) @@ -622,14 +626,9 @@ impl<'a> LeaderNode<'a> { // Container port used for the docker network, does not have to be unique const CONTAINER_PORT: u16 = 3000; - pub async fn run( - ctx: &Context<'a>, - sign_nodes: Vec, - near_root_account: &AccountId, - account_creator_id: &AccountId, - account_creator_sk: &workspaces::types::SecretKey, - ) -> anyhow::Result> { + pub async fn run(ctx: &Context<'a>, sign_nodes: Vec) -> anyhow::Result> { tracing::info!("Running leader node container..."); + let account_creator = &ctx.relayer_ctx.creator_account; let image = GenericImage::new("near/mpc-recovery", "latest") .with_wait_for(WaitFor::Nothing) @@ -637,13 +636,13 @@ impl<'a> LeaderNode<'a> { .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); let args = mpc_recovery::Cli::StartLeader { - env: "dev".to_string(), + env: ctx.env.clone(), web_port: Self::CONTAINER_PORT, sign_nodes, near_rpc: ctx.relayer_ctx.sandbox.address.clone(), - near_root_account: near_root_account.to_string(), - account_creator_id: account_creator_id.clone(), - account_creator_sk: Some(account_creator_sk.to_string()), + near_root_account: ctx.relayer_ctx.worker.root_account()?.id().to_string(), + account_creator_id: account_creator.id().clone(), + account_creator_sk: Some(account_creator.secret_key().to_string()), fast_auth_partners: Some( serde_json::json!([ { diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index 5aea52155..aaf0659ea 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -14,6 +14,7 @@ use crate::util; pub struct SignerNode { pub address: String, + env: String, node_id: usize, sk_share: ExpandedKeyPair, cipher_key: GenericArray, @@ -27,18 +28,17 @@ pub struct SignerNode { impl SignerNode { pub async fn run( - web_port: u16, + ctx: &super::Context<'_>, node_id: u64, sk_share: &ExpandedKeyPair, cipher_key: &GenericArray, - ctx: &super::Context<'_>, - release: bool, ) -> anyhow::Result { - let executable = util::executable(release) + let executable = util::executable(ctx.release) .context("could not find target dir while running signing node")?; + let web_port = util::pick_unused_port().await?; let args = mpc_recovery::Cli::StartSign { - env: "dev".to_string(), + env: ctx.env.clone(), node_id, web_port, sk_share: Some(serde_json::to_string(&sk_share)?), @@ -88,6 +88,7 @@ impl SignerNode { Ok(Self { address, + env: ctx.env.clone(), node_id: node_id as usize, sk_share: sk_share.clone(), cipher_key: *cipher_key, @@ -100,6 +101,7 @@ impl SignerNode { pub fn api(&self) -> SignerNodeApi { SignerNodeApi { address: self.address.clone(), + env: self.env.clone(), node_id: self.node_id, sk_share: self.sk_share.clone(), cipher_key: self.cipher_key, @@ -120,27 +122,21 @@ pub struct LeaderNode { } impl LeaderNode { - pub async fn run( - ctx: &super::Context<'_>, - web_port: u16, - sign_nodes: Vec, - near_root_account: &workspaces::AccountId, - account_creator_id: &workspaces::AccountId, - account_creator_sk: &workspaces::types::SecretKey, - release: bool, - ) -> anyhow::Result { + pub async fn run(ctx: &super::Context<'_>, sign_nodes: Vec) -> anyhow::Result { tracing::info!("Running leader node..."); - let executable = util::executable(release) + let executable = util::executable(ctx.release) .context("could not find target dir while running leader node")?; + let account_creator = &ctx.relayer_ctx.creator_account; + let web_port = util::pick_unused_port().await?; let args = mpc_recovery::Cli::StartLeader { - env: "dev".to_string(), + env: ctx.env.clone(), web_port, sign_nodes, near_rpc: ctx.relayer_ctx.sandbox.local_address.clone(), - near_root_account: near_root_account.to_string(), - account_creator_id: account_creator_id.clone(), - account_creator_sk: Some(account_creator_sk.to_string()), + near_root_account: ctx.relayer_ctx.worker.root_account()?.id().to_string(), + account_creator_id: account_creator.id().clone(), + account_creator_sk: Some(account_creator.secret_key().to_string()), fast_auth_partners: Some( serde_json::json!([ { diff --git a/integration-tests/src/env/mod.rs b/integration-tests/src/env/mod.rs index 29893e995..37568632c 100644 --- a/integration-tests/src/env/mod.rs +++ b/integration-tests/src/env/mod.rs @@ -12,14 +12,16 @@ use mpc_recovery::relayer::NearRpcAndRelayerClient; use mpc_recovery::GenerateResult; use crate::env::containers::DockerClient; -use crate::{initialize_relayer, util, RelayerCtx}; +use crate::{initialize_relayer, RelayerCtx}; -pub const NETWORK: &str = "mpc_it_network"; -pub const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; +const ENV: &str = "dev"; +const NETWORK: &str = "mpc_it_network"; +const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; // TODO: figure out how to instantiate and use a local firebase deployment -pub const FIREBASE_AUDIENCE_ID: &str = "test_audience"; +const FIREBASE_AUDIENCE_ID: &str = "test_audience"; pub struct SignerNodeApi { + pub env: String, pub address: String, pub node_id: usize, pub sk_share: ExpandedKeyPair, @@ -84,10 +86,12 @@ impl Nodes<'_> { } pub struct Context<'a> { + pub env: String, pub docker_client: &'a DockerClient, pub docker_network: String, pub gcp_project_id: String, pub audience_id: String, + pub release: bool, pub relayer_ctx: RelayerCtx<'a>, pub datastore: containers::Datastore<'a>, @@ -112,10 +116,12 @@ pub async fn setup(docker_client: &DockerClient) -> anyhow::Result> let oidc_provider = oidc_provider?; Ok(Context { + env: ENV.to_string(), docker_client, docker_network: docker_network.to_string(), gcp_project_id: gcp_project_id.to_string(), audience_id: FIREBASE_AUDIENCE_ID.to_string(), + release: true, relayer_ctx, datastore, oidc_provider, @@ -136,17 +142,8 @@ pub async fn docker(nodes: usize, docker_client: &DockerClient) -> anyhow::Resul .await .into_iter() .collect::, _>>()?; - let signer_urls: &Vec<_> = &signer_nodes.iter().map(|n| n.address.clone()).collect(); - - let near_root_account = ctx.relayer_ctx.worker.root_account()?; - let leader_node = containers::LeaderNode::run( - &ctx, - signer_urls.clone(), - near_root_account.id(), - ctx.relayer_ctx.creator_account.id(), - ctx.relayer_ctx.creator_account.secret_key(), - ) - .await?; + let sign_nodes = signer_nodes.iter().map(|n| n.address.clone()).collect(); + let leader_node = containers::LeaderNode::run(&ctx, sign_nodes).await?; Ok(Nodes::Docker { ctx, @@ -161,31 +158,15 @@ pub async fn host(nodes: usize, docker_client: &DockerClient) -> anyhow::Result< let GenerateResult { pk_set, secrets } = mpc_recovery::generate(nodes); let mut signer_node_futures = Vec::with_capacity(nodes); for (i, (share, cipher_key)) in secrets.iter().enumerate().take(nodes) { - signer_node_futures.push(local::SignerNode::run( - util::pick_unused_port().await?, - i as u64, - share, - cipher_key, - &ctx, - true, - )); + signer_node_futures.push(local::SignerNode::run(&ctx, i as u64, share, cipher_key)); } let signer_nodes = futures::future::join_all(signer_node_futures) .await .into_iter() .collect::, _>>()?; - let near_root_account = ctx.relayer_ctx.worker.root_account()?; - let leader_node = local::LeaderNode::run( - &ctx, - util::pick_unused_port().await?, - signer_nodes.iter().map(|n| n.address.clone()).collect(), - near_root_account.id(), - ctx.relayer_ctx.creator_account.id(), - ctx.relayer_ctx.creator_account.secret_key(), - true, - ) - .await?; + let sign_nodes = signer_nodes.iter().map(|n| n.address.clone()).collect(); + let leader_node = local::LeaderNode::run(&ctx, sign_nodes).await?; Ok(Nodes::Local { ctx, @@ -194,3 +175,11 @@ pub async fn host(nodes: usize, docker_client: &DockerClient) -> anyhow::Result< signer_nodes, }) } + +pub async fn run(nodes: usize, docker_client: &DockerClient) -> anyhow::Result { + #[cfg(feature = "docker-test")] + return docker(nodes, docker_client).await; + + #[cfg(not(feature = "docker-test"))] + return host(nodes, docker_client).await; +} diff --git a/integration-tests/tests/lib.rs b/integration-tests/tests/lib.rs index a6415b3ed..2a0e3066e 100644 --- a/integration-tests/tests/lib.rs +++ b/integration-tests/tests/lib.rs @@ -13,6 +13,7 @@ use mpc_recovery_integration_tests::env::containers::DockerClient; use workspaces::{network::Sandbox, Worker}; pub struct TestContext { + env: String, leader_node: env::LeaderNodeApi, pk_set: Vec>, worker: Worker, @@ -24,7 +25,7 @@ pub struct TestContext { impl TestContext { pub async fn gcp_service(&self) -> anyhow::Result { GcpService::new( - "dev".into(), + self.env.clone(), self.gcp_project_id.clone(), Some(self.gcp_datastore_url.clone()), ) @@ -38,13 +39,10 @@ where Fut: core::future::Future>, { let docker_client = DockerClient::default(); - let nodes = if cfg!(feature = "docker-test") { - env::docker(nodes, &docker_client).await? - } else { - env::host(nodes, &docker_client).await? - }; + let nodes = env::run(nodes, &docker_client).await?; f(TestContext { + env: nodes.ctx().env.clone(), pk_set: nodes.pk_set(), leader_node: nodes.leader_api(), signer_nodes: nodes.signer_apis(), From 9fbf7f53d38133e2b555275fb7e0c1e23d7308d7 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Oct 2023 15:05:45 -0700 Subject: [PATCH 15/21] Use env::run for setup-env --- integration-tests/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/src/main.rs b/integration-tests/src/main.rs index 47f0ff912..edfe95596 100644 --- a/integration-tests/src/main.rs +++ b/integration-tests/src/main.rs @@ -19,7 +19,7 @@ async fn main() -> anyhow::Result<()> { Cli::SetupEnv { nodes } => { tracing::info!("Setting up an environment with {} nodes", nodes); let docker_client = DockerClient::default(); - let nodes = env::host(nodes, &docker_client).await?; + let nodes = env::run(nodes, &docker_client).await?; let ctx = nodes.ctx(); tracing::info!(""); From 392162371690a1e22c630efcd5978c9202fc8dd1 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Oct 2023 15:32:12 -0700 Subject: [PATCH 16/21] More cleanup --- integration-tests/src/env/containers.rs | 23 ++++----- integration-tests/src/env/local.rs | 64 +++++-------------------- integration-tests/src/util.rs | 31 +++++++++++- 3 files changed, 51 insertions(+), 67 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index e9e340da4..e59ebda4f 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -1,7 +1,7 @@ #![allow(clippy::too_many_arguments)] use aes_gcm::{Aes256Gcm, KeyInit}; -use anyhow::{anyhow, Ok}; +use anyhow::anyhow; use bollard::{container::LogsOptions, network::CreateNetworkOptions, service::Ipam, Docker}; use ed25519_dalek::ed25519::signature::digest::{consts::U32, generic_array::GenericArray}; use ed25519_dalek::{PublicKey as PublicKeyEd25519, Verifier}; @@ -135,7 +135,7 @@ impl DockerClient { tokio::spawn(async move { let mut stdout = tokio::io::stdout(); - while let Some(Result::Ok(output)) = output.next().await { + while let Some(Ok(output)) = output.next().await { stdout .write_all(output.into_bytes().as_ref()) .await @@ -504,11 +504,6 @@ impl SignerNode<'_> { cipher_key: &GenericArray, ) -> anyhow::Result> { tracing::info!("Running signer node container {}...", node_id); - let image: GenericImage = GenericImage::new("near/mpc-recovery", "latest") - .with_wait_for(WaitFor::Nothing) - .with_exposed_port(Self::CONTAINER_PORT) - .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); - let args = mpc_recovery::Cli::StartSign { env: ctx.env.clone(), node_id: node_id as u64, @@ -531,6 +526,10 @@ impl SignerNode<'_> { } .into_str_args(); + let image: GenericImage = GenericImage::new("near/mpc-recovery", "latest") + .with_wait_for(WaitFor::Nothing) + .with_exposed_port(Self::CONTAINER_PORT) + .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); let image: RunnableImage = (image, args).into(); let image = image.with_network(&ctx.docker_network); let container = ctx.docker_client.cli.run(image); @@ -629,12 +628,6 @@ impl<'a> LeaderNode<'a> { pub async fn run(ctx: &Context<'a>, sign_nodes: Vec) -> anyhow::Result> { tracing::info!("Running leader node container..."); let account_creator = &ctx.relayer_ctx.creator_account; - - let image = GenericImage::new("near/mpc-recovery", "latest") - .with_wait_for(WaitFor::Nothing) - .with_exposed_port(Self::CONTAINER_PORT) - .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); - let args = mpc_recovery::Cli::StartLeader { env: ctx.env.clone(), web_port: Self::CONTAINER_PORT, @@ -665,6 +658,10 @@ impl<'a> LeaderNode<'a> { } .into_str_args(); + let image = GenericImage::new("near/mpc-recovery", "latest") + .with_wait_for(WaitFor::Nothing) + .with_exposed_port(Self::CONTAINER_PORT) + .with_env_var("RUST_LOG", "mpc_recovery=DEBUG"); let image: RunnableImage = (image, args).into(); let image = image.with_network(&ctx.docker_network); let container = ctx.docker_client.cli.run(image); diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index aaf0659ea..12f3ec109 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -2,9 +2,7 @@ use aes_gcm::aead::consts::U32; use aes_gcm::aead::generic_array::GenericArray; -use anyhow::Context; -use async_process::{Child, Command, Stdio}; -use hyper::StatusCode; +use async_process::Child; use mpc_recovery::firewall::allowed::DelegateActionRelayer; use mpc_recovery::relayer::NearRpcAndRelayerClient; use multi_party_eddsa::protocols::ExpandedKeyPair; @@ -33,10 +31,7 @@ impl SignerNode { sk_share: &ExpandedKeyPair, cipher_key: &GenericArray, ) -> anyhow::Result { - let executable = util::executable(ctx.release) - .context("could not find target dir while running signing node")?; let web_port = util::pick_unused_port().await?; - let args = mpc_recovery::Cli::StartSign { env: ctx.env.clone(), node_id, @@ -59,31 +54,11 @@ impl SignerNode { } .into_str_args(); + let sign_node_id = format!("sign/{node_id}"); + let process = util::spawn_mpc(ctx.release, &sign_node_id, &args)?; let address = format!("http://localhost:{web_port}"); - let child = Command::new(&executable) - .args(&args) - .env("RUST_LOG", "mpc_recovery=INFO") - .envs(std::env::vars()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn() - .with_context(|| { - format!( - "failed to run signing node: [node_id={node_id}, {}]", - executable.display() - ) - })?; - - tracing::info!("Signer node is start on {}", address); - loop { - let x: anyhow::Result = util::get(&address).await; - match x { - std::result::Result::Ok(status) if status == StatusCode::OK => break, - _ => (), - } - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - } + tracing::info!("Signer node is starting at {}", address); + util::ping_until_ok(&address, 60).await?; tracing::info!("Signer node started [node_id={node_id}, {address}]"); Ok(Self { @@ -94,7 +69,7 @@ impl SignerNode { cipher_key: *cipher_key, gcp_project_id: ctx.gcp_project_id.clone(), gcp_datastore_url: ctx.datastore.local_address.clone(), - process: child, + process, }) } @@ -124,11 +99,8 @@ pub struct LeaderNode { impl LeaderNode { pub async fn run(ctx: &super::Context<'_>, sign_nodes: Vec) -> anyhow::Result { tracing::info!("Running leader node..."); - let executable = util::executable(ctx.release) - .context("could not find target dir while running leader node")?; let account_creator = &ctx.relayer_ctx.creator_account; let web_port = util::pick_unused_port().await?; - let args = mpc_recovery::Cli::StartLeader { env: ctx.env.clone(), web_port, @@ -137,6 +109,7 @@ impl LeaderNode { near_root_account: ctx.relayer_ctx.worker.root_account()?.id().to_string(), account_creator_id: account_creator.id().clone(), account_creator_sk: Some(account_creator.secret_key().to_string()), + fast_auth_partners_filepath: None, fast_auth_partners: Some( serde_json::json!([ { @@ -152,38 +125,23 @@ impl LeaderNode { ]) .to_string(), ), - fast_auth_partners_filepath: None, gcp_project_id: ctx.gcp_project_id.clone(), gcp_datastore_url: Some(ctx.datastore.local_address.clone()), jwt_signature_pk_url: ctx.oidc_provider.jwt_local_url.clone(), } .into_str_args(); - let child = Command::new(&executable) - .args(&args) - .env("RUST_LOG", "mpc_recovery=INFO") - .envs(std::env::vars()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn() - .with_context(|| format!("failed to run leader node: {}", executable.display()))?; - + let process = util::spawn_mpc(ctx.release, "leader", &args)?; let address = format!("http://localhost:{web_port}"); tracing::info!("Leader node container is starting at {}", address); - loop { - match util::get(&address).await { - std::result::Result::Ok(status) if status == StatusCode::OK => break, - _ => tokio::time::sleep(std::time::Duration::from_secs(1)).await, - } - } + util::ping_until_ok(&address, 60).await?; + tracing::info!("Leader node running at {address}"); - tracing::info!("Leader node container is running at {address}"); Ok(Self { address, near_rpc: ctx.relayer_ctx.sandbox.local_address.clone(), relayer_url: ctx.relayer_ctx.relayer.local_address.clone(), - process: child, + process, }) } diff --git a/integration-tests/src/util.rs b/integration-tests/src/util.rs index 3ae82bb78..4cc341a98 100644 --- a/integration-tests/src/util.rs +++ b/integration-tests/src/util.rs @@ -4,7 +4,8 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::{Context, Ok}; +use anyhow::Context; +use async_process::{Child, Command, Stdio}; use hyper::{Body, Client, Method, Request, StatusCode, Uri}; use serde::{Deserialize, Serialize}; use toml::Value; @@ -209,6 +210,19 @@ pub async fn pick_unused_port() -> anyhow::Result { Ok(port) } +pub async fn ping_until_ok(addr: &str, timeout: u64) -> anyhow::Result<()> { + tokio::time::timeout(std::time::Duration::from_secs(timeout), async { + loop { + match get(addr).await { + Ok(status) if status == StatusCode::OK => break, + _ => tokio::time::sleep(std::time::Duration::from_millis(500)).await, + } + } + }) + .await?; + Ok(()) +} + pub fn target_dir() -> Option { let mut out_dir = Path::new(std::env!("OUT_DIR")); loop { @@ -229,3 +243,18 @@ pub fn executable(release: bool) -> Option { .join(EXECUTABLE); Some(executable) } + +pub fn spawn_mpc(release: bool, node: &str, args: &[String]) -> anyhow::Result { + let executable = executable(release) + .with_context(|| format!("could not find target dir while starting {node} node"))?; + + Command::new(&executable) + .args(args) + .env("RUST_LOG", "mpc_recovery=INFO") + .envs(std::env::vars()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("failed to run {node} node: {}", executable.display())) +} From 8ca535a44e57f7f789e105b6a77437540f90cd11 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Oct 2023 16:38:24 -0700 Subject: [PATCH 17/21] Better setup-env logging --- integration-tests/src/env/containers.rs | 10 +++- integration-tests/src/env/local.rs | 6 +-- integration-tests/src/env/mod.rs | 3 ++ integration-tests/src/main.rs | 62 +++++++++++++------------ 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index e59ebda4f..860818a3b 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -167,12 +167,16 @@ pub struct Redis<'a> { pub container: Container<'a, GenericImage>, pub address: String, pub full_address: String, + pub local_address: String, } impl<'a> Redis<'a> { + const CONTAINER_PORT: u16 = 3000; + pub async fn run(docker_client: &'a DockerClient, network: &str) -> anyhow::Result> { tracing::info!("Running Redis container..."); let image = GenericImage::new("redis", "latest") + .with_exposed_port(Self::CONTAINER_PORT) .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections")); let image: RunnableImage = image.into(); let image = image.with_network(network); @@ -183,12 +187,14 @@ impl<'a> Redis<'a> { // Note: this port is hardcoded in the Redis image let full_address = format!("redis://{}:{}", address, 6379); + let host_port = container.get_host_port_ipv4(Self::CONTAINER_PORT); tracing::info!("Redis container is running at {}", full_address); Ok(Redis { container, address, full_address, + local_address: format!("http://localhost:{host_port}"), }) } } @@ -514,7 +520,7 @@ impl SignerNode<'_> { oidc_providers: Some( serde_json::json!([ { - "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "issuer": ctx.issuer, "audience": ctx.audience_id, }, ]) @@ -640,7 +646,7 @@ impl<'a> LeaderNode<'a> { serde_json::json!([ { "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "issuer": ctx.issuer, "audience": ctx.audience_id, }, "relayer": { diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index 12f3ec109..1ad3afcb0 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -1,5 +1,3 @@ -#![allow(clippy::too_many_arguments)] - use aes_gcm::aead::consts::U32; use aes_gcm::aead::generic_array::GenericArray; use async_process::Child; @@ -42,7 +40,7 @@ impl SignerNode { oidc_providers: Some( serde_json::json!([ { - "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "issuer": ctx.issuer, "audience": ctx.audience_id, }, ]) @@ -114,7 +112,7 @@ impl LeaderNode { serde_json::json!([ { "oidc_provider": { - "issuer": format!("https://securetoken.google.com/{}", ctx.audience_id), + "issuer": ctx.issuer, "audience": ctx.audience_id, }, "relayer": { diff --git a/integration-tests/src/env/mod.rs b/integration-tests/src/env/mod.rs index 37568632c..91cb9f651 100644 --- a/integration-tests/src/env/mod.rs +++ b/integration-tests/src/env/mod.rs @@ -19,6 +19,7 @@ const NETWORK: &str = "mpc_it_network"; const GCP_PROJECT_ID: &str = "mpc-recovery-gcp-project"; // TODO: figure out how to instantiate and use a local firebase deployment const FIREBASE_AUDIENCE_ID: &str = "test_audience"; +const ISSUER: &str = "https://securetoken.google.com/test_audience"; pub struct SignerNodeApi { pub env: String, @@ -91,6 +92,7 @@ pub struct Context<'a> { pub docker_network: String, pub gcp_project_id: String, pub audience_id: String, + pub issuer: String, pub release: bool, pub relayer_ctx: RelayerCtx<'a>, @@ -121,6 +123,7 @@ pub async fn setup(docker_client: &DockerClient) -> anyhow::Result> docker_network: docker_network.to_string(), gcp_project_id: gcp_project_id.to_string(), audience_id: FIREBASE_AUDIENCE_ID.to_string(), + issuer: ISSUER.to_string(), release: true, relayer_ctx, datastore, diff --git a/integration-tests/src/main.rs b/integration-tests/src/main.rs index edfe95596..562ed6463 100644 --- a/integration-tests/src/main.rs +++ b/integration-tests/src/main.rs @@ -17,41 +17,43 @@ async fn main() -> anyhow::Result<()> { subscriber.init(); match Cli::parse() { Cli::SetupEnv { nodes } => { - tracing::info!("Setting up an environment with {} nodes", nodes); + println!("Setting up an environment with {} nodes...", nodes); let docker_client = DockerClient::default(); let nodes = env::run(nodes, &docker_client).await?; let ctx = nodes.ctx(); - tracing::info!(""); - tracing::info!("Environment is ready"); - tracing::info!(" Docker network: {}", ctx.docker_network); - tracing::info!(" GCP project id: {}", ctx.gcp_project_id); - tracing::info!(" Audience id: {}", ctx.audience_id); - - tracing::info!("Datastore address: {}", nodes.datastore_addr()); - tracing::info!("Sandbox address: {}", ctx.relayer_ctx.sandbox.local_address); - tracing::info!( - "Sandbox root account: {}", - ctx.relayer_ctx.worker.root_account()?.id() - ); - tracing::info!("Relayer address: {}", ctx.relayer_ctx.relayer.local_address); - tracing::info!( - "Relayer creator account: {}", - ctx.relayer_ctx.creator_account.id() - ); - tracing::info!("OidcProvider address: {}", ctx.oidc_provider.jwt_local_url); - tracing::info!( - "Signer node URLs:\n{:#?}", - nodes - .signer_apis() - .iter() - .map(|n| n.address.as_str()) - .collect::>() - ); - tracing::info!("pk set: {:?}", nodes.pk_set()); - tracing::info!("Leader node address: {}", nodes.leader_api().address); - tracing::info!("Press any button to exit and destroy all containers..."); + println!("\nEnvironment is ready:"); + println!(" docker-network: {}", ctx.docker_network); + println!(" gcp-project-id: {}", ctx.gcp_project_id); + println!(" audience-id: {}", ctx.audience_id); + println!(" issuer: {}", ctx.issuer); + println!(" release: {}", ctx.release); + println!(" env: {}", ctx.env); + println!("\nAccounts:"); + println!(" creator: {}", ctx.relayer_ctx.creator_account.id()); + println!(" root: {}", ctx.relayer_ctx.worker.root_account()?.id()); + + println!("\nExternal services:"); + println!(" oidc-provider: {}", ctx.oidc_provider.jwt_local_url); + println!(" datastore: {}", nodes.datastore_addr()); + println!(" sandbox: {}", ctx.relayer_ctx.sandbox.local_address); + println!(" relayer: {}", ctx.relayer_ctx.relayer.local_address); + println!(" redis: {}", ctx.relayer_ctx.redis.local_address); + + println!("\nNode services:"); + println!(" leader node: {}", nodes.leader_api().address); + println!(" signer nodes:"); + for node in nodes.signer_apis() { + println!(" {}: {}", node.node_id, node.address); + } + + println!("\nSigner public key set:"); + for pk in nodes.pk_set() { + println!(" {pk:?}"); + } + + println!("\nPress any button to exit and destroy all containers..."); while stdin().read(&mut [0]).await? == 0 { tokio::time::sleep(std::time::Duration::from_millis(25)).await; } From 3325261179655bad3c14bc29d596a054df2f4924 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Mon, 16 Oct 2023 18:05:33 +1100 Subject: [PATCH 18/21] prefer 127.0.0.1 over localhost to enforce IPv4 --- integration-tests/src/env/containers.rs | 14 +++++++------- integration-tests/src/env/local.rs | 4 ++-- integration-tests/src/lib.rs | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index 860818a3b..e78601e0c 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -194,7 +194,7 @@ impl<'a> Redis<'a> { container, address, full_address, - local_address: format!("http://localhost:{host_port}"), + local_address: format!("http://127.0.0.1:{host_port}"), }) } } @@ -257,7 +257,7 @@ impl<'a> Sandbox<'a> { Ok(Sandbox { container, address: full_address, - local_address: format!("http://localhost:{host_port}"), + local_address: format!("http://127.0.0.1:{host_port}"), }) } } @@ -372,7 +372,7 @@ impl<'a> Relayer<'a> { Ok(Relayer { container, address: full_address, - local_address: format!("http://localhost:{host_port}"), + local_address: format!("http://127.0.0.1:{host_port}"), id: relayer_id.to_string(), }) } @@ -413,7 +413,7 @@ impl<'a> OidcProvider<'a> { let host_port = container.get_host_port_ipv4(Self::CONTAINER_PORT); let full_address = format!("http://{}:{}", ip_address, Self::CONTAINER_PORT); let jwt_pk_url = format!("{}/jwt_signature_public_keys", full_address); - let jwt_local_url = format!("http://localhost:{}/jwt_signature_public_keys", host_port); + let jwt_local_url = format!("http://127.0.0.1:{}/jwt_signature_public_keys", host_port); tracing::info!( "OIDC provider container is running, jwt signature pk url: {}", @@ -476,7 +476,7 @@ impl<'a> Datastore<'a> { let host_port = container.get_host_port_ipv4(Self::CONTAINER_PORT); let full_address = format!("http://{}:{}/", ip_address, Self::CONTAINER_PORT); - let local_address = format!("http://localhost:{}/", host_port); + let local_address = format!("http://127.0.0.1:{}/", host_port); tracing::info!("Datastore container is running at {}", full_address); Ok(Datastore { container, @@ -559,7 +559,7 @@ impl SignerNode<'_> { Ok(SignerNode { container, address: full_address, - local_address: format!("http://localhost:{host_port}"), + local_address: format!("http://127.0.0.1:{host_port}"), env: ctx.env.clone(), node_id, @@ -687,7 +687,7 @@ impl<'a> LeaderNode<'a> { Ok(LeaderNode { container, address: full_address, - local_address: format!("http://localhost:{host_port}"), + local_address: format!("http://127.0.0.1:{host_port}"), local_rpc_url: ctx.relayer_ctx.sandbox.local_address.clone(), local_relayer_url: ctx.relayer_ctx.relayer.local_address.clone(), }) diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index 1ad3afcb0..e24198b2e 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -54,7 +54,7 @@ impl SignerNode { let sign_node_id = format!("sign/{node_id}"); let process = util::spawn_mpc(ctx.release, &sign_node_id, &args)?; - let address = format!("http://localhost:{web_port}"); + let address = format!("http://127.0.0.1:{web_port}"); tracing::info!("Signer node is starting at {}", address); util::ping_until_ok(&address, 60).await?; tracing::info!("Signer node started [node_id={node_id}, {address}]"); @@ -130,7 +130,7 @@ impl LeaderNode { .into_str_args(); let process = util::spawn_mpc(ctx.release, "leader", &args)?; - let address = format!("http://localhost:{web_port}"); + let address = format!("http://127.0.0.1:{web_port}"); tracing::info!("Leader node container is starting at {}", address); util::ping_until_ok(&address, 60).await?; tracing::info!("Leader node running at {address}"); diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs index 649094248..cf79c3c1a 100644 --- a/integration-tests/src/lib.rs +++ b/integration-tests/src/lib.rs @@ -74,7 +74,7 @@ pub async fn initialize_relayer<'a>( tracing::info!("Initializing sandbox worker..."); let worker = workspaces::sandbox() .rpc_addr(&format!( - "http://localhost:{}", + "http://127.0.0.1:{}", sandbox .container .get_host_port_ipv4(crate::containers::Sandbox::CONTAINER_RPC_PORT) From c7d3837cb56c6f507133c363da1452c5aeb3471e Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Mon, 16 Oct 2023 11:14:41 -0700 Subject: [PATCH 19/21] Separate out docker image dep --- .github/workflows/docker-image.yml | 36 +++++++++++++++++++ .../{build-and-test.yml => integrations.yml} | 29 --------------- integration-tests/Cargo.toml | 3 +- 3 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/docker-image.yml rename .github/workflows/{build-and-test.yml => integrations.yml} (68%) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 000000000..dd78c67d0 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,36 @@ +name: Docker Image + +on: + push: + branches: + - develop + pull_request: + +jobs: + build-image: + runs-on: ubuntu-latest + name: Build and Push + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Login to GCP Artifact Registry + run: echo "$GOOGLE_CREDENTIALS" | docker login -u _json_key --password-stdin https://us-east1-docker.pkg.dev + env: + GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build Docker image + uses: docker/build-push-action@v4 + with: + context: . + tags: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Push Docker image + run: docker push us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/integrations.yml similarity index 68% rename from .github/workflows/build-and-test.yml rename to .github/workflows/integrations.yml index cf42c4f2e..9745351f1 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/integrations.yml @@ -7,36 +7,7 @@ on: pull_request: jobs: - build-image: - runs-on: ubuntu-latest - name: Build and Push - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Login to GCP Artifact Registry - run: echo "$GOOGLE_CREDENTIALS" | docker login -u _json_key --password-stdin https://us-east1-docker.pkg.dev - env: - GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build Docker image - uses: docker/build-push-action@v4 - with: - context: . - tags: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Push Docker image - run: docker push us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} - integrations: - needs: build-image name: Integration strategy: matrix: diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 38c6ab60f..207a67a40 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -41,6 +41,5 @@ tokio-util = { version = "0.7", features = ["full"] } reqwest = "0.11.16" [features] -default = ["drop-containers"] -drop-containers = [] +default = [] docker-test = [] From 6c56325a832aaf52b548561b2cbf263ce9034772 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Mon, 16 Oct 2023 11:25:04 -0700 Subject: [PATCH 20/21] Rename local_url to pk_local_url --- integration-tests/src/env/containers.rs | 4 ++-- integration-tests/src/env/local.rs | 4 ++-- integration-tests/src/main.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integration-tests/src/env/containers.rs b/integration-tests/src/env/containers.rs index e78601e0c..6cacd50bc 100644 --- a/integration-tests/src/env/containers.rs +++ b/integration-tests/src/env/containers.rs @@ -387,7 +387,7 @@ impl<'a> Relayer<'a> { pub struct OidcProvider<'a> { pub container: Container<'a, GenericImage>, pub jwt_pk_url: String, - pub jwt_local_url: String, + pub jwt_pk_local_url: String, } impl<'a> OidcProvider<'a> { @@ -422,7 +422,7 @@ impl<'a> OidcProvider<'a> { Ok(OidcProvider { container, jwt_pk_url, - jwt_local_url, + jwt_pk_local_url: jwt_local_url, }) } } diff --git a/integration-tests/src/env/local.rs b/integration-tests/src/env/local.rs index e24198b2e..4b3f146f8 100644 --- a/integration-tests/src/env/local.rs +++ b/integration-tests/src/env/local.rs @@ -48,7 +48,7 @@ impl SignerNode { ), gcp_project_id: ctx.gcp_project_id.clone(), gcp_datastore_url: Some(ctx.datastore.local_address.clone()), - jwt_signature_pk_url: ctx.oidc_provider.jwt_local_url.clone(), + jwt_signature_pk_url: ctx.oidc_provider.jwt_pk_local_url.clone(), } .into_str_args(); @@ -125,7 +125,7 @@ impl LeaderNode { ), gcp_project_id: ctx.gcp_project_id.clone(), gcp_datastore_url: Some(ctx.datastore.local_address.clone()), - jwt_signature_pk_url: ctx.oidc_provider.jwt_local_url.clone(), + jwt_signature_pk_url: ctx.oidc_provider.jwt_pk_local_url.clone(), } .into_str_args(); diff --git a/integration-tests/src/main.rs b/integration-tests/src/main.rs index 562ed6463..7f26becb4 100644 --- a/integration-tests/src/main.rs +++ b/integration-tests/src/main.rs @@ -35,7 +35,7 @@ async fn main() -> anyhow::Result<()> { println!(" root: {}", ctx.relayer_ctx.worker.root_account()?.id()); println!("\nExternal services:"); - println!(" oidc-provider: {}", ctx.oidc_provider.jwt_local_url); + println!(" oidc-provider: {}", ctx.oidc_provider.jwt_pk_local_url); println!(" datastore: {}", nodes.datastore_addr()); println!(" sandbox: {}", ctx.relayer_ctx.sandbox.local_address); println!(" relayer: {}", ctx.relayer_ctx.relayer.local_address); From 0b925a85113c76e7f1d862ee68dd06ae08cbfb90 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Mon, 16 Oct 2023 12:52:56 -0700 Subject: [PATCH 21/21] Move mpc-recovery build step --- .github/workflows/integrations.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 9745351f1..c23e3e67a 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -47,10 +47,6 @@ jobs: env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} - - name: Build MPC Recovery Binary Locally - run: | - cargo build --release - - name: Install stable toolchain uses: actions-rs/toolchain@v1 with: @@ -64,6 +60,10 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build MPC Recovery Binary Locally + run: | + cargo build -p mpc-recovery --release + - name: Test run: cargo test -p mpc-recovery-integration-tests --jobs 1 -- --test-threads 1 env: