feat: add persistent secret storage

contract/Cargo.toml

@@ -7,7 +7,8 @@ edition = "2021"
 crate-type = ["cdylib", "lib"]
 
 [dependencies]
-near-sdk = "4.1.1"
+borsh = "1.3.0"
+near-sdk = "5.0.0-alpha.1"
 serde = { version = "1", features = ["derive"] }
 schemars = "0.8"
 

contract/src/lib.rs

@@ -64,6 +64,7 @@ pub struct ResharingContractState {
 
 #[derive(BorshDeserialize, BorshSerialize, Serialize, Deserialize, Debug)]
 pub enum ProtocolContractState {
+    NotInitialized,
     Initializing(InitializingContractState),
     Running(RunningContractState),
     Resharing(ResharingContractState),
@@ -77,7 +78,6 @@ pub struct MpcContract {
 
 #[near_bindgen]
 impl MpcContract {
-    // TODO: only add ignore_state for dev environments
     #[init(ignore_state)]
     pub fn init(threshold: usize, participants: BTreeMap<AccountId, ParticipantInfo>) -> Self {
         MpcContract {
@@ -296,4 +296,15 @@ impl MpcContract {
 
     #[allow(unused_variables)]
     pub fn respond(&mut self, receipt_id: [u8; 32], big_r: String, s: String) {}
+
+    #[private]
+    #[init(ignore_state)]
+    pub fn clean(keys: Vec<near_sdk::json_types::Base64VecU8>) -> Self {
+        for key in keys.iter() {
+            env::storage_remove(&key.0);
+        }
+        Self {
+            protocol_state: ProtocolContractState::NotInitialized,
+        }
+    }
 }

infra/modules/multichain/main.tf

@@ -101,6 +101,14 @@ resource "google_cloud_run_v2_service" "node" {
         name  = "AWS_DEFAULT_REGION"
         value = var.indexer_options.s3_region
       }
+      env {
+        name  = "MPC_RECOVERY_GCP_PROJECT_ID"
+        value = var.project
+      }
+      env {
+        name  = "MPC_RECOVERY_SK_SHARE_SECRET_ID"
+        value = var.sk_share_secret_id
+      }
 
       env {
         name  = "MPC_RECOVERY_WEB_PORT"

infra/modules/multichain/variables.tf

@@ -1,3 +1,7 @@
+variable "project" {
+  type = string
+}
+
 variable "region" {
 }
 
@@ -67,3 +71,7 @@ variable "aws_access_key_secret_id" {
 variable "aws_secret_key_secret_id" {
   type = string
 }
+
+variable "sk_share_secret_id" {
+  type = string
+}

infra/multichain-dev/main.tf

@@ -82,11 +82,28 @@ resource "google_secret_manager_secret_iam_member" "aws_secret_key_secret_access
   member    = "serviceAccount:${google_service_account.service_account.email}"
 }
 
+resource "google_secret_manager_secret_iam_member" "sk_share_secret_access" {
+  count = length(var.node_configs)
+
+  secret_id = var.node_configs[count.index].sk_share_secret_id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.service_account.email}"
+}
+
+resource "google_secret_manager_secret_iam_member" "sk_share_secret_manager" {
+  count = length(var.node_configs)
+
+  secret_id = var.node_configs[count.index].sk_share_secret_id
+  role      = "roles/secretmanager.secretVersionManager"
+  member    = "serviceAccount:${google_service_account.service_account.email}"
+}
+
 module "node" {
   count  = length(var.node_configs)
   source = "../modules/multichain"
 
   service_name          = "multichain-${var.env}-${count.index}"
+  project               = var.project
   region                = var.region
   service_account_email = google_service_account.service_account.email
   docker_image          = var.docker_image
@@ -103,11 +120,14 @@ module "node" {
   cipher_sk_secret_id      = var.node_configs[count.index].cipher_sk_secret_id
   aws_access_key_secret_id = var.aws_access_key_secret_id
   aws_secret_key_secret_id = var.aws_secret_key_secret_id
+  sk_share_secret_id       = var.node_configs[count.index].sk_share_secret_id
 
   depends_on = [
     google_secret_manager_secret_iam_member.account_sk_secret_access,
     google_secret_manager_secret_iam_member.cipher_sk_secret_access,
     google_secret_manager_secret_iam_member.aws_access_key_secret_access,
-    google_secret_manager_secret_iam_member.aws_secret_key_secret_access
+    google_secret_manager_secret_iam_member.aws_secret_key_secret_access,
+    google_secret_manager_secret_iam_member.sk_share_secret_access,
+    google_secret_manager_secret_iam_member.sk_share_secret_manager
   ]
 }

infra/multichain-dev/terraform-dev.tfvars

@@ -7,7 +7,7 @@ indexer_options = {
   s3_bucket          = "near-lake-data-testnet"
   s3_region          = "eu-central-1"
   s3_url             = null
-  start_block_height = 149685857
+  start_block_height = 152754054
 }
 
 aws_access_key_secret_id = "multichain-indexer-aws-access-key"
@@ -19,19 +19,22 @@ node_configs = [
     address              = "https://multichain-dev-0-7tk2cmmtcq-ue.a.run.app"
     account_sk_secret_id = "multichain-account-sk-dev-0"
     cipher_sk_secret_id  = "multichain-cipher-sk-dev-0"
+    sk_share_secret_id   = "multichain-sk-share-dev-0"
   },
   {
     account              = "multichain-node-dev-1.testnet"
     cipher_pk            = "349f89f6717a02aaf7a649b98ac7af09e6517ffd4cb7ea8a9f6edee8f84a330c"
     address              = "https://multichain-dev-1-7tk2cmmtcq-ue.a.run.app"
     account_sk_secret_id = "multichain-account-sk-dev-1"
     cipher_sk_secret_id  = "multichain-cipher-sk-dev-1"
+    sk_share_secret_id   = "multichain-sk-share-dev-1"
   },
   {
     account              = "multichain-node-dev-2.testnet"
     cipher_pk            = "78ae67d38afaa6d329bba0175c200a8c248a5a82d78fbb8f71e060e6c186d800"
     address              = "https://multichain-dev-2-7tk2cmmtcq-ue.a.run.app"
     account_sk_secret_id = "multichain-account-sk-dev-2"
     cipher_sk_secret_id  = "multichain-cipher-sk-dev-2"
+    sk_share_secret_id   = "multichain-sk-share-dev-2"
   }
 ]

infra/multichain-dev/variables.tf

@@ -44,6 +44,7 @@ variable "node_configs" {
     address              = string
     account_sk_secret_id = string
     cipher_sk_secret_id  = string
+    sk_share_secret_id   = string
   }))
 }
 

integration-tests/Cargo.toml

@@ -32,7 +32,7 @@ serde_json = "1"
 testcontainers = { version = "0.15", features = ["experimental"] }
 tokio = { version = "1.28", features = ["full"] }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-near-workspaces = "0.8.0"
+near-workspaces = { git = "https://github.com/near/near-workspaces-rs.git", branch = "main" }
 toml = "0.8.1"
 sha2 = "0.10.8"
 
@@ -42,7 +42,6 @@ near-jsonrpc-client = "0.6"
 near-primitives = "0.17"
 near-lake-framework = { git = "https://github.com/near/near-lake-framework-rs.git", branch = "daniyar/upgrade-sdk" }
 near-lake-primitives = { git = "https://github.com/near/near-lake-framework-rs.git", branch = "daniyar/upgrade-sdk" }
-near-units = "0.2.0"
 
 mpc-contract = { path = "../contract" }
 mpc-keys = { path = "../keys" }

integration-tests/src/lib.rs

@@ -1,14 +1,14 @@
 use bollard::exec::{CreateExecOptions, StartExecResults};
 use futures::StreamExt;
 use near_crypto::KeyFile;
-use near_units::parse_near;
 use near_workspaces::{
     network::{Sandbox, ValidatorKey},
     types::SecretKey,
     Account, Worker,
 };
 
 use crate::env::containers::{self, LocalStack};
+use near_workspaces::types::NearToken;
 use testcontainers::{Container, GenericImage};
 
 pub mod env;
@@ -156,13 +156,15 @@ pub async fn initialize_relayer<'a>(
     sandbox::initialize_linkdrop(&worker).await?;
     tracing::info!("Initializing relayer accounts...");
     let relayer_account =
-        sandbox::create_account(&worker, "relayer", parse_near!("1000 N")).await?;
+        sandbox::create_account(&worker, "relayer", NearToken::from_near(1000)).await?;
     let relayer_account_keys = sandbox::gen_rotating_keys(&relayer_account, 5).await?;
 
-    let creator_account = sandbox::create_account(&worker, "creator", parse_near!("200 N")).await?;
+    let creator_account =
+        sandbox::create_account(&worker, "creator", NearToken::from_near(200)).await?;
     let creator_account_keys = sandbox::gen_rotating_keys(&creator_account, 5).await?;
 
-    let social_account = sandbox::create_account(&worker, "social", parse_near!("1000 N")).await?;
+    let social_account =
+        sandbox::create_account(&worker, "social", NearToken::from_near(1000)).await?;
     tracing::info!(
         "Relayer accounts initialized. Relayer account: {}, Creator account: {}, Social account: {}",
         relayer_account.id(),

integration-tests/src/multichain/containers.rs

@@ -54,6 +54,10 @@ impl<'a> Node<'a> {
                 start_block_height: 0,
             },
             my_address: None,
+            storage_options: mpc_recovery_node::storage::Options {
+                gcp_project_id: None,
+                sk_share_secret_id: None,
+            },
         }
         .into_str_args();
         let image: GenericImage = GenericImage::new("near/mpc-recovery-node", "latest")

integration-tests/src/multichain/local.rs

@@ -42,6 +42,10 @@ impl Node {
                 start_block_height: 0,
             },
             my_address: None,
+            storage_options: mpc_recovery_node::storage::Options {
+                gcp_project_id: None,
+                sk_share_secret_id: None,
+            },
         };
 
         let mpc_node_id = format!("multichain/{node_id}");

integration-tests/src/sandbox.rs

@@ -1,3 +1,4 @@
+use near_workspaces::types::NearToken;
 use near_workspaces::{network::Sandbox, types::SecretKey, AccessKey, Account, Contract, Worker};
 
 const BATCH_COUNT_LIMIT: usize = 100;
@@ -41,7 +42,7 @@ pub async fn initialize_linkdrop(worker: &Worker<Sandbox>) -> anyhow::Result<()>
 pub async fn create_account(
     worker: &Worker<Sandbox>,
     prefix: &str,
-    initial_balance: u128,
+    initial_balance: NearToken,
 ) -> anyhow::Result<Account> {
     tracing::info!("Creating account with random account_id...");
     let new_account = worker

node/Cargo.toml

@@ -19,6 +19,7 @@ cait-sith = { git = "https://github.com/LIT-Protocol/cait-sith.git", features =
     "k256",
 ] }
 clap = { version = "4.2", features = ["derive", "env"] }
+google-secretmanager1 = "5"
 hex = "0.4.3"
 hkdf = "0.12.4"
 k256 = { version = "0.13.1", features = ["sha256", "ecdsa", "serde"] }
@@ -40,7 +41,7 @@ near-fetch = "0.0.12"
 near-lake-framework = { git = "https://github.com/near/near-lake-framework-rs.git", branch = "daniyar/upgrade-sdk" }
 near-lake-primitives = { git = "https://github.com/near/near-lake-framework-rs.git", branch = "daniyar/upgrade-sdk" }
 near-primitives = "0.17"
-near-sdk = "4.1.1"
+near-sdk = "5.0.0-alpha.1"
 
 mpc-contract = { path = "../contract" }
 mpc-keys = { path = "../keys" }

node/src/cli.rs

@@ -1,5 +1,5 @@
 use crate::protocol::{MpcSignProtocol, SignQueue};
-use crate::{indexer, web};
+use crate::{indexer, storage, web};
 use cait_sith::protocol::Participant;
 use clap::Parser;
 use local_ip_address::local_ip;
@@ -51,6 +51,9 @@ pub enum Cli {
         /// Local address that other peers can use to message this node.
         #[arg(long, env("MPC_RECOVERY_LOCAL_ADDRESS"))]
         my_address: Option<Url>,
+        /// Storage options
+        #[clap(flatten)]
+        storage_options: storage::Options,
     },
 }
 
@@ -73,6 +76,7 @@ impl Cli {
                 cipher_sk,
                 indexer_options,
                 my_address,
+                storage_options,
             } => {
                 let mut args = vec![
                     "start".to_string(),
@@ -97,6 +101,7 @@ impl Cli {
                     args.extend(vec!["--my-address".to_string(), my_address.to_string()]);
                 }
                 args.extend(indexer_options.into_str_args());
+                args.extend(storage_options.into_str_args());
                 args
             }
         }
@@ -128,6 +133,7 @@ pub fn run(cmd: Cli) -> anyhow::Result<()> {
             cipher_sk,
             indexer_options,
             my_address,
+            storage_options,
         } => {
             let sign_queue = Arc::new(RwLock::new(SignQueue::new()));
             let a = indexer_options.clone();
@@ -141,6 +147,7 @@ pub fn run(cmd: Cli) -> anyhow::Result<()> {
                 .build()?
                 .block_on(async {
                     let (sender, receiver) = mpsc::channel(16384);
+                    let key_storage = storage::init(&storage_options).await?;
 
                     let my_address = my_address.unwrap_or_else(|| {
                         let my_ip = local_ip().unwrap();
@@ -159,6 +166,7 @@ pub fn run(cmd: Cli) -> anyhow::Result<()> {
                         receiver,
                         sign_queue.clone(),
                         hpke::PublicKey::try_from_bytes(&hex::decode(cipher_pk)?)?,
+                        key_storage,
                     );
                     tracing::debug!("protocol initialized");
                     let protocol_handle = tokio::spawn(async move { protocol.run().await });

node/src/indexer.rs

@@ -7,8 +7,9 @@ use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
-/// Configures exporter of span and trace data.
+/// Configures indexer.
 #[derive(Debug, Clone, clap::Parser)]
+#[group(id = "indexer_options")]
 pub struct Options {
     /// AWS S3 bucket name for NEAR Lake Indexer
     #[clap(

node/src/lib.rs

@@ -4,6 +4,7 @@ pub mod indexer;
 pub mod kdf;
 pub mod protocol;
 pub mod rpc_client;
+pub mod storage;
 pub mod types;
 pub mod util;
 pub mod web;

node/src/protocol/consensus.rs

@@ -8,7 +8,8 @@ use crate::protocol::presignature::PresignatureManager;
 use crate::protocol::signature::SignatureManager;
 use crate::protocol::state::{GeneratingState, ResharingState};
 use crate::protocol::triple::TripleManager;
-use crate::types::PrivateKeyShare;
+use crate::storage::{SecretNodeStorageBox, SecretStorageError};
+use crate::types::SecretKeyShare;
 use crate::util::AffinePointExt;
 use crate::{http_client, rpc_client};
 use async_trait::async_trait;
@@ -34,6 +35,7 @@ pub trait ConsensusCtx {
     fn cipher_pk(&self) -> &hpke::PublicKey;
     fn sign_pk(&self) -> near_crypto::PublicKey;
     fn sign_sk(&self) -> &near_crypto::SecretKey;
+    fn secret_storage(&self) -> &SecretNodeStorageBox;
 }
 
 #[derive(thiserror::Error, Debug)]
@@ -52,6 +54,8 @@ pub enum ConsensusError {
     HasBeenKicked,
     #[error("cait-sith initialization error: {0}")]
     CaitSithInitializationError(#[from] InitializationError),
+    #[error("secret storage error: {0}")]
+    SecretStorageError(#[from] SecretStorageError),
 }
 
 #[async_trait]
@@ -607,8 +611,8 @@ impl ConsensusProtocol for NodeState {
     ) -> Result<NodeState, ConsensusError> {
         match self {
             NodeState::Starting => {
-                // TODO: Load from persistent storage
-                Ok(NodeState::Started(StartedState(None)))
+                let node_data = ctx.secret_storage().load().await?;
+                Ok(NodeState::Started(StartedState(node_data)))
             }
             NodeState::Started(state) => state.advance(ctx, contract_state).await,
             NodeState::Generating(state) => state.advance(ctx, contract_state).await,
@@ -621,7 +625,7 @@ impl ConsensusProtocol for NodeState {
 }
 
 fn start_resharing<C: ConsensusCtx>(
-    private_share: Option<PrivateKeyShare>,
+    private_share: Option<SecretKeyShare>,
     ctx: C,
     contract_state: ResharingContractState,
 ) -> Result<NodeState, ConsensusError> {

node/src/protocol/contract.rs

@@ -180,6 +180,7 @@ impl TryFrom<ProtocolContractState> for ProtocolState {
 
     fn try_from(value: ProtocolContractState) -> Result<Self, Self::Error> {
         match value {
+            ProtocolContractState::NotInitialized => Err(()),
             ProtocolContractState::Initializing(state) => {
                 Ok(ProtocolState::Initializing(state.into()))
             }